From f817923d2f76bde8dce54700b75d2584b6d28b39 Mon Sep 17 00:00:00 2001 From: Swaroop Sridhar Date: Wed, 11 Mar 2020 20:58:50 -0700 Subject: [PATCH 01/18] Single-File: Process bundles in the framework This change implements the host changes proposed in the [design](https://github.com/dotnet/designs/blob/master/accepted/2020/single-file/design.md#startup) The main changes for single-file bundles are: * Bundle processing code is moved from apphost to hostpolicy * HostFxr and HostPolicy process deps.json and runtimeconfig.json files directly from the bundle. * HostPolicy performs verification wrt deps.json based on the contents of the single-file bundle. * AppContext.BaseDirectory is set as explained [here](https://github.com/dotnet/designs/blob/master/accepted/2020/single-file/design.md#appcontextbasedirectory) Currently, all files except deps.json and runtimeconfig.json are extracted to disk. Once the runtime is able to processing assemblies directly from the bundle, they will no longer be extracted. Notable details: * The bundle driver (formarly runner.cpp) is divided into two parts: * bundle::info which describes basic information about the bundle available from the headers only * bundle::runner which has information about all embedded files, and the ability to drive extraction This facilitates linking only parts of the bundle handling code with hostfxr, while all code is linked with hostpolicy. * The AppHost only links with bundle_marker to identify itself as a single-file bundle. * If the AppHost is a single-file bundle, it notifies hostfxr using the new hostfxr_main_bundle_startup_info() API * The HostFxr comminucates the single-file-information with HostPolicy using the host_interface_t structure. Fixes https://github.com/dotnet/runtime/issues/32821 --- .../corehost/cli/apphost/CMakeLists.txt | 19 +-- .../corehost/cli/apphost/bundle/runner.cpp | 65 -------- .../corehost/cli/apphost/bundle/runner.h | 40 ----- .../{bundle/marker.cpp => bundle_marker.cpp} | 8 +- .../{bundle/marker.h => bundle_marker.h} | 12 +- .../cli/{apphost => }/bundle/dir_utils.cpp | 0 .../cli/{apphost => }/bundle/dir_utils.h | 0 .../cli/{apphost => }/bundle/extractor.cpp | 10 +- .../cli/{apphost => }/bundle/extractor.h | 0 .../cli/{apphost => }/bundle/file_entry.cpp | 15 ++ .../cli/{apphost => }/bundle/file_entry.h | 1 + .../cli/{apphost => }/bundle/file_type.h | 0 .../cli/{apphost => }/bundle/header.cpp | 28 ++-- .../cli/{apphost => }/bundle/header.h | 28 ++-- src/installer/corehost/cli/bundle/info.cpp | 145 ++++++++++++++++++ src/installer/corehost/cli/bundle/info.h | 100 ++++++++++++ .../cli/{apphost => }/bundle/manifest.cpp | 4 +- .../cli/{apphost => }/bundle/manifest.h | 8 + .../cli/{apphost => }/bundle/reader.cpp | 5 +- .../cli/{apphost => }/bundle/reader.h | 5 +- src/installer/corehost/cli/bundle/runner.cpp | 79 ++++++++++ src/installer/corehost/cli/bundle/runner.h | 51 ++++++ src/installer/corehost/cli/deps_entry.cpp | 144 +++++++++++++---- src/installer/corehost/cli/deps_entry.h | 13 +- src/installer/corehost/cli/deps_format.cpp | 7 +- .../corehost/cli/fxr/command_line.cpp | 3 +- .../corehost/cli/fxr/corehost_init.cpp | 3 + .../corehost/cli/fxr/corehost_init.h | 3 +- src/installer/corehost/cli/fxr/fx_muxer.cpp | 8 + src/installer/corehost/cli/fxr/hostfxr.cpp | 18 +++ .../corehost/cli/fxr/hostpolicy_resolver.cpp | 5 +- src/installer/corehost/cli/host_interface.h | 5 +- .../corehost/cli/host_startup_info.h | 1 + .../corehost/cli/hostcommon/CMakeLists.txt | 6 + src/installer/corehost/cli/hostfxr.h | 7 + src/installer/corehost/cli/hostmisc/pal.h | 7 +- .../corehost/cli/hostmisc/pal.unix.cpp | 17 +- .../corehost/cli/hostmisc/pal.windows.cpp | 17 +- src/installer/corehost/cli/hostmisc/utils.cpp | 19 ++- src/installer/corehost/cli/hostmisc/utils.h | 2 + .../corehost/cli/hostpolicy/CMakeLists.txt | 10 ++ .../corehost/cli/hostpolicy/args.cpp | 37 ++++- .../corehost/cli/hostpolicy/deps_resolver.cpp | 6 + .../corehost/cli/hostpolicy/deps_resolver.h | 12 ++ .../corehost/cli/hostpolicy/hostpolicy.cpp | 15 +- .../cli/hostpolicy/hostpolicy_init.cpp | 7 + .../corehost/cli/hostpolicy/hostpolicy_init.h | 1 + .../corehost/cli/ijwhost/ijwthunk.cpp | 6 +- .../cli/json/rapidjson/cursorstreamwrapper.h | 2 +- .../corehost/cli/json/rapidjson/document.h | 4 +- src/installer/corehost/cli/json_parser.cpp | 75 +++++++-- src/installer/corehost/cli/json_parser.h | 14 +- src/installer/corehost/cli/runtime_config.cpp | 7 +- .../test/mockhostpolicy/mockhostpolicy.cpp | 1 + src/installer/corehost/corehost.cpp | 105 ++++++++----- .../Bundle/Manifest.cs | 2 +- 56 files changed, 906 insertions(+), 306 deletions(-) delete mode 100644 src/installer/corehost/cli/apphost/bundle/runner.cpp delete mode 100644 src/installer/corehost/cli/apphost/bundle/runner.h rename src/installer/corehost/cli/apphost/{bundle/marker.cpp => bundle_marker.cpp} (86%) rename src/installer/corehost/cli/apphost/{bundle/marker.h => bundle_marker.h} (83%) rename src/installer/corehost/cli/{apphost => }/bundle/dir_utils.cpp (100%) rename src/installer/corehost/cli/{apphost => }/bundle/dir_utils.h (100%) rename src/installer/corehost/cli/{apphost => }/bundle/extractor.cpp (98%) rename src/installer/corehost/cli/{apphost => }/bundle/extractor.h (100%) rename src/installer/corehost/cli/{apphost => }/bundle/file_entry.cpp (77%) rename src/installer/corehost/cli/{apphost => }/bundle/file_entry.h (98%) rename src/installer/corehost/cli/{apphost => }/bundle/file_type.h (100%) rename src/installer/corehost/cli/{apphost => }/bundle/header.cpp (50%) rename src/installer/corehost/cli/{apphost => }/bundle/header.h (74%) create mode 100644 src/installer/corehost/cli/bundle/info.cpp create mode 100644 src/installer/corehost/cli/bundle/info.h rename src/installer/corehost/cli/{apphost => }/bundle/manifest.cpp (72%) rename src/installer/corehost/cli/{apphost => }/bundle/manifest.h (76%) rename src/installer/corehost/cli/{apphost => }/bundle/reader.cpp (93%) rename src/installer/corehost/cli/{apphost => }/bundle/reader.h (90%) create mode 100644 src/installer/corehost/cli/bundle/runner.cpp create mode 100644 src/installer/corehost/cli/bundle/runner.h diff --git a/src/installer/corehost/cli/apphost/CMakeLists.txt b/src/installer/corehost/cli/apphost/CMakeLists.txt index d77f2bfa9811e8..ba01e32ec47c74 100644 --- a/src/installer/corehost/cli/apphost/CMakeLists.txt +++ b/src/installer/corehost/cli/apphost/CMakeLists.txt @@ -18,26 +18,11 @@ endif() set(SKIP_VERSIONING 1) set(SOURCES - ./bundle/file_entry.cpp - ./bundle/manifest.cpp - ./bundle/header.cpp - ./bundle/marker.cpp - ./bundle/reader.cpp - ./bundle/extractor.cpp - ./bundle/runner.cpp - ./bundle/dir_utils.cpp + ./bundle_marker.cpp ) set(HEADERS - ./bundle/file_type.h - ./bundle/file_entry.h - ./bundle/manifest.h - ./bundle/header.h - ./bundle/marker.h - ./bundle/reader.h - ./bundle/extractor.h - ./bundle/runner.h - ./bundle/dir_utils.h + ./bundle_marker.h ) if(CLR_CMAKE_TARGET_WIN32) diff --git a/src/installer/corehost/cli/apphost/bundle/runner.cpp b/src/installer/corehost/cli/apphost/bundle/runner.cpp deleted file mode 100644 index 50d74235d7a604..00000000000000 --- a/src/installer/corehost/cli/apphost/bundle/runner.cpp +++ /dev/null @@ -1,65 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -#include -#include "extractor.h" -#include "runner.h" -#include "trace.h" -#include "header.h" -#include "marker.h" -#include "manifest.h" - -using namespace bundle; - -void runner_t::map_host() -{ - m_bundle_map = (int8_t *) pal::map_file_readonly(m_bundle_path, m_bundle_length); - - if (m_bundle_map == nullptr) - { - trace::error(_X("Failure processing application bundle.")); - trace::error(_X("Couldn't memory map the bundle file for reading.")); - throw StatusCode::BundleExtractionIOError; - } -} - -void runner_t::unmap_host() -{ - if (!pal::unmap_file(m_bundle_map, m_bundle_length)) - { - trace::warning(_X("Failed to unmap bundle after extraction.")); - } -} - -// Current support for executing single-file bundles involves -// extraction of embedded files to actual files on disk. -// This method implements the file extraction functionality at startup. -StatusCode runner_t::extract() -{ - try - { - map_host(); - reader_t reader(m_bundle_map, m_bundle_length); - - // Read the bundle header - reader.set_offset(marker_t::header_offset()); - header_t header = header_t::read(reader, /* need_exact_version: */ true); - - // Read the bundle manifest - // Reader is at the correct offset - manifest_t manifest = manifest_t::read(reader, header.num_embedded_files()); - - // Extract the files - extractor_t extractor(header.bundle_id(), m_bundle_path, manifest); - m_extraction_dir = extractor.extract(reader); - - unmap_host(); - return StatusCode::Success; - } - catch (StatusCode e) - { - return e; - } -} - diff --git a/src/installer/corehost/cli/apphost/bundle/runner.h b/src/installer/corehost/cli/apphost/bundle/runner.h deleted file mode 100644 index 07dadede0fd611..00000000000000 --- a/src/installer/corehost/cli/apphost/bundle/runner.h +++ /dev/null @@ -1,40 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -#ifndef __RUNNER_H__ -#define __RUNNER_H__ - -#include "error_codes.h" - -namespace bundle -{ - class runner_t - { - public: - runner_t(const pal::string_t& bundle_path) - : m_bundle_path(bundle_path) - , m_bundle_map(nullptr) - , m_bundle_length(0) - { - } - - StatusCode extract(); - - pal::string_t extraction_dir() - { - return m_extraction_dir; - } - - private: - void map_host(); - void unmap_host(); - - pal::string_t m_bundle_path; - pal::string_t m_extraction_dir; - int8_t* m_bundle_map; - size_t m_bundle_length; - }; -} - -#endif // __RUNNER_H__ diff --git a/src/installer/corehost/cli/apphost/bundle/marker.cpp b/src/installer/corehost/cli/apphost/bundle_marker.cpp similarity index 86% rename from src/installer/corehost/cli/apphost/bundle/marker.cpp rename to src/installer/corehost/cli/apphost/bundle_marker.cpp index d23029232261c3..d8ea15bdd4f569 100644 --- a/src/installer/corehost/cli/apphost/bundle/marker.cpp +++ b/src/installer/corehost/cli/apphost/bundle_marker.cpp @@ -2,14 +2,12 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -#include "marker.h" +#include "bundle_marker.h" #include "pal.h" #include "trace.h" #include "utils.h" -using namespace bundle; - -int64_t marker_t::header_offset() +int64_t bundle_marker_t::header_offset() { // Contains the bundle_placeholder default value at compile time. // If this is a single-file bundle, the last 8 bytes are replaced @@ -27,7 +25,7 @@ int64_t marker_t::header_offset() 0xee, 0x3b, 0x2d, 0xce, 0x24, 0xb3, 0x6a, 0xae }; - volatile marker_t* marker = reinterpret_cast(placeholder); + volatile bundle_marker_t* marker = reinterpret_cast(placeholder); return marker->locator.bundle_header_offset; } diff --git a/src/installer/corehost/cli/apphost/bundle/marker.h b/src/installer/corehost/cli/apphost/bundle_marker.h similarity index 83% rename from src/installer/corehost/cli/apphost/bundle/marker.h rename to src/installer/corehost/cli/apphost/bundle_marker.h index 52e185fbe7920d..c5065fdac19be8 100644 --- a/src/installer/corehost/cli/apphost/bundle/marker.h +++ b/src/installer/corehost/cli/apphost/bundle_marker.h @@ -2,15 +2,13 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -#ifndef __MARKER_H__ -#define __MARKER_H__ +#ifndef __BUNDLE_MARKER_H__ +#define __BUNDLE_MARKER_H__ #include -namespace bundle -{ #pragma pack(push, 1) - union marker_t + union bundle_marker_t { public: uint8_t placeholder[40]; @@ -28,5 +26,5 @@ namespace bundle }; #pragma pack(pop) -} -#endif // __MARKER_H__ + +#endif // __BUNDLE_MARKER_H__ diff --git a/src/installer/corehost/cli/apphost/bundle/dir_utils.cpp b/src/installer/corehost/cli/bundle/dir_utils.cpp similarity index 100% rename from src/installer/corehost/cli/apphost/bundle/dir_utils.cpp rename to src/installer/corehost/cli/bundle/dir_utils.cpp diff --git a/src/installer/corehost/cli/apphost/bundle/dir_utils.h b/src/installer/corehost/cli/bundle/dir_utils.h similarity index 100% rename from src/installer/corehost/cli/apphost/bundle/dir_utils.h rename to src/installer/corehost/cli/bundle/dir_utils.h diff --git a/src/installer/corehost/cli/apphost/bundle/extractor.cpp b/src/installer/corehost/cli/bundle/extractor.cpp similarity index 98% rename from src/installer/corehost/cli/apphost/bundle/extractor.cpp rename to src/installer/corehost/cli/bundle/extractor.cpp index 71bb2259e9d11a..04c5205da9d537 100644 --- a/src/installer/corehost/cli/apphost/bundle/extractor.cpp +++ b/src/installer/corehost/cli/bundle/extractor.cpp @@ -197,7 +197,10 @@ void extractor_t::extract_new(reader_t& reader) begin(); for (const file_entry_t& entry : m_manifest.files) { - extract(entry, reader); + if (entry.needs_extraction()) + { + extract(entry, reader); + } } commit_dir(); } @@ -211,6 +214,11 @@ void extractor_t::verify_recover_extraction(reader_t& reader) for (const file_entry_t& entry : m_manifest.files) { + if (!entry.needs_extraction()) + { + continue; + } + pal::string_t file_path = ext_dir; append_path(&file_path, entry.relative_path().c_str()); diff --git a/src/installer/corehost/cli/apphost/bundle/extractor.h b/src/installer/corehost/cli/bundle/extractor.h similarity index 100% rename from src/installer/corehost/cli/apphost/bundle/extractor.h rename to src/installer/corehost/cli/bundle/extractor.h diff --git a/src/installer/corehost/cli/apphost/bundle/file_entry.cpp b/src/installer/corehost/cli/bundle/file_entry.cpp similarity index 77% rename from src/installer/corehost/cli/apphost/bundle/file_entry.cpp rename to src/installer/corehost/cli/bundle/file_entry.cpp index ba4b235fa93b2c..775117fb9a1f6d 100644 --- a/src/installer/corehost/cli/apphost/bundle/file_entry.cpp +++ b/src/installer/corehost/cli/bundle/file_entry.cpp @@ -33,3 +33,18 @@ file_entry_t file_entry_t::read(reader_t &reader) return entry; } + +bool file_entry_t::needs_extraction() const +{ + switch (m_type) + { + // Once the runtime can load assemblies from bundle, + // file_type_t::assembly should be in this category + case file_type_t::deps_json: + case file_type_t::runtime_config_json: + return false; + + default: + return true; + } +} diff --git a/src/installer/corehost/cli/apphost/bundle/file_entry.h b/src/installer/corehost/cli/bundle/file_entry.h similarity index 98% rename from src/installer/corehost/cli/apphost/bundle/file_entry.h rename to src/installer/corehost/cli/bundle/file_entry.h index efe405cc9a6cc1..c89d3ce0a5fde2 100644 --- a/src/installer/corehost/cli/apphost/bundle/file_entry.h +++ b/src/installer/corehost/cli/bundle/file_entry.h @@ -58,6 +58,7 @@ namespace bundle int64_t offset() const { return m_offset; } int64_t size() const { return m_size; } file_type_t type() const { return m_type; } + bool needs_extraction() const; static file_entry_t read(reader_t &reader); diff --git a/src/installer/corehost/cli/apphost/bundle/file_type.h b/src/installer/corehost/cli/bundle/file_type.h similarity index 100% rename from src/installer/corehost/cli/apphost/bundle/file_type.h rename to src/installer/corehost/cli/bundle/file_type.h diff --git a/src/installer/corehost/cli/apphost/bundle/header.cpp b/src/installer/corehost/cli/bundle/header.cpp similarity index 50% rename from src/installer/corehost/cli/apphost/bundle/header.cpp rename to src/installer/corehost/cli/bundle/header.cpp index 23695bb93f11b1..73c166232c9acd 100644 --- a/src/installer/corehost/cli/apphost/bundle/header.cpp +++ b/src/installer/corehost/cli/bundle/header.cpp @@ -9,29 +9,23 @@ using namespace bundle; -// The AppHost expects the bundle_header to be an exact_match for which it was built. -// The framework accepts backwards compatible header versions. -bool header_fixed_t::is_valid(bool exact_match) const +bool header_fixed_t::is_valid() const { if (num_embedded_files <= 0) { return false; } - if (exact_match) - { - return (major_version == header_t::major_version) && (minor_version == header_t::minor_version); - } - - return ((major_version < header_t::major_version) || - (major_version == header_t::major_version && minor_version <= header_t::minor_version)); + // .net 5 host expects the version information to be 2.0 + // .net core 3 single-file bundles are handled within the netcoreapp3.x apphost, and are not processed here in the framework. + return (major_version == header_t::major_version) && (minor_version == header_t::minor_version); } -header_t header_t::read(reader_t& reader, bool need_exact_version) +header_t header_t::read(reader_t& reader) { const header_fixed_t* fixed_header = reinterpret_cast(reader.read_direct(sizeof(header_fixed_t))); - if (!fixed_header->is_valid(need_exact_version)) + if (!fixed_header->is_valid()) { trace::error(_X("Failure processing application bundle.")); trace::error(_X("Bundle header version compatibility check failed.")); @@ -42,12 +36,12 @@ header_t header_t::read(reader_t& reader, bool need_exact_version) header_t header(fixed_header->num_embedded_files); // bundle_id is a component of the extraction path - reader.read_path_string(header.m_bundle_id); + size_t bundle_id_size = reader.read_path_string(header.m_bundle_id); - if (fixed_header->major_version > 1) - { - header.m_v2_header = reinterpret_cast(reader.read_direct(sizeof(header_fixed_v2_t))); - } + const header_fixed_v2_t *v2_header = reinterpret_cast(reader.read_direct(sizeof(header_fixed_v2_t))); + header.m_v2_header = *v2_header; + + header.m_header_size = sizeof(header_fixed_t) + bundle_id_size + sizeof(header_fixed_v2_t); return header; } diff --git a/src/installer/corehost/cli/apphost/bundle/header.h b/src/installer/corehost/cli/bundle/header.h similarity index 74% rename from src/installer/corehost/cli/apphost/bundle/header.h rename to src/installer/corehost/cli/bundle/header.h index 1b1696219dc8b6..f9dbe6b94b0852 100644 --- a/src/installer/corehost/cli/apphost/bundle/header.h +++ b/src/installer/corehost/cli/bundle/header.h @@ -32,9 +32,8 @@ namespace bundle uint32_t minor_version; int32_t num_embedded_files; - bool is_valid(bool exact_match = false) const; + bool is_valid() const; }; -#pragma pack(pop) // netcoreapp3_compat_mode flag is set on a .net5 app, which chooses to build single-file apps in .netcore3.x compat mode, // This indicates that: @@ -46,12 +45,13 @@ namespace bundle netcoreapp3_compat_mode = 1 }; -#pragma pack(push, 1) struct location_t { public: int64_t offset; int64_t size; + + bool is_valid() const { return offset != 0; } }; // header_fixed_v2_t is available in single-file apps targetting .net5+ frameworks. @@ -65,6 +65,8 @@ namespace bundle location_t deps_json_location; location_t runtimeconfig_json_location; header_flags_t flags; + + bool is_netcoreapp3_compat_mode() const { return (flags & header_flags_t::netcoreapp3_compat_mode) != 0; } }; #pragma pack(pop) @@ -74,14 +76,20 @@ namespace bundle header_t(int32_t num_embedded_files = 0) : m_num_embedded_files(num_embedded_files) , m_bundle_id() - , m_v2_header(NULL) - + , m_v2_header() + , m_header_size(0) { } - static header_t read(reader_t& reader, bool need_exact_version); - const pal::string_t& bundle_id() { return m_bundle_id; } - int32_t num_embedded_files() { return m_num_embedded_files; } + static header_t read(reader_t& reader); + const pal::string_t& bundle_id() const { return m_bundle_id; } + int32_t num_embedded_files() const { return m_num_embedded_files; } + + const location_t& deps_json_location() const { return m_v2_header.deps_json_location; } + const location_t& runtimeconfig_json_location() const { return m_v2_header.runtimeconfig_json_location; } + bool is_netcoreapp3_compat_mode() const { return m_v2_header.is_netcoreapp3_compat_mode(); } + + size_t size() const { return m_header_size; } static const uint32_t major_version = 2; static const uint32_t minor_version = 0; @@ -89,8 +97,8 @@ namespace bundle private: int32_t m_num_embedded_files; pal::string_t m_bundle_id; - const header_fixed_v2_t* m_v2_header; - + header_fixed_v2_t m_v2_header; + size_t m_header_size; }; } #endif // __HEADER_H__ diff --git a/src/installer/corehost/cli/bundle/info.cpp b/src/installer/corehost/cli/bundle/info.cpp new file mode 100644 index 00000000000000..706f9f79d4e042 --- /dev/null +++ b/src/installer/corehost/cli/bundle/info.cpp @@ -0,0 +1,145 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +#include "trace.h" +#include "info.h" +#include "utils.h" + +using namespace bundle; + +// Global single-file bundle information, if any +const info_t* info_t::the_app = nullptr; + +StatusCode info_t::process_bundle(const pal::char_t* bundle_path, const pal::char_t* app_path, int64_t header_offset) +{ + if (header_offset == 0) + { + // Not a single-file bundle. + return StatusCode::Success; + } + + static info_t info(bundle_path, header_offset); + StatusCode status = info.process_header(); + + if (status != StatusCode::Success) + { + return status; + } + + info.init_config(app_path); + + trace::info(_X("Single-File bundle details:")); + trace::info(_X("DepsJson Offset:[%lx] Size[%lx]"), info.m_header.deps_json_location().offset, info.m_header.deps_json_location().size); + trace::info(_X("RuntimeConfigJson Offset:[%lx] Size[%lx]"), info.m_header.runtimeconfig_json_location().offset, info.m_header.runtimeconfig_json_location().size); + trace::info(_X(".net core 3 compatibility mode: [%s]"), info.m_header.is_netcoreapp3_compat_mode() ? _X("Yes") : _X("No")); + + the_app = &info; + + return StatusCode::Success; +} + +StatusCode info_t::process_header() +{ + try + { + const int8_t* addr = map_bundle(); + + reader_t reader(addr, m_bundle_size, m_header_offset); + + m_header = header_t::read(reader); + + unmap_bundle(addr); + + return StatusCode::Success; + } + catch (StatusCode e) + { + return e; + } +} + +void info_t::init_config(const pal::string_t& app_path) +{ + // Single-file bundles currently only support deps/runtime config json files + // named based on the app.dll. Any other name for these configuration files + // mentioned via the command line are assumed to be actual files on disk. + // + // Supporting custom names for these config files is straightforward (with associated changes in bundler and SDK). + // There is no known use-case for it yet, and the facility is TBD. + + m_base_path = get_directory(m_bundle_path); + pal::string_t deps_json_name = get_deps_from_app_binary(m_base_path, app_path); + pal::string_t runtimeconfig_json_name = get_runtime_config_path(m_base_path, get_filename_without_ext(app_path)); + + m_deps_json = config_t(deps_json_name, &m_header.deps_json_location()); + m_runtimeconfig_json = config_t(runtimeconfig_json_name, &m_header.runtimeconfig_json_location()); +} + +const int8_t* info_t::config_t::map(const pal::string_t& path, const location_t* &location) +{ + const bundle::info_t* app = bundle::info_t::the_app; + + if (app->m_deps_json.matches(path)) + { + location = app->m_deps_json.m_location; + } + else if (app->m_runtimeconfig_json.matches(path)) + { + location = app->m_runtimeconfig_json.m_location; + } + else + { + return nullptr; + } + + // When necessary to map the deps.json or runtimeconfig.json files, we map the whole single-file bundle, + // and return the address at the appropriate offset. + // This is because: + // * The host is the only code that is currently running, and attempting to map the bundle at this time + // * Files can only be memory mapped at page-aligned offsets, and in whole page units. + // Therefore, mapping only portions of the bundle will involve align-down/round-up calculations, and associated offset adjustments. + // We choose the simpler approach of rounding to the whole file + // * There is no performance limitation due to a larger sized mapping, since we actually only read the pages with relevant contents. + + const int8_t* addr = (const int8_t*)pal::mmap_read(app->m_bundle_path); + if (addr == nullptr) + { + trace::error(_X("Failure processing application bundle.")); + trace::error(_X("Failed to map bundle file [%s]"), path.c_str()); + } + + return addr + location->offset; +} + +void info_t::config_t::unmap(const int8_t* addr, const location_t* location) +{ + // Adjust to the beginning of the bundle. + addr -= location->offset; + + bundle::info_t::the_app->unmap_bundle(addr); +} + +const int8_t* info_t::map_bundle() +{ + void *addr = pal::mmap_read(m_bundle_path, &m_bundle_size); + + if (addr == nullptr) + { + trace::error(_X("Failure processing application bundle.")); + trace::error(_X("Couldn't memory map the bundle file for reading.")); + throw StatusCode::BundleExtractionIOError; + } + + return (int8_t *)addr; +} + +void info_t::unmap_bundle(const int8_t* addr) const +{ + if (!pal::munmap((void*)addr, m_bundle_size)) + { + trace::warning(_X("Failed to unmap bundle after extraction.")); + } +} + + diff --git a/src/installer/corehost/cli/bundle/info.h b/src/installer/corehost/cli/bundle/info.h new file mode 100644 index 00000000000000..60982315160df0 --- /dev/null +++ b/src/installer/corehost/cli/bundle/info.h @@ -0,0 +1,100 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +#ifndef __INFO_H_ +#define __INFO_H_ + +#include "error_codes.h" +#include "header.h" + +// bundle::info supports: +// * API for identification of a single-file app bundle, and +// * Minimal probing and mapping functionality only for the app.runtimeconfig.json and app.deps.json files. +// bundle::info is used by HostFxr to read the above config files. + +namespace bundle +{ + struct info_t + { + struct config_t + { + config_t() {} + + config_t(const config_t& config) + { + m_path = config.m_path; + m_location = config.m_location; + } + + config_t(const pal::string_t& path, const location_t *location) + { + m_path = path; + m_location = location; + } + + bool matches(const pal::string_t& path) const + { + return m_location->is_valid() && path.compare(m_path) == 0; + } + + static bool probe(const pal::string_t& path) + { + return bundle::info_t::the_app->m_deps_json.matches(path) || + bundle::info_t::the_app->m_runtimeconfig_json.matches(path); + } + + static const int8_t* map(const pal::string_t& path, const location_t* &location); + static void unmap(const int8_t* addr, const location_t* location); + + private: + pal::string_t m_path; + const location_t *m_location; + } json_info; + + static StatusCode process_bundle(const pal::char_t* bundle_path, const pal::char_t *app_path, int64_t header_offset); + static bool is_single_file_bundle() { return the_app != nullptr; } + + bool is_netcoreapp3_compat_mode() const { return m_header.is_netcoreapp3_compat_mode(); } + const pal::string_t& base_path() const { return m_base_path; } + + // Global single-file info object + static const info_t* the_app; + + protected: + info_t(const pal::char_t* bundle_path_value, + int64_t header_offset_value) + : m_bundle_path(bundle_path_value) + , m_bundle_size(0) + , m_header_offset(header_offset_value) + , m_deps_json() + , m_runtimeconfig_json() {} + + info_t(const info_t* info) + { + m_bundle_path = info->m_bundle_path; + m_base_path = info->m_base_path; + m_bundle_size = info->m_bundle_size; + m_header_offset = info->m_header_offset; + m_header = info->m_header; + m_deps_json = info->m_deps_json; + m_runtimeconfig_json = info->m_runtimeconfig_json; + } + + const int8_t* map_bundle(); + void unmap_bundle(const int8_t* addr) const; + + pal::string_t m_bundle_path; + pal::string_t m_base_path; + size_t m_bundle_size; + int64_t m_header_offset; + header_t m_header; + config_t m_deps_json; + config_t m_runtimeconfig_json; + + private: + void init_config(const pal::string_t& app_path); + StatusCode process_header(); + }; +} +#endif // __INFO_H_ diff --git a/src/installer/corehost/cli/apphost/bundle/manifest.cpp b/src/installer/corehost/cli/bundle/manifest.cpp similarity index 72% rename from src/installer/corehost/cli/apphost/bundle/manifest.cpp rename to src/installer/corehost/cli/bundle/manifest.cpp index 6de65d3a96b45a..991c8e8408ca65 100644 --- a/src/installer/corehost/cli/apphost/bundle/manifest.cpp +++ b/src/installer/corehost/cli/bundle/manifest.cpp @@ -12,7 +12,9 @@ manifest_t manifest_t::read(reader_t& reader, int32_t num_files) for (int32_t i = 0; i < num_files; i++) { - manifest.files.emplace_back(file_entry_t::read(reader)); + file_entry_t entry = file_entry_t::read(reader); + manifest.files.emplace_back(entry); + manifest.m_need_extraction |= entry.needs_extraction(); } return manifest; diff --git a/src/installer/corehost/cli/apphost/bundle/manifest.h b/src/installer/corehost/cli/bundle/manifest.h similarity index 76% rename from src/installer/corehost/cli/apphost/bundle/manifest.h rename to src/installer/corehost/cli/bundle/manifest.h index ee8cd1edab5caf..af1f9fa9198057 100644 --- a/src/installer/corehost/cli/apphost/bundle/manifest.h +++ b/src/installer/corehost/cli/bundle/manifest.h @@ -16,9 +16,17 @@ namespace bundle class manifest_t { public: + manifest_t() + : m_need_extraction(false) {} + std::vector files; static manifest_t read(reader_t &reader, int32_t num_files); + + bool files_need_extraction() { return m_need_extraction; } + + private: + bool m_need_extraction; }; } #endif // __MANIFEST_H__ diff --git a/src/installer/corehost/cli/apphost/bundle/reader.cpp b/src/installer/corehost/cli/bundle/reader.cpp similarity index 93% rename from src/installer/corehost/cli/apphost/bundle/reader.cpp rename to src/installer/corehost/cli/bundle/reader.cpp index e2fa9b6574c8d4..26c306d60c4daa 100644 --- a/src/installer/corehost/cli/apphost/bundle/reader.cpp +++ b/src/installer/corehost/cli/bundle/reader.cpp @@ -88,11 +88,14 @@ size_t reader_t::read_path_length() return length; } -void reader_t::read_path_string(pal::string_t &str) +size_t reader_t::read_path_string(pal::string_t &str) { + const int8_t* start_ptr = m_ptr; size_t size = read_path_length(); std::unique_ptr buffer{ new uint8_t[size + 1] }; read(buffer.get(), size); buffer[size] = 0; // null-terminator pal::clr_palstring(reinterpret_cast(buffer.get()), &str); + + return m_ptr - start_ptr; // This subtraction can't overflow because addition above is bounds_checked } diff --git a/src/installer/corehost/cli/apphost/bundle/reader.h b/src/installer/corehost/cli/bundle/reader.h similarity index 90% rename from src/installer/corehost/cli/apphost/bundle/reader.h rename to src/installer/corehost/cli/bundle/reader.h index 1824ece515aae1..38eaf37561ec4f 100644 --- a/src/installer/corehost/cli/apphost/bundle/reader.h +++ b/src/installer/corehost/cli/bundle/reader.h @@ -13,12 +13,13 @@ namespace bundle // Helper class for reading sequentially from the memory-mapped bundle file. struct reader_t { - reader_t(const int8_t* base_ptr, int64_t bound) + reader_t(const int8_t* base_ptr, int64_t bound, int64_t start_offset = 0) : m_base_ptr(base_ptr) , m_ptr(base_ptr) , m_bound(bound) , m_bound_ptr(add_without_overflow(base_ptr, bound)) { + set_offset(start_offset); } public: @@ -55,7 +56,7 @@ namespace bundle } size_t read_path_length(); - void read_path_string(pal::string_t &str); + size_t read_path_string(pal::string_t &str); private: diff --git a/src/installer/corehost/cli/bundle/runner.cpp b/src/installer/corehost/cli/bundle/runner.cpp new file mode 100644 index 00000000000000..8b8fd58a1d51f0 --- /dev/null +++ b/src/installer/corehost/cli/bundle/runner.cpp @@ -0,0 +1,79 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +#include +#include "extractor.h" +#include "runner.h" +#include "trace.h" +#include "header.h" +#include "manifest.h" +#include "utils.h" + +using namespace bundle; + +// This method processes the bundle manifest. +// It also implements the extraction of files that cannot be directly processed from the bundle. +StatusCode runner_t::extract() +{ + try + { + const int8_t* addr = map_bundle(); + + // Set the Reader offset to read post the bundle header + reader_t reader(addr, m_bundle_size, m_header_offset + m_header.size()); + + // Read the bundle manifest + m_manifest = manifest_t::read(reader, m_header.num_embedded_files()); + + // Extract the files if necessary + if (m_manifest.files_need_extraction()) + { + extractor_t extractor(m_header.bundle_id(), m_bundle_path, m_manifest); + m_extraction_path = extractor.extract(reader); + } + + unmap_bundle(addr); + + return StatusCode::Success; + } + catch (StatusCode e) + { + return e; + } +} + +const file_entry_t* runner_t::probe(const pal::string_t& path) const +{ + for (const file_entry_t& entry : m_manifest.files) + { + if (entry.relative_path() == path) + { + return &entry; + } + } + + return nullptr; +} + +bool runner_t::locate(const pal::string_t& relative_path, pal::string_t& full_path) const +{ + const bundle::runner_t* app = bundle::runner_t::app(); + const bundle::file_entry_t* entry = app->probe(relative_path); + + if (entry == nullptr) + { + return false; + } + + bool needs_extraction = entry->needs_extraction(); + pal::string_t file_base = entry->needs_extraction() ? app->extraction_path() : app->base_path(); + + // Reserve space for the path below + full_path.reserve(file_base.length() + relative_path.length() + 3); + + full_path.assign(file_base); + append_path(&full_path, relative_path.c_str()); + + return true; +} diff --git a/src/installer/corehost/cli/bundle/runner.h b/src/installer/corehost/cli/bundle/runner.h new file mode 100644 index 00000000000000..a3873c33410b1a --- /dev/null +++ b/src/installer/corehost/cli/bundle/runner.h @@ -0,0 +1,51 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +#ifndef __RUNNER_H__ +#define __RUNNER_H__ + +#include "error_codes.h" +#include "header.h" +#include "manifest.h" +#include "info.h" + +// bundle::runner extends bundle::info to supports: +// * Reading the bundle manifest and identifying file locations for the runtime +// * Extracting bundled files to disk when necessary +// bundle::runner is used by HostPolicy. + +namespace bundle +{ + class runner_t : public info_t + { + public: + runner_t(const pal::char_t* bundle_path_value, + int64_t header_offset_value) + : info_t(bundle_path_value, header_offset_value) {} + + runner_t(const bundle::info_t* info) + : info_t(info) {} + + const pal::string_t& extraction_path() const { return m_extraction_path; } + + const file_entry_t *probe(const pal::string_t& path) const; + bool locate(const pal::string_t& relative_path, pal::string_t& full_path) const; + + static StatusCode process_manifest_and_extract() + { + return ((runner_t*) the_app)->extract(); + } + + static const runner_t* app() { return (const runner_t*)the_app; } + + private: + + StatusCode extract(); + + manifest_t m_manifest; + pal::string_t m_extraction_path; + }; +} + +#endif // __RUNNER_H__ diff --git a/src/installer/corehost/cli/deps_entry.cpp b/src/installer/corehost/cli/deps_entry.cpp index 08d196c17419c6..dd25735811d6af 100644 --- a/src/installer/corehost/cli/deps_entry.cpp +++ b/src/installer/corehost/cli/deps_entry.cpp @@ -6,7 +6,45 @@ #include "utils.h" #include "deps_entry.h" #include "trace.h" +#include "bundle/runner.h" +static pal::string_t normalize_dir_separator(const pal::string_t& path) +{ + // Entry relative path contains '/' separator, sanitize it to use + // platform separator. Perf: avoid extra copy if it matters. + pal::string_t normalized_path = path; + if (_X('/') != DIR_SEPARATOR) + { + replace_char(&normalized_path, _X('/'), DIR_SEPARATOR); + } + + return normalized_path; +} + +void deps_entry_t::append_resource_path(pal::string_t& base) const +{ + assert(asset_type == asset_types::resources); + + pal::string_t pal_relative_path = asset.relative_path; + if (_X('/') != DIR_SEPARATOR) + { + replace_char(&pal_relative_path, _X('/'), DIR_SEPARATOR); + } + + // Resources are represented as "lib///" in the deps.json. + // The is the "directory" in the pal_relative_path below, so extract it. + pal::string_t ietf_dir = get_directory(pal_relative_path); + pal::string_t ietf = ietf_dir; + + // get_directory returns with DIR_SEPARATOR appended that we need to remove. + remove_trailing_dir_seperator(&ietf); + + // Extract IETF code from "lib//" + ietf = get_filename(ietf); + + append_path(&base, ietf.c_str()); + trace::verbose(_X("Detected a resource asset, will query dir/ietf-tag/resource base: %s asset: %s"), base.c_str(), asset.name.c_str()); +} bool deps_entry_t::to_path(const pal::string_t& base, bool look_in_base, pal::string_t* str) const { @@ -20,17 +58,10 @@ bool deps_entry_t::to_path(const pal::string_t& base, bool look_in_base, pal::st return false; } - // Entry relative path contains '/' separator, sanitize it to use - // platform separator. Perf: avoid extra copy if it matters. - pal::string_t pal_relative_path = asset.relative_path; - if (_X('/') != DIR_SEPARATOR) - { - replace_char(&pal_relative_path, _X('/'), DIR_SEPARATOR); - } + pal::string_t pal_relative_path = normalize_dir_separator(asset.relative_path); // Reserve space for the path below - candidate.reserve(base.length() + - pal_relative_path.length() + 3); + candidate.reserve(base.length() + pal_relative_path.length() + 3); candidate.assign(base); pal::string_t sub_path = look_in_base ? get_filename(pal_relative_path) : pal_relative_path; @@ -47,6 +78,7 @@ bool deps_entry_t::to_path(const pal::string_t& base, bool look_in_base, pal::st { trace::verbose(_X(" %s path query exists %s"), query_type, candidate.c_str()); } + return exists; } @@ -55,8 +87,7 @@ bool deps_entry_t::to_path(const pal::string_t& base, bool look_in_base, pal::st // // Parameters: // base - The base directory to look for the relative path of this entry -// str - If the method returns true, contains the file path for this deps -// entry relative to the "base" directory +// str - If the method returns true, contains the file path for this deps entry // // Returns: // If the file exists in the path relative to the "base" directory. @@ -65,38 +96,22 @@ bool deps_entry_t::to_dir_path(const pal::string_t& base, pal::string_t* str) co { if (asset_type == asset_types::resources) { - pal::string_t pal_relative_path = asset.relative_path; - if (_X('/') != DIR_SEPARATOR) - { - replace_char(&pal_relative_path, _X('/'), DIR_SEPARATOR); - } - - // Resources are represented as "lib///" in the deps.json. - // The is the "directory" in the pal_relative_path below, so extract it. - pal::string_t ietf_dir = get_directory(pal_relative_path); - pal::string_t ietf = ietf_dir; - - // get_directory returns with DIR_SEPARATOR appended that we need to remove. - remove_trailing_dir_seperator(&ietf); - - // Extract IETF code from "lib//" - ietf = get_filename(ietf); - pal::string_t base_ietf_dir = base; - append_path(&base_ietf_dir, ietf.c_str()); - trace::verbose(_X("Detected a resource asset, will query dir/ietf-tag/resource base: %s asset: %s"), base_ietf_dir.c_str(), asset.name.c_str()); + append_resource_path(base_ietf_dir); + return to_path(base_ietf_dir, true, str); } + return to_path(base, true, str); } + // ----------------------------------------------------------------------------- // Given a "base" directory, yield the relative path of this file in the package // layout. // // Parameters: // base - The base directory to look for the relative path of this entry -// str - If the method returns true, contains the file path for this deps -// entry relative to the "base" directory +// str - If the method returns true, contains the file path for this deps entry // // Returns: // If the file exists in the path relative to the "base" directory. @@ -112,8 +127,7 @@ bool deps_entry_t::to_rel_path(const pal::string_t& base, pal::string_t* str) co // // Parameters: // base - The base directory to look for the relative path of this entry -// str - If the method returns true, contains the file path for this deps -// entry relative to the "base" directory +// str - If the method returns true, contains the file path for this deps entry // // Returns: // If the file exists in the path relative to the "base" directory. @@ -142,3 +156,65 @@ bool deps_entry_t::to_full_path(const pal::string_t& base, pal::string_t* str) c return to_rel_path(new_base, str); } + +// ----------------------------------------------------------------------------- +// Given a "base" directory, if this file exists within the single-file bundle, +// return +// * If the file was extracted to disk, the full-path to the extracted file. +// * Otherwise, the path within the bundle, relative to the "base" directory. +// The runtime expects the entries in the TPA, NativeDllSearchDirectories, etc +// to be absolute paths. Therefore, the relative-paths within the bundle are +// expressed as absolute paths with respect to the location of the bundle-file. +// +// Parameters: +// base - The directory containing the single-file bundle. +// str - If the method returns true, contains the file path for this deps entry +// +// Returns: +// If the file exists in the single-file bundle +// +bool deps_entry_t::to_bundle_path(const pal::string_t& base, pal::string_t* str) const +{ + if (!bundle::info_t::is_single_file_bundle()) + { + return false; + } + + const bundle::runner_t* app = bundle::runner_t::app(); + + // Bundled files are only searched relative to the app-directory. + if (base.compare(app->base_path()) != 0) + { + trace::verbose(_X(" Base directory %s is different from bundle-base %s"), base.c_str(), app->base_path().c_str()); + return false; + } + + pal::string_t& candidate = *str; + candidate.clear(); + + pal::string_t pal_relative_path = normalize_dir_separator(asset.relative_path); + pal::string_t bundle_path; + pal::string_t sub_path; + if (asset_type == asset_types::resources) + { + append_resource_path(bundle_path); + sub_path = get_filename(pal_relative_path); + } + else + { + sub_path = pal_relative_path; + } + + bool exists = app->locate(sub_path, candidate); + + if (exists) + { + trace::verbose(_X(" %s found in bundle [%s]"), sub_path.c_str(), candidate.c_str()); + } + else + { + trace::verbose(_X(" %s not found in bundle"), sub_path.c_str()); + } + + return exists; +} diff --git a/src/installer/corehost/cli/deps_entry.h b/src/installer/corehost/cli/deps_entry.h index d06e2988aa3256..1e0fa05e2e51d4 100644 --- a/src/installer/corehost/cli/deps_entry.h +++ b/src/installer/corehost/cli/deps_entry.h @@ -52,9 +52,6 @@ struct deps_entry_t bool is_serviceable; bool is_rid_specific; - // Given a "base" dir, yield the filepath within this directory or relative to this directory based on "look_in_base" - bool to_path(const pal::string_t& base, bool look_in_base, pal::string_t* str) const; - // Given a "base" dir, yield the file path within this directory. bool to_dir_path(const pal::string_t& base, pal::string_t* str) const; @@ -63,6 +60,16 @@ struct deps_entry_t // Given a "base" dir, yield the relative path with package name, version in the package layout. bool to_full_path(const pal::string_t& root, pal::string_t* str) const; + + // Given a "base" dir, yield the file path within the single-file bundle. + bool to_bundle_path(const pal::string_t& base, pal::string_t* str) const; + +private: + // Given a "base" dir, yield the filepath within this directory or relative to this directory based on "look_in_base" + bool to_path(const pal::string_t& base, bool look_in_base, pal::string_t* str) const; + + // For resource assets, append the IETF code to the base path + void append_resource_path(pal::string_t& base) const; }; #endif // __DEPS_ENTRY_H_ diff --git a/src/installer/corehost/cli/deps_format.cpp b/src/installer/corehost/cli/deps_format.cpp index b098d95302517d..873dd815c7adc3 100644 --- a/src/installer/corehost/cli/deps_format.cpp +++ b/src/installer/corehost/cli/deps_format.cpp @@ -6,6 +6,7 @@ #include "deps_format.h" #include "utils.h" #include "trace.h" +#include "bundle/info.h" #include #include #include @@ -426,16 +427,16 @@ bool deps_json_t::has_package(const pal::string_t& name, const pal::string_t& ve bool deps_json_t::load(bool is_framework_dependent, const pal::string_t& deps_path, const rid_fallback_graph_t& rid_fallback_graph) { m_deps_file = deps_path; - m_file_exists = pal::file_exists(deps_path); + m_file_exists = bundle::info_t::config_t::probe(deps_path) || pal::file_exists(deps_path); - // If file doesn't exist, then assume parsed. + json_parser_t json; if (!m_file_exists) { + // If file doesn't exist, then assume parsed. trace::verbose(_X("Could not locate the dependencies manifest file [%s]. Some libraries may fail to resolve."), deps_path.c_str()); return true; } - json_parser_t json; if (!json.parse_file(deps_path)) { return false; diff --git a/src/installer/corehost/cli/fxr/command_line.cpp b/src/installer/corehost/cli/fxr/command_line.cpp index e3563947c3c945..045f7700e74269 100644 --- a/src/installer/corehost/cli/fxr/command_line.cpp +++ b/src/installer/corehost/cli/fxr/command_line.cpp @@ -9,6 +9,7 @@ #include "sdk_info.h" #include #include +#include "bundle/info.h" namespace { @@ -144,7 +145,7 @@ namespace if (mode == host_mode_t::apphost) { app_candidate = host_info.app_path; - doesAppExist = pal::realpath(&app_candidate); + doesAppExist = bundle::info_t::is_single_file_bundle() || pal::realpath(&app_candidate); } else { diff --git a/src/installer/corehost/cli/fxr/corehost_init.cpp b/src/installer/corehost/cli/fxr/corehost_init.cpp index 22a3921f8e9892..c388c77e03ddeb 100644 --- a/src/installer/corehost/cli/fxr/corehost_init.cpp +++ b/src/installer/corehost/cli/fxr/corehost_init.cpp @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. #include "corehost_init.h" +#include "bundle/info.h" void make_cstr_arr(const std::vector& arr, std::vector* out) { @@ -132,6 +133,8 @@ const host_interface_t& corehost_init_t::get_host_init_data() hi.host_info_dotnet_root = m_host_info_dotnet_root.c_str(); hi.host_info_app_path = m_host_info_app_path.c_str(); + hi.app_bundle = bundle::info_t::the_app; + return hi; } diff --git a/src/installer/corehost/cli/fxr/corehost_init.h b/src/installer/corehost/cli/fxr/corehost_init.h index eedc2e70d2260f..0d7f87ae92f6d2 100644 --- a/src/installer/corehost/cli/fxr/corehost_init.h +++ b/src/installer/corehost/cli/fxr/corehost_init.h @@ -8,7 +8,7 @@ #include "host_interface.h" #include "host_startup_info.h" #include "fx_definition.h" - + class corehost_init_t { private: @@ -37,6 +37,7 @@ class corehost_init_t const pal::string_t m_host_info_host_path; const pal::string_t m_host_info_dotnet_root; const pal::string_t m_host_info_app_path; + public: corehost_init_t( const pal::string_t& host_command, diff --git a/src/installer/corehost/cli/fxr/fx_muxer.cpp b/src/installer/corehost/cli/fxr/fx_muxer.cpp index 00584e4a91ef4e..31c8c177978078 100644 --- a/src/installer/corehost/cli/fxr/fx_muxer.cpp +++ b/src/installer/corehost/cli/fxr/fx_muxer.cpp @@ -27,6 +27,7 @@ #include "sdk_info.h" #include "sdk_resolver.h" #include "roll_fwd_on_no_candidate_fx_option.h" +#include "bundle/info.h" namespace { @@ -262,6 +263,7 @@ namespace pal::string_t& runtime_config, const runtime_config_t::settings_t& override_settings) { + // Check for the runtimeconfig.json file specified at the command line if (!runtime_config.empty() && !pal::realpath(&runtime_config)) { trace::error(_X("The specified runtimeconfig.json [%s] does not exist"), runtime_config.c_str()); @@ -293,6 +295,11 @@ namespace host_mode_t detect_operating_mode(const host_startup_info_t& host_info) { + if (bundle::info_t::is_single_file_bundle()) + { + return host_mode_t::apphost; + } + if (coreclr_exists_in_dir(host_info.dotnet_root)) { // Detect between standalone apphost or legacy split mode (specifying --depsfile and --runtimeconfig) @@ -357,6 +364,7 @@ namespace { pal::string_t runtime_config = command_line::get_option_value(opts, known_options::runtime_config, _X("")); + // This check is for --depsfile option, which must be an actual file. pal::string_t deps_file = command_line::get_option_value(opts, known_options::deps_file, _X("")); if (!deps_file.empty() && !pal::realpath(&deps_file)) { diff --git a/src/installer/corehost/cli/fxr/hostfxr.cpp b/src/installer/corehost/cli/fxr/hostfxr.cpp index c9a346b799bcbf..1d165b949d963f 100644 --- a/src/installer/corehost/cli/fxr/hostfxr.cpp +++ b/src/installer/corehost/cli/fxr/hostfxr.cpp @@ -14,6 +14,7 @@ #include "sdk_resolver.h" #include "hostfxr.h" #include "host_context.h" +#include "bundle/info.h" namespace { @@ -24,6 +25,23 @@ namespace } } +SHARED_API int HOSTFXR_CALLTYPE hostfxr_main_bundle_startup_info(const int argc, const pal::char_t* argv[], const pal::char_t* host_path, const pal::char_t* dotnet_root, const pal::char_t* app_path, int64_t bundle_header_offset) +{ + trace_hostfxr_entry_point(_X("hostfxr_main_bundle_startup_info")); + + StatusCode bundleStatus = bundle::info_t::process_bundle(host_path, app_path, bundle_header_offset); + if (bundleStatus != StatusCode::Success) + { + trace::error(_X("A fatal error occured while processing application bundle")); + return bundleStatus; + } + + host_startup_info_t startup_info(host_path, dotnet_root, app_path); + + return fx_muxer_t::execute(pal::string_t(), argc, argv, startup_info, nullptr, 0, nullptr); +} + + SHARED_API int HOSTFXR_CALLTYPE hostfxr_main_startupinfo(const int argc, const pal::char_t* argv[], const pal::char_t* host_path, const pal::char_t* dotnet_root, const pal::char_t* app_path) { trace_hostfxr_entry_point(_X("hostfxr_main_startupinfo")); diff --git a/src/installer/corehost/cli/fxr/hostpolicy_resolver.cpp b/src/installer/corehost/cli/fxr/hostpolicy_resolver.cpp index 049d61a62a5c4d..12a04aa1a3a58b 100644 --- a/src/installer/corehost/cli/fxr/hostpolicy_resolver.cpp +++ b/src/installer/corehost/cli/fxr/hostpolicy_resolver.cpp @@ -12,6 +12,7 @@ #include #include "json_parser.h" +#include "bundle/info.h" namespace { @@ -29,7 +30,6 @@ namespace trace::verbose(_X("--- Resolving %s version from deps json [%s]"), LIBHOSTPOLICY_NAME, deps_json.c_str()); pal::string_t retval; - json_parser_t json; if (!json.parse_file(deps_json)) { @@ -238,7 +238,8 @@ bool hostpolicy_resolver::try_get_dir( // Resolve hostpolicy version out of the deps file. pal::string_t version = resolve_hostpolicy_version_from_deps(resolved_deps); - if (trace::is_enabled() && version.empty() && pal::file_exists(resolved_deps)) + if (trace::is_enabled() && version.empty() && + (bundle::info_t::config_t::probe(resolved_deps)|| pal::file_exists(resolved_deps))) { trace::warning(_X("Dependency manifest %s does not contain an entry for %s"), resolved_deps.c_str(), _STRINGIFY(HOST_POLICY_PKG_NAME)); diff --git a/src/installer/corehost/cli/host_interface.h b/src/installer/corehost/cli/host_interface.h index c7148e0ddc65af..34894b0b698665 100644 --- a/src/installer/corehost/cli/host_interface.h +++ b/src/installer/corehost/cli/host_interface.h @@ -7,6 +7,7 @@ #include #include "pal.h" +#include "bundle/info.h" enum host_mode_t { @@ -58,6 +59,7 @@ struct host_interface_t const pal::char_t* host_info_host_path; const pal::char_t* host_info_dotnet_root; const pal::char_t* host_info_app_path; + const bundle::info_t *app_bundle; // !! WARNING / WARNING / WARNING / WARNING / WARNING / WARNING / WARNING / WARNING / WARNING // !! 1. Only append to this structure to maintain compat. // !! 2. Any nested structs should not use compiler specific padding (pack with _HOST_INTERFACE_PACK) @@ -91,7 +93,8 @@ static_assert(offsetof(host_interface_t, host_command) == 26 * sizeof(size_t), " static_assert(offsetof(host_interface_t, host_info_host_path) == 27 * sizeof(size_t), "Struct offset breaks backwards compatibility"); static_assert(offsetof(host_interface_t, host_info_dotnet_root) == 28 * sizeof(size_t), "Struct offset breaks backwards compatibility"); static_assert(offsetof(host_interface_t, host_info_app_path) == 29 * sizeof(size_t), "Struct offset breaks backwards compatibility"); -static_assert(sizeof(host_interface_t) == 30 * sizeof(size_t), "Did you add static asserts for the newly added fields?"); +static_assert(offsetof(host_interface_t, app_bundle) == 30 * sizeof(size_t), "Struct offset breaks backwards compatibility"); +static_assert(sizeof(host_interface_t) == 31 * sizeof(size_t), "Did you add static asserts for the newly added fields?"); #define HOST_INTERFACE_LAYOUT_VERSION_HI 0x16041101 // YYMMDD:nn always increases when layout breaks compat. #define HOST_INTERFACE_LAYOUT_VERSION_LO sizeof(host_interface_t) diff --git a/src/installer/corehost/cli/host_startup_info.h b/src/installer/corehost/cli/host_startup_info.h index 4cb4b3e92b97c7..cdc2746021aefa 100644 --- a/src/installer/corehost/cli/host_startup_info.h +++ b/src/installer/corehost/cli/host_startup_info.h @@ -11,6 +11,7 @@ struct host_startup_info_t { host_startup_info_t() {} + host_startup_info_t( const pal::char_t* host_path_value, const pal::char_t* dotnet_root_value, diff --git a/src/installer/corehost/cli/hostcommon/CMakeLists.txt b/src/installer/corehost/cli/hostcommon/CMakeLists.txt index deac26114aa016..8ad0c8d09f47c2 100644 --- a/src/installer/corehost/cli/hostcommon/CMakeLists.txt +++ b/src/installer/corehost/cli/hostcommon/CMakeLists.txt @@ -23,6 +23,9 @@ set(SOURCES ../version.cpp ../version_compatibility_range.cpp ../runtime_config.cpp + ../bundle/info.cpp + ../bundle/reader.cpp + ../bundle/header.cpp ) set(HEADERS @@ -37,6 +40,9 @@ set(HEADERS ../version.h ../version_compatibility_range.h ../runtime_config.h + ../bundle/info.h + ../bundle/reader.h + ../bundle/header.h ) set(SKIP_VERSIONING 1) diff --git a/src/installer/corehost/cli/hostfxr.h b/src/installer/corehost/cli/hostfxr.h index 040b7c78d87b14..30a7e4a8b4e920 100644 --- a/src/installer/corehost/cli/hostfxr.h +++ b/src/installer/corehost/cli/hostfxr.h @@ -37,6 +37,13 @@ typedef int32_t(HOSTFXR_CALLTYPE *hostfxr_main_startupinfo_fn)( const char_t *host_path, const char_t *dotnet_root, const char_t *app_path); +typedef int32_t(HOSTFXR_CALLTYPE* hostfxr_main_bundle_startup_info_fn)( + const int argc, + const char_t** argv, + const char_t* host_path, + const char_t* dotnet_root, + const char_t* app_path, + int64_t bundle_header_offset); typedef void(HOSTFXR_CALLTYPE *hostfxr_error_writer_fn)(const char_t *message); typedef hostfxr_error_writer_fn(HOSTFXR_CALLTYPE *hostfxr_set_error_writer_fn)(hostfxr_error_writer_fn error_writer); diff --git a/src/installer/corehost/cli/hostmisc/pal.h b/src/installer/corehost/cli/hostmisc/pal.h index d713fc5b6640a2..007d76c5044fe7 100644 --- a/src/installer/corehost/cli/hostmisc/pal.h +++ b/src/installer/corehost/cli/hostmisc/pal.h @@ -164,7 +164,7 @@ namespace pal inline bool rmdir (const char_t* path) { return RemoveDirectoryW(path) != 0; } inline int rename(const char_t* old_name, const char_t* new_name) { return ::_wrename(old_name, new_name); } inline int remove(const char_t* path) { return ::_wremove(path); } - inline bool unmap_file(void* addr, size_t length) { return UnmapViewOfFile(addr) != 0; } + inline bool munmap(void* addr, size_t length) { return UnmapViewOfFile(addr) != 0; } inline int get_pid() { return GetCurrentProcessId(); } inline void sleep(uint32_t milliseconds) { Sleep(milliseconds); } #else @@ -221,7 +221,7 @@ namespace pal inline bool rmdir(const char_t* path) { return ::rmdir(path) == 0; } inline int rename(const char_t* old_name, const char_t* new_name) { return ::rename(old_name, new_name); } inline int remove(const char_t* path) { return ::remove(path); } - inline bool unmap_file(void* addr, size_t length) { return munmap(addr, length) == 0; } + inline bool munmap(void* addr, size_t length) { return ::munmap(addr, length) == 0; } inline int get_pid() { return getpid(); } inline void sleep(uint32_t milliseconds) { usleep(milliseconds * 1000); } @@ -256,7 +256,8 @@ namespace pal return fallbackRid; } - void* map_file_readonly(const string_t& path, size_t& length); + void* mmap_read(const string_t& path, size_t* length = nullptr); + bool touch_file(const string_t& path); bool realpath(string_t* path, bool skip_error_logging = false); bool file_exists(const string_t& path); diff --git a/src/installer/corehost/cli/hostmisc/pal.unix.cpp b/src/installer/corehost/cli/hostmisc/pal.unix.cpp index ba4fa60141c51d..b04802a3c61dbc 100644 --- a/src/installer/corehost/cli/hostmisc/pal.unix.cpp +++ b/src/installer/corehost/cli/hostmisc/pal.unix.cpp @@ -58,7 +58,7 @@ bool pal::touch_file(const pal::string_t& path) return true; } -void* pal::map_file_readonly(const pal::string_t& path, size_t& length) +void* pal::mmap_read(const string_t& path, size_t* length) { int fd = open(path.c_str(), O_RDONLY); if (fd == -1) @@ -74,11 +74,16 @@ void* pal::map_file_readonly(const pal::string_t& path, size_t& length) close(fd); return nullptr; } + size_t size = buf.st_size; - length = buf.st_size; - void* address = mmap(nullptr, length, PROT_READ, MAP_SHARED, fd, 0); + if (length != nullptr) + { + *length = size; + } + + void* address = mmap(nullptr, size, PROT_READ, MAP_SHARED, fd, 0); - if(address == nullptr) + if (address == nullptr) { trace::error(_X("Failed to map file. mmap(%s) failed with error %d"), path.c_str(), errno); close(fd); @@ -478,10 +483,10 @@ bool pal::get_default_installation_dir(pal::string_t* recv) pal::string_t trim_quotes(pal::string_t stringToCleanup) { pal::char_t quote_array[2] = {'\"', '\''}; - for(size_t index = 0; index < sizeof(quote_array)/sizeof(quote_array[0]); index++) + for (size_t index = 0; index < sizeof(quote_array)/sizeof(quote_array[0]); index++) { size_t pos = stringToCleanup.find(quote_array[index]); - while(pos != std::string::npos) + while (pos != std::string::npos) { stringToCleanup = stringToCleanup.erase(pos, 1); pos = stringToCleanup.find(quote_array[index]); diff --git a/src/installer/corehost/cli/hostmisc/pal.windows.cpp b/src/installer/corehost/cli/hostmisc/pal.windows.cpp index 0cb6d2f91d9e92..7ef85cc40d6d32 100644 --- a/src/installer/corehost/cli/hostmisc/pal.windows.cpp +++ b/src/installer/corehost/cli/hostmisc/pal.windows.cpp @@ -76,7 +76,7 @@ bool pal::touch_file(const pal::string_t& path) return true; } -void* pal::map_file_readonly(const pal::string_t& path, size_t &length) +void* pal::mmap_read(const string_t& path, size_t *length) { HANDLE file = CreateFileW(path.c_str(), GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); @@ -86,14 +86,17 @@ void* pal::map_file_readonly(const pal::string_t& path, size_t &length) return nullptr; } - LARGE_INTEGER fileSize; - if (GetFileSizeEx(file, &fileSize) == 0) + if (length != nullptr) { - trace::error(_X("Failed to map file. GetFileSizeEx(%s) failed with error %d"), path.c_str(), GetLastError()); - CloseHandle(file); - return nullptr; + LARGE_INTEGER fileSize; + if (GetFileSizeEx(file, &fileSize) == 0) + { + trace::error(_X("Failed to map file. GetFileSizeEx(%s) failed with error %d"), path.c_str(), GetLastError()); + CloseHandle(file); + return nullptr; + } + *length = (size_t)fileSize.QuadPart; } - length = (size_t)fileSize.QuadPart; HANDLE map = CreateFileMappingW(file, NULL, PAGE_READONLY, 0, 0, NULL); diff --git a/src/installer/corehost/cli/hostmisc/utils.cpp b/src/installer/corehost/cli/hostmisc/utils.cpp index 4e0d8ab3170659..4a3208af23751c 100644 --- a/src/installer/corehost/cli/hostmisc/utils.cpp +++ b/src/installer/corehost/cli/hostmisc/utils.cpp @@ -4,6 +4,7 @@ #include "utils.h" #include "trace.h" +#include "bundle/info.h" bool library_exists_in_dir(const pal::string_t& lib_dir, const pal::string_t& lib_name, pal::string_t* p_lib_path) { @@ -365,6 +366,7 @@ pal::string_t get_deps_from_app_binary(const pal::string_t& app_base, const pal: { pal::string_t deps_file; auto app_name = get_filename(app); + deps_file.reserve(app_base.length() + 1 + app_name.length() + 5); deps_file.append(app_base); @@ -377,19 +379,28 @@ pal::string_t get_deps_from_app_binary(const pal::string_t& app_base, const pal: return deps_file; } -void get_runtime_config_paths(const pal::string_t& path, const pal::string_t& name, pal::string_t* cfg, pal::string_t* dev_cfg) +pal::string_t get_runtime_config_path(const pal::string_t& path, const pal::string_t& name) { auto json_path = path; auto json_name = name + _X(".runtimeconfig.json"); append_path(&json_path, json_name.c_str()); - cfg->assign(json_path); + return json_path; +} +pal::string_t get_runtime_config_dev_path(const pal::string_t& path, const pal::string_t& name) +{ auto dev_json_path = path; auto dev_json_name = name + _X(".runtimeconfig.dev.json"); append_path(&dev_json_path, dev_json_name.c_str()); - dev_cfg->assign(dev_json_path); + return dev_json_path; +} + +void get_runtime_config_paths(const pal::string_t& path, const pal::string_t& name, pal::string_t* cfg, pal::string_t* dev_cfg) +{ + cfg->assign(get_runtime_config_path(path, name)); + dev_cfg->assign(get_runtime_config_dev_path(path, name)); - trace::verbose(_X("Runtime config is cfg=%s dev=%s"), json_path.c_str(), dev_json_path.c_str()); + trace::verbose(_X("Runtime config is cfg=%s dev=%s"), cfg->c_str(), dev_cfg->c_str()); } pal::string_t get_dotnet_root_from_fxr_path(const pal::string_t &fxr_path) diff --git a/src/installer/corehost/cli/hostmisc/utils.h b/src/installer/corehost/cli/hostmisc/utils.h index 2f13193e608887..7de346c7bcee98 100644 --- a/src/installer/corehost/cli/hostmisc/utils.h +++ b/src/installer/corehost/cli/hostmisc/utils.h @@ -45,6 +45,8 @@ size_t index_of_non_numeric(const pal::string_t& str, unsigned i); bool try_stou(const pal::string_t& str, unsigned* num); pal::string_t get_dotnet_root_env_var_name(); pal::string_t get_deps_from_app_binary(const pal::string_t& app_base, const pal::string_t& app); +pal::string_t get_runtime_config_path(const pal::string_t& path, const pal::string_t& name); +pal::string_t get_runtime_config_dev_path(const pal::string_t& path, const pal::string_t& name); void get_runtime_config_paths(const pal::string_t& path, const pal::string_t& name, pal::string_t* cfg, pal::string_t* dev_cfg); pal::string_t get_dotnet_root_from_fxr_path(const pal::string_t &fxr_path); diff --git a/src/installer/corehost/cli/hostpolicy/CMakeLists.txt b/src/installer/corehost/cli/hostpolicy/CMakeLists.txt index c05c876e1d92ab..ca2c78fa27951d 100644 --- a/src/installer/corehost/cli/hostpolicy/CMakeLists.txt +++ b/src/installer/corehost/cli/hostpolicy/CMakeLists.txt @@ -19,6 +19,11 @@ set(SOURCES ./hostpolicy_context.cpp ./hostpolicy.cpp ./hostpolicy_init.cpp + ../bundle/dir_utils.cpp + ../bundle/extractor.cpp + ../bundle/file_entry.cpp + ../bundle/manifest.cpp + ../bundle/runner.cpp ) set(HEADERS @@ -30,6 +35,11 @@ set(HEADERS ./hostpolicy_context.h ../hostpolicy.h ./hostpolicy_init.h + ../bundle/dir_utils.h + ../bundle/extractor.h + ../bundle/file_entry.h + ../bundle/manifest.h + ../bundle/runner.h ) include(../lib.cmake) diff --git a/src/installer/corehost/cli/hostpolicy/args.cpp b/src/installer/corehost/cli/hostpolicy/args.cpp index 562f70baea8e96..8c9b7912ca093a 100644 --- a/src/installer/corehost/cli/hostpolicy/args.cpp +++ b/src/installer/corehost/cli/hostpolicy/args.cpp @@ -4,6 +4,7 @@ #include "args.h" #include +#include "bundle/runner.h" arguments_t::arguments_t() : host_mode(host_mode_t::invalid) @@ -115,13 +116,43 @@ bool init_arguments( args.host_path = host_info.host_path; args.additional_deps_serialized = additional_deps_serialized; - args.managed_application = managed_application_path; - if (!args.managed_application.empty() && !pal::realpath(&args.managed_application)) + if (!managed_application_path.empty()) + { + pal::string_t managed_application_name = get_filename(managed_application_path); + if (bundle::info_t::is_single_file_bundle()) + { + args.app_root = bundle::runner_t::app()->base_path(); + const bundle::runner_t* app = bundle::runner_t::app(); + + // Check for the main app within the bundle. + if (!app->locate(managed_application_name, args.managed_application)) + { + // If the main app is not found in the bundle, check beside the bundle + // for very unlikely case where the main app.dll was itself excluded from the app bundle. + args.managed_application = managed_application_path; + if (!pal::realpath(&args.managed_application)) + { + trace::error(_X("Failed to locate managed application [%s]"), args.managed_application.c_str()); + return false; + } + } + } + else + { + args.app_root = get_directory(args.managed_application); + args.managed_application = managed_application_path; + if (!pal::realpath(&args.managed_application)) + { + trace::error(_X("Failed to locate managed application [%s]"), args.managed_application.c_str()); + return false; + } + } + } + else { trace::error(_X("Failed to locate managed application [%s]"), args.managed_application.c_str()); return false; } - args.app_root = get_directory(args.managed_application); if (!deps_file.empty()) { diff --git a/src/installer/corehost/cli/hostpolicy/deps_resolver.cpp b/src/installer/corehost/cli/hostpolicy/deps_resolver.cpp index 2e245b56dee49a..e96ed836070042 100644 --- a/src/installer/corehost/cli/hostpolicy/deps_resolver.cpp +++ b/src/installer/corehost/cli/hostpolicy/deps_resolver.cpp @@ -343,6 +343,12 @@ bool deps_resolver_t::probe_deps_entry(const deps_entry_t& entry, const pal::str else { // Non-rid assets, lookup in the published dir. + if (entry.to_bundle_path(deps_dir, candidate)) + { + trace::verbose(_X(" Probed bundle and matched '%s'"), candidate->c_str()); + return true; + } + if (entry.to_dir_path(deps_dir, candidate)) { trace::verbose(_X(" Probed deps dir and matched '%s'"), candidate->c_str()); diff --git a/src/installer/corehost/cli/hostpolicy/deps_resolver.h b/src/installer/corehost/cli/hostpolicy/deps_resolver.h index d61a78c31795d5..f2b27d4620a100 100644 --- a/src/installer/corehost/cli/hostpolicy/deps_resolver.h +++ b/src/installer/corehost/cli/hostpolicy/deps_resolver.h @@ -14,6 +14,7 @@ #include "deps_format.h" #include "deps_entry.h" #include "runtime_config.h" +#include "bundle/runner.h" // Probe paths to be resolved for ordering struct probe_paths_t @@ -185,6 +186,17 @@ class deps_resolver_t static const pal::string_t s_empty; return s_empty; } + if (m_host_mode == host_mode_t::apphost) + { + if (bundle::info_t::is_single_file_bundle()) + { + const bundle::runner_t* app = bundle::runner_t::app(); + if (app->is_netcoreapp3_compat_mode()) + { + return app->extraction_path(); + } + } + } return m_app_dir; } diff --git a/src/installer/corehost/cli/hostpolicy/hostpolicy.cpp b/src/installer/corehost/cli/hostpolicy/hostpolicy.cpp index 5c7d766a87ddf7..c2a933ec3649b9 100644 --- a/src/installer/corehost/cli/hostpolicy/hostpolicy.cpp +++ b/src/installer/corehost/cli/hostpolicy/hostpolicy.cpp @@ -18,6 +18,7 @@ #include #include #include "hostpolicy_context.h" +#include "bundle/runner.h" namespace { @@ -354,7 +355,7 @@ int corehost_init( } int corehost_main_init( - hostpolicy_init_t &hostpolicy_init, + hostpolicy_init_t& hostpolicy_init, const int argc, const pal::char_t* argv[], const pal::string_t& location, @@ -367,6 +368,15 @@ int corehost_main_init( hostpolicy_init.host_info.parse(argc, argv); } + if (bundle::info_t::is_single_file_bundle()) + { + StatusCode status = bundle::runner_t::process_manifest_and_extract(); + if (status != StatusCode::Success) + { + return status; + } + } + return corehost_init(hostpolicy_init, argc, argv, location, args); } @@ -434,6 +444,9 @@ int corehost_libhost_init(const hostpolicy_init_t &hostpolicy_init, const pal::s // Host info should always be valid in the delegate scenario assert(hostpolicy_init.host_info.is_valid(host_mode_t::libhost)); + // Single-file bundle is only expected in apphost mode. + assert(!bundle::info_t::is_single_file_bundle()); + return corehost_init(hostpolicy_init, 0, nullptr, location, args); } diff --git a/src/installer/corehost/cli/hostpolicy/hostpolicy_init.cpp b/src/installer/corehost/cli/hostpolicy/hostpolicy_init.cpp index c6103f7af91f0f..18e839eca6b451 100644 --- a/src/installer/corehost/cli/hostpolicy/hostpolicy_init.cpp +++ b/src/installer/corehost/cli/hostpolicy/hostpolicy_init.cpp @@ -4,6 +4,7 @@ #include "hostpolicy_init.h" #include +#include "bundle/runner.h" void make_palstr_arr(int argc, const pal::char_t** argv, std::vector* out) { @@ -126,6 +127,12 @@ bool hostpolicy_init_t::init(host_interface_t* input, hostpolicy_init_t* init) // For the backwards compat case, this will be later initialized with argv[0] } + if (input->version_lo >= offsetof(host_interface_t, app_bundle) + sizeof(input->app_bundle)) + { + static bundle::runner_t bundle_runner(input->app_bundle); + bundle::info_t::the_app = &bundle_runner; + } + return true; } diff --git a/src/installer/corehost/cli/hostpolicy/hostpolicy_init.h b/src/installer/corehost/cli/hostpolicy/hostpolicy_init.h index 18af97c9c25ed1..8d2ea616fcc36e 100644 --- a/src/installer/corehost/cli/hostpolicy/hostpolicy_init.h +++ b/src/installer/corehost/cli/hostpolicy/hostpolicy_init.h @@ -8,6 +8,7 @@ #include "host_interface.h" #include "host_startup_info.h" #include "fx_definition.h" +#include "bundle/info.h" struct hostpolicy_init_t { diff --git a/src/installer/corehost/cli/ijwhost/ijwthunk.cpp b/src/installer/corehost/cli/ijwhost/ijwthunk.cpp index b2329112162e23..2b05bcfe7089a9 100644 --- a/src/installer/corehost/cli/ijwhost/ijwthunk.cpp +++ b/src/installer/corehost/cli/ijwhost/ijwthunk.cpp @@ -73,7 +73,7 @@ bool patch_vtable_entries(PEDecoder& pe) error_writer_scope_t writer_scope(swallow_trace); size_t currentThunk = 0; - for(size_t i = 0; i < numFixupRecords; ++i) + for (size_t i = 0; i < numFixupRecords; ++i) { if (pFixupTable[i].Type & COR_VTABLE_PTRSIZED) { @@ -81,7 +81,7 @@ bool patch_vtable_entries(PEDecoder& pe) #ifdef _WIN64 DWORD oldProtect; - if(!VirtualProtect(pointers, (sizeof(BYTE*) * pFixupTable[i].Count), PAGE_READWRITE, &oldProtect)) + if (!VirtualProtect(pointers, (sizeof(BYTE*) * pFixupTable[i].Count), PAGE_READWRITE, &oldProtect)) { trace::error(_X("Failed to change the vtfixup table from RO to R/W failed.\n")); return false; @@ -101,7 +101,7 @@ bool patch_vtable_entries(PEDecoder& pe) #ifdef _WIN64 DWORD _; - if(!VirtualProtect(pointers, (sizeof(BYTE*) * pFixupTable[i].Count), oldProtect, &_)) + if (!VirtualProtect(pointers, (sizeof(BYTE*) * pFixupTable[i].Count), oldProtect, &_)) { trace::warning(_X("Failed to change the vtfixup table from R/W back to RO failed.\n")); } diff --git a/src/installer/corehost/cli/json/rapidjson/cursorstreamwrapper.h b/src/installer/corehost/cli/json/rapidjson/cursorstreamwrapper.h index 52c11a7c01d7d9..be470fbc278b13 100644 --- a/src/installer/corehost/cli/json/rapidjson/cursorstreamwrapper.h +++ b/src/installer/corehost/cli/json/rapidjson/cursorstreamwrapper.h @@ -46,7 +46,7 @@ class CursorStreamWrapper : public GenericStreamWrapper { // counting line and column number Ch Take() { Ch ch = this->is_.Take(); - if(ch == '\n') { + if (ch == '\n') { line_ ++; col_ = 0; } else { diff --git a/src/installer/corehost/cli/json/rapidjson/document.h b/src/installer/corehost/cli/json/rapidjson/document.h index 9783fe4acc9c53..74666e3423ee7d 100644 --- a/src/installer/corehost/cli/json/rapidjson/document.h +++ b/src/installer/corehost/cli/json/rapidjson/document.h @@ -2094,11 +2094,11 @@ class GenericValue { const SizeType len1 = GetStringLength(); const SizeType len2 = rhs.GetStringLength(); - if(len1 != len2) { return false; } + if (len1 != len2) { return false; } const Ch* const str1 = GetString(); const Ch* const str2 = rhs.GetString(); - if(str1 == str2) { return true; } // fast path for constant string + if (str1 == str2) { return true; } // fast path for constant string return (std::memcmp(str1, str2, sizeof(Ch) * len1) == 0); } diff --git a/src/installer/corehost/cli/json_parser.cpp b/src/installer/corehost/cli/json_parser.cpp index 9814f3215d66be..c02f07142cf5fa 100644 --- a/src/installer/corehost/cli/json_parser.cpp +++ b/src/installer/corehost/cli/json_parser.cpp @@ -15,6 +15,7 @@ #include "utils.h" #include #include +#include "bundle/info.h" namespace { @@ -42,9 +43,9 @@ std::streampos get_utf8_bom_length(pal::istream_t& stream) return 3; } -void get_line_column_from_offset(const std::vector& json, size_t offset, int *line, int *column) +void get_line_column_from_offset(const char* data, uint64_t size, size_t offset, int *line, int *column) { - assert(offset < json.size()); + assert(offset < size); *line = *column = 1; @@ -52,12 +53,12 @@ void get_line_column_from_offset(const std::vector& json, size_t offset, i { (*column)++; - if (json[i] == '\n') + if (data[i] == '\n') { (*line)++; *column = 1; } - else if (json[i] == '\r' && json[i + 1] == '\n') + else if (data[i] == '\r' && data[i + 1] == '\n') { (*line)++; *column = 1; @@ -75,32 +76,36 @@ void json_parser_t::realloc_buffer(size_t size) m_json[size] = '\0'; } -bool json_parser_t::parse_json(const pal::string_t& context) -{ - assert(!m_json.empty()); - #ifdef _WIN32 +void json_parser_t::parse_json(const char* data) +{ // Can't use in-situ parsing on Windows, as JSON data is encoded in // UTF-8 and the host expects wide strings. m_document will store // data in UTF-16 (with pal::char_t as the character type), but it // has to know that data is encoded in UTF-8 to convert during parsing. constexpr auto flags = rapidjson::ParseFlag::kParseStopWhenDoneFlag - | rapidjson::ParseFlag::kParseCommentsFlag; - m_document.Parse>(m_json.data()); -#else - m_document.ParseInsitu(m_json.data()); -#endif + | rapidjson::ParseFlag::kParseCommentsFlag; + m_document.Parse>(data); +} +#else // _WIN32 +void json_parser_t::parse_json(char* data) +{ + m_document.ParseInsitu(data); +} +#endif // _WIN32 +bool json_parser_t::validate_json(const char* data, int64_t size, const pal::string_t& context) +{ if (m_document.HasParseError()) { int line, column; size_t offset = m_document.GetErrorOffset(); - get_line_column_from_offset(m_json, offset, &line, &column); + get_line_column_from_offset(data, size, offset, &line, &column); trace::error(_X("A JSON parsing exception occurred in [%s], offset %zu (line %d, column %d): %s"), - context.c_str(), offset, line, column, - rapidjson::GetParseError_En(m_document.GetParseError())); + context.c_str(), offset, line, column, + rapidjson::GetParseError_En(m_document.GetParseError())); return false; } @@ -130,5 +135,41 @@ bool json_parser_t::parse_stream(pal::istream_t& stream, realloc_buffer(stream_size - current_pos); stream.read(m_json.data(), stream_size - current_pos); - return parse_json(context); + parse_json(m_json.data()); + return validate_json(m_json.data(), m_json.size(), context); +} + +bool json_parser_t::parse_file(const pal::string_t& path) +{ + // This code assumes that the caller has checked that the file `path` exists + // either within the bundle, or as a real file on disk. + + if (bundle::info_t::is_single_file_bundle()) + { + const bundle::location_t *location = nullptr; + const char* data = (const char*) bundle::info_t::config_t::map(path, location); + + if (data != nullptr) + { +#ifdef _WIN32 // _WIN32 + // Windows doesn't use in-situ parsing. Therefore, pass the memory mapped data directly. + // The parser will then copy/convert the data appropriately. + parse_json(data); + bool result = validate_json(data, location->size, path); +#else // _WIN32 + // On Linux, copy the input data, since m_document.ParseInsitu + // requires a mutable data input. + realloc_buffer(location->size); + memcpy(m_json.data(), data, location->size); + parse_json(m_json.data()); + bool result = validate_json(m_json.data(), location->size, path); +#endif // _WIN32 + + bundle::info_t::config_t::unmap((const int8_t*)data, location); + return result; + } + } + + pal::ifstream_t file{ path }; + return parse_stream(file, path); } diff --git a/src/installer/corehost/cli/json_parser.h b/src/installer/corehost/cli/json_parser.h index e5c477b1171e44..aa81879ff26ffb 100644 --- a/src/installer/corehost/cli/json_parser.h +++ b/src/installer/corehost/cli/json_parser.h @@ -21,12 +21,9 @@ class json_parser_t { using document_t = rapidjson::GenericDocument; const document_t& document() const { return m_document; } + bool parse_stream(pal::istream_t& stream, const pal::string_t& context); - bool parse_file(const pal::string_t& path) - { - pal::ifstream_t file{path}; - return parse_stream(file, path); - } + bool parse_file(const pal::string_t& path); private: // This is a vector of char and not pal::char_t because JSON data @@ -37,7 +34,12 @@ class json_parser_t { document_t m_document; void realloc_buffer(size_t size); - bool parse_json(const pal::string_t& context); +#ifdef _WIN32 + void parse_json(const char* data); +#else // _WIN32 + void parse_json(char* data); +#endif // _WIN32 + bool validate_json(const char* data, int64_t size, const pal::string_t& context); }; #endif // __JSON_PARSER_H__ diff --git a/src/installer/corehost/cli/runtime_config.cpp b/src/installer/corehost/cli/runtime_config.cpp index 2026da52d81d75..4dc26e58368f39 100644 --- a/src/installer/corehost/cli/runtime_config.cpp +++ b/src/installer/corehost/cli/runtime_config.cpp @@ -9,6 +9,7 @@ #include "runtime_config.h" #include "trace.h" #include "utils.h" +#include "bundle/info.h" #include // The semantics of applying the runtimeconfig.json values follows, in the following steps from @@ -338,6 +339,8 @@ bool runtime_config_t::ensure_dev_config_parsed() return true; } + // runtimeconfig.dev.json is never bundled into the single-file app. + // So, only a file on disk is processed. json_parser_t json; if (!json.parse_file(m_dev_path)) { @@ -398,13 +401,13 @@ bool runtime_config_t::ensure_parsed() trace::verbose(_X("Did not successfully parse the runtimeconfig.dev.json")); } - if (!pal::file_exists(m_path)) + json_parser_t json; + if (!bundle::info_t::config_t::probe(m_path) && !pal::file_exists(m_path)) { // Not existing is not an error. return true; } - json_parser_t json; if (!json.parse_file(m_path)) { return false; diff --git a/src/installer/corehost/cli/test/mockhostpolicy/mockhostpolicy.cpp b/src/installer/corehost/cli/test/mockhostpolicy/mockhostpolicy.cpp index 4a0567f3f358ac..9f5386982d24ff 100644 --- a/src/installer/corehost/cli/test/mockhostpolicy/mockhostpolicy.cpp +++ b/src/installer/corehost/cli/test/mockhostpolicy/mockhostpolicy.cpp @@ -66,6 +66,7 @@ SHARED_API int HOSTPOLICY_CALLTYPE corehost_load(host_interface_t* init) std::cout << "mock host_info_host_path:" << tostr(init->host_info_host_path).data() << std::endl; std::cout << "mock host_info_dotnet_root:" << tostr(init->host_info_dotnet_root).data() << std::endl; std::cout << "mock host_info_app_path:" << tostr(init->host_info_app_path).data() << std::endl; + std::cout << "mock app_bundle:" << std::hex << init->app_bundle << std::endl; if (init->fx_names.len == 0) { diff --git a/src/installer/corehost/corehost.cpp b/src/installer/corehost/corehost.cpp index bb6b4ce60e5dd4..623dba8e98711f 100644 --- a/src/installer/corehost/corehost.cpp +++ b/src/installer/corehost/corehost.cpp @@ -11,8 +11,7 @@ #include "utils.h" #if defined(FEATURE_APPHOST) -#include "cli/apphost/bundle/marker.h" -#include "cli/apphost/bundle/runner.h" +#include "bundle_marker.h" #if defined(_WIN32) #include "cli/apphost/apphost.windows.h" @@ -98,7 +97,7 @@ int exe_start(const int argc, const pal::char_t* argv[]) pal::string_t app_path; pal::string_t app_root; bool requires_v2_hostfxr_interface = false; - + #if defined(FEATURE_APPHOST) pal::string_t embedded_app_name; if (!is_exe_enabled_for_execution(&embedded_app_name)) @@ -118,26 +117,14 @@ int exe_start(const int argc, const pal::char_t* argv[]) requires_v2_hostfxr_interface = true; } - if (bundle::marker_t::is_bundle()) - { - bundle::runner_t bundle_runner(host_path); - StatusCode bundle_status = bundle_runner.extract(); - - if (bundle_status != StatusCode::Success) - { - trace::error(_X("A fatal error was encountered. Could not extract contents of the bundle")); - return bundle_status; - } + app_path.assign(get_directory(host_path)); + append_path(&app_path, embedded_app_name.c_str()); - app_path.assign(bundle_runner.extraction_dir()); - } - else + if (bundle_marker_t::is_bundle()) { - app_path.assign(get_directory(host_path)); + trace::info(_X("Detected Single-File app bundle")); } - - append_path(&app_path, embedded_app_name.c_str()); - if (!pal::realpath(&app_path)) + else if (!pal::realpath(&app_path)) { trace::error(_X("The application to execute does not exist: '%s'."), app_path.c_str()); return StatusCode::LibHostAppRootFindFailure; @@ -200,23 +187,32 @@ int exe_start(const int argc, const pal::char_t* argv[]) // Obtain the entrypoints. int rc; - hostfxr_main_startupinfo_fn main_fn_v2 = reinterpret_cast(pal::get_symbol(fxr, "hostfxr_main_startupinfo")); - if (main_fn_v2 != nullptr) +#if defined(FEATURE_APPHOST) + if (bundle_marker_t::is_bundle()) { + hostfxr_main_bundle_startup_info_fn main_fn_v3 = reinterpret_cast(pal::get_symbol(fxr, "hostfxr_main_bundle_startup_info")); + if (main_fn_v3 == nullptr) + { + trace::error(_X("The required library %s does not support single-file apps."), fxr_path.c_str()); + rc = StatusCode::CoreHostEntryPointFailure; + } + const pal::char_t* host_path_cstr = host_path.c_str(); const pal::char_t* dotnet_root_cstr = dotnet_root.empty() ? nullptr : dotnet_root.c_str(); const pal::char_t* app_path_cstr = app_path.empty() ? nullptr : app_path.c_str(); + int64_t bundle_header_offset = bundle_marker_t::header_offset(); - trace::info(_X("Invoking fx resolver [%s] v2"), fxr_path.c_str()); + trace::info(_X("Invoking fx resolver [%s] v3"), fxr_path.c_str()); trace::info(_X("Host path: [%s]"), host_path.c_str()); trace::info(_X("Dotnet path: [%s]"), dotnet_root.c_str()); trace::info(_X("App path: [%s]"), app_path.c_str()); + trace::info(_X("Bundle Header Offset: [%lx]"), bundle_header_offset); hostfxr_set_error_writer_fn set_error_writer_fn = reinterpret_cast(pal::get_symbol(fxr, "hostfxr_set_error_writer")); { propagate_error_writer_t propagate_error_writer_to_hostfxr(set_error_writer_fn); - rc = main_fn_v2(argc, argv, host_path_cstr, dotnet_root_cstr, app_path_cstr); + rc = main_fn_v3(argc, argv, host_path_cstr, dotnet_root_cstr, app_path_cstr,bundle_header_offset); if (trace::get_error_writer() != nullptr && rc == static_cast(StatusCode::FrameworkMissingFailure) && !set_error_writer_fn) { @@ -228,30 +224,61 @@ int exe_start(const int argc, const pal::char_t* argv[]) } } else +#endif // defined(FEATURE_APPHOST) { - if (requires_v2_hostfxr_interface) + hostfxr_main_startupinfo_fn main_fn_v2 = reinterpret_cast(pal::get_symbol(fxr, "hostfxr_main_startupinfo")); + if (main_fn_v2 != nullptr) { - trace::error(_X("The required library %s does not support relative app dll paths."), fxr_path.c_str()); - rc = StatusCode::CoreHostEntryPointFailure; + const pal::char_t* host_path_cstr = host_path.c_str(); + const pal::char_t* dotnet_root_cstr = dotnet_root.empty() ? nullptr : dotnet_root.c_str(); + const pal::char_t* app_path_cstr = app_path.empty() ? nullptr : app_path.c_str(); + + trace::info(_X("Invoking fx resolver [%s] v2"), fxr_path.c_str()); + trace::info(_X("Host path: [%s]"), host_path.c_str()); + trace::info(_X("Dotnet path: [%s]"), dotnet_root.c_str()); + trace::info(_X("App path: [%s]"), app_path.c_str()); + + hostfxr_set_error_writer_fn set_error_writer_fn = reinterpret_cast(pal::get_symbol(fxr, "hostfxr_set_error_writer")); + { + propagate_error_writer_t propagate_error_writer_to_hostfxr(set_error_writer_fn); + + rc = main_fn_v2(argc, argv, host_path_cstr, dotnet_root_cstr, app_path_cstr); + + if (trace::get_error_writer() != nullptr && rc == static_cast(StatusCode::FrameworkMissingFailure) && !set_error_writer_fn) + { + pal::string_t url = get_download_url(); + trace::error(_X(" _ To run this application, you need to install a newer version of .NET Core.")); + trace::error(_X("")); + trace::error(_X(" - %s"), url.c_str()); + } + } } else { - trace::info(_X("Invoking fx resolver [%s] v1"), fxr_path.c_str()); - - // Previous corehost trace messages must be printed before calling trace::setup in hostfxr - trace::flush(); - - // For compat, use the v1 interface. This requires additional file I\O to re-parse parameters and - // for apphost, does not support DOTNET_ROOT or dll with different name for exe. - hostfxr_main_fn main_fn_v1 = reinterpret_cast(pal::get_symbol(fxr, "hostfxr_main")); - if (main_fn_v1 != nullptr) + if (requires_v2_hostfxr_interface) { - rc = main_fn_v1(argc, argv); + trace::error(_X("The required library %s does not support relative app dll paths."), fxr_path.c_str()); + rc = StatusCode::CoreHostEntryPointFailure; } else { - trace::error(_X("The required library %s does not contain the expected entry point."), fxr_path.c_str()); - rc = StatusCode::CoreHostEntryPointFailure; + trace::info(_X("Invoking fx resolver [%s] v1"), fxr_path.c_str()); + + // Previous corehost trace messages must be printed before calling trace::setup in hostfxr + trace::flush(); + + // For compat, use the v1 interface. This requires additional file I\O to re-parse parameters and + // for apphost, does not support DOTNET_ROOT or dll with different name for exe. + hostfxr_main_fn main_fn_v1 = reinterpret_cast(pal::get_symbol(fxr, "hostfxr_main")); + if (main_fn_v1 != nullptr) + { + rc = main_fn_v1(argc, argv); + } + else + { + trace::error(_X("The required library %s does not contain the expected entry point."), fxr_path.c_str()); + rc = StatusCode::CoreHostEntryPointFailure; + } } } } diff --git a/src/installer/managed/Microsoft.NET.HostModel/Bundle/Manifest.cs b/src/installer/managed/Microsoft.NET.HostModel/Bundle/Manifest.cs index bd2b340971e035..9c171ae2da7dd0 100644 --- a/src/installer/managed/Microsoft.NET.HostModel/Bundle/Manifest.cs +++ b/src/installer/managed/Microsoft.NET.HostModel/Bundle/Manifest.cs @@ -59,7 +59,7 @@ public class Manifest enum HeaderFlags : ulong { None = 0, - NetcoreApp3CompatMode = 2 + NetcoreApp3CompatMode = 1 } // Bundle ID is a string that is used to uniquely From 3e3ae6dfda7d2f491cf58b8c3d3a7c7e6d40880a Mon Sep 17 00:00:00 2001 From: swaroop-sridhar Date: Mon, 30 Mar 2020 20:18:02 -0700 Subject: [PATCH 02/18] Fix tests --- .../BundleExtractToSpecificPath.cs | 67 ++++++++----------- .../Helpers/BundleHelper.cs | 58 +++++++++++++--- .../BundleAndRun.cs | 3 +- .../BundlerConsistencyTests.cs | 10 +-- 4 files changed, 80 insertions(+), 58 deletions(-) diff --git a/src/installer/test/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleExtractToSpecificPath.cs b/src/installer/test/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleExtractToSpecificPath.cs index 5f35a406c915c5..c2b844e59731f5 100644 --- a/src/installer/test/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleExtractToSpecificPath.cs +++ b/src/installer/test/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleExtractToSpecificPath.cs @@ -7,14 +7,17 @@ using Microsoft.DotNet.CoreSetup.Test; using Microsoft.NET.HostModel.Bundle; using System; -using System.Collections.Generic; using System.IO; -using System.Linq; using System.Threading; using Xunit; namespace AppHost.Bundle.Tests { + // Once the static-host (AppHost, HostFxr, HostPolicy linked together) is available, + // this test should test with /p:IncludeNativeBinariesInSingleFile=true. + // This testing is currently not possible because the two native DLLs HostFxr.dll and HostPolicy.dll + // must remain adjacent to AppHost (since the bundle processing is dong within these binaries). + public class BundleExtractToSpecificPath : IClassFixture { private SharedTestState sharedTestState; @@ -29,24 +32,19 @@ private void Bundle_Extraction_To_Specific_Path_Succeeds() { var fixture = sharedTestState.TestFixture.Copy(); var hostName = BundleHelper.GetHostName(fixture); - var appName = Path.GetFileNameWithoutExtension(hostName); - string publishPath = BundleHelper.GetPublishPath(fixture); // Publish the bundle - var bundleDir = BundleHelper.GetBundleDir(fixture); - var bundler = new Bundler(hostName, bundleDir.FullName, BundleOptions.BundleAllContent); - string singleFile = BundleHelper.GenerateBundle(bundler, publishPath); - - // Compute bundled files - var bundledFiles = bundler.BundleManifest.Files.Select(file => file.RelativePath).ToList(); + string singleFile; + Bundler bundler = BundleHelper.BundleApp(fixture, out singleFile); // Verify expected files in the bundle directory + var bundleDir = BundleHelper.GetBundleDir(fixture); bundleDir.Should().HaveFile(hostName); - bundleDir.Should().NotHaveFiles(bundledFiles); + bundleDir.Should().NotHaveFiles(BundleHelper.GetBundledFiles(fixture)); // Create a directory for extraction. - var extractBaseDir = BundleHelper.GetExtractDir(fixture); - extractBaseDir.Should().NotHaveDirectory(appName); + var extractBaseDir = BundleHelper.GetExtractionRootDir(fixture); + extractBaseDir.Should().NotHaveDirectory(BundleHelper.GetAppBaseName(fixture)); // Run the bundled app for the first time, and extract files to // $DOTNET_BUNDLE_EXTRACT_BASE_DIR//bundle-id @@ -60,9 +58,8 @@ private void Bundle_Extraction_To_Specific_Path_Succeeds() .And .HaveStdOutContaining("Hello World"); - string extractPath = Path.Combine(extractBaseDir.FullName, appName, bundler.BundleManifest.BundleID); - var extractDir = new DirectoryInfo(extractPath); - extractDir.Should().OnlyHaveFiles(bundledFiles); + var extractDir = BundleHelper.GetExtractionDir(fixture, bundler); + extractDir.Should().OnlyHaveFiles(BundleHelper.GetExtractedFiles(fixture)); extractDir.Should().NotHaveFile(hostName); } @@ -70,17 +67,13 @@ private void Bundle_Extraction_To_Specific_Path_Succeeds() private void Bundle_extraction_is_reused() { var fixture = sharedTestState.TestFixture.Copy(); - var hostName = BundleHelper.GetHostName(fixture); - var appName = Path.GetFileNameWithoutExtension(hostName); - string publishPath = BundleHelper.GetPublishPath(fixture); // Publish the bundle - var bundleDir = BundleHelper.GetBundleDir(fixture); - var bundler = new Bundler(hostName, bundleDir.FullName, BundleOptions.BundleAllContent); - string singleFile = BundleHelper.GenerateBundle(bundler, publishPath); + string singleFile; + Bundler bundler = BundleHelper.BundleApp(fixture, out singleFile); // Create a directory for extraction. - var extractBaseDir = BundleHelper.GetExtractDir(fixture); + var extractBaseDir = BundleHelper.GetExtractionRootDir(fixture); // Run the bunded app for the first time, and extract files to // $DOTNET_BUNDLE_EXTRACT_BASE_DIR//bundle-id @@ -94,8 +87,8 @@ private void Bundle_extraction_is_reused() .And .HaveStdOutContaining("Hello World"); - string extractPath = Path.Combine(extractBaseDir.FullName, appName, bundler.BundleManifest.BundleID); - var extractDir = new DirectoryInfo(extractPath); + var appBaseName = BundleHelper.GetAppBaseName(fixture); + var extractDir = BundleHelper.GetExtractionDir(fixture, bundler); extractDir.Refresh(); DateTime firstWriteTime = extractDir.LastWriteTimeUtc; @@ -125,20 +118,14 @@ private void Bundle_extraction_can_recover_missing_files() var fixture = sharedTestState.TestFixture.Copy(); var hostName = BundleHelper.GetHostName(fixture); var appName = Path.GetFileNameWithoutExtension(hostName); - string publishPath = BundleHelper.GetPublishPath(fixture); // Publish the bundle - var bundleDir = BundleHelper.GetBundleDir(fixture); - var bundler = new Bundler(hostName, bundleDir.FullName, BundleOptions.BundleAllContent); - string singleFile = BundleHelper.GenerateBundle(bundler, publishPath); - - // Compute bundled files - List bundledFiles = bundler.BundleManifest.Files.Select(file => file.RelativePath).ToList(); + string singleFile; + Bundler bundler = BundleHelper.BundleApp(fixture, out singleFile); // Create a directory for extraction. - var extractBaseDir = BundleHelper.GetExtractDir(fixture); - string extractPath = Path.Combine(extractBaseDir.FullName, appName, bundler.BundleManifest.BundleID); - var extractDir = new DirectoryInfo(extractPath); + var extractBaseDir = BundleHelper.GetExtractionRootDir(fixture); + // Run the bunded app for the first time, and extract files to // $DOTNET_BUNDLE_EXTRACT_BASE_DIR//bundle-id @@ -152,10 +139,14 @@ private void Bundle_extraction_can_recover_missing_files() .And .HaveStdOutContaining("Hello World"); - bundledFiles.ForEach(file => File.Delete(Path.Combine(extractPath, file))); + // Remove the extracted files, but keep the extraction directory + var extractDir = BundleHelper.GetExtractionDir(fixture, bundler); + var extractedFiles = BundleHelper.GetExtractedFiles(fixture); + + Array.ForEach(extractedFiles, file => File.Delete(Path.Combine(extractDir.FullName, file))); extractDir.Should().Exist(); - extractDir.Should().NotHaveFiles(bundledFiles); + extractDir.Should().NotHaveFiles(extractedFiles); // Run the bundled app again (recover deleted files) Command.Create(singleFile) @@ -168,7 +159,7 @@ private void Bundle_extraction_can_recover_missing_files() .And .HaveStdOutContaining("Hello World"); - extractDir.Should().HaveFiles(bundledFiles); + extractDir.Should().HaveFiles(extractedFiles); } diff --git a/src/installer/test/Microsoft.NET.HostModel.Tests/Helpers/BundleHelper.cs b/src/installer/test/Microsoft.NET.HostModel.Tests/Helpers/BundleHelper.cs index e941becc23699d..15eeb2bf9b0116 100644 --- a/src/installer/test/Microsoft.NET.HostModel.Tests/Helpers/BundleHelper.cs +++ b/src/installer/test/Microsoft.NET.HostModel.Tests/Helpers/BundleHelper.cs @@ -40,6 +40,23 @@ public static string GetAppName(TestProjectFixture fixture) return Path.GetFileName(fixture.TestProject.AppDll); } + public static string GetAppBaseName(TestProjectFixture fixture) + { + return Path.GetFileNameWithoutExtension(GetAppName(fixture)); + } + + public static string[] GetBundledFiles(TestProjectFixture fixture) + { + string appBaseName = GetAppBaseName(fixture); + return new string[] { $"{appBaseName}.dll", $"{appBaseName}.deps.json", $"{appBaseName}.runtimeconfig.json" }; + } + + public static string[] GetExtractedFiles(TestProjectFixture fixture) + { + string appBaseName = GetAppBaseName(fixture); + return new string[] { $"{appBaseName}.dll" }; + } + public static string GetPublishPath(TestProjectFixture fixture) { return Path.Combine(fixture.TestProject.ProjectDirectory, "publish"); @@ -50,9 +67,24 @@ public static DirectoryInfo GetBundleDir(TestProjectFixture fixture) return Directory.CreateDirectory(Path.Combine(fixture.TestProject.ProjectDirectory, "bundle")); } - public static DirectoryInfo GetExtractDir(TestProjectFixture fixture) + public static string GetExtractionRootPath(TestProjectFixture fixture) + { + return Path.Combine(fixture.TestProject.ProjectDirectory, "extract"); + } + + public static DirectoryInfo GetExtractionRootDir(TestProjectFixture fixture) { - return Directory.CreateDirectory(Path.Combine(fixture.TestProject.ProjectDirectory, "extract")); + return Directory.CreateDirectory(GetExtractionRootPath(fixture)); + } + + public static string GetExtractionPath(TestProjectFixture fixture, Bundler bundler) + { + return Path.Combine(GetExtractionRootPath(fixture), GetAppBaseName(fixture), bundler.BundleManifest.BundleID); + + } + public static DirectoryInfo GetExtractionDir(TestProjectFixture fixture, Bundler bundler) + { + return new DirectoryInfo(GetExtractionPath(fixture, bundler)); } /// Generate a bundle containind the (embeddable) files in sourceDir @@ -81,20 +113,26 @@ public static string GenerateBundle(Bundler bundler, string sourceDir) // instead of the SDK via /p:PublishSingleFile=true. // This is necessary when the test needs the latest changes in the AppHost, // which may not (yet) be available in the SDK. - // - // Currently, AppHost can only handle bundles if all content is extracted to disk on startup. - // Therefore, the BundleOption is BundleAllContent by default. - // The default should be BundleOptions.None once host/runtime no longer requires full-extraction. - public static string BundleApp(TestProjectFixture fixture, - BundleOptions options = BundleOptions.BundleAllContent, - Version targetFrameworkVersion = null) + public static Bundler BundleApp(TestProjectFixture fixture, + out string singleFile, + BundleOptions options = BundleOptions.None, + Version targetFrameworkVersion = null) { var hostName = GetHostName(fixture); string publishPath = GetPublishPath(fixture); var bundleDir = GetBundleDir(fixture); var bundler = new Bundler(hostName, bundleDir.FullName, options, targetFrameworkVersion: targetFrameworkVersion); - string singleFile = GenerateBundle(bundler, publishPath); + singleFile = GenerateBundle(bundler, publishPath); + return bundler; + } + + public static string BundleApp(TestProjectFixture fixture, + BundleOptions options = BundleOptions.None, + Version targetFrameworkVersion = null) + { + string singleFile; + BundleApp(fixture, out singleFile, options, targetFrameworkVersion); return singleFile; } diff --git a/src/installer/test/Microsoft.NET.HostModel.Tests/Microsoft.NET.HostModel.Bundle.Tests/BundleAndRun.cs b/src/installer/test/Microsoft.NET.HostModel.Tests/Microsoft.NET.HostModel.Bundle.Tests/BundleAndRun.cs index 3199b1559ffda9..386c7ec0c5d12e 100644 --- a/src/installer/test/Microsoft.NET.HostModel.Tests/Microsoft.NET.HostModel.Bundle.Tests/BundleAndRun.cs +++ b/src/installer/test/Microsoft.NET.HostModel.Tests/Microsoft.NET.HostModel.Bundle.Tests/BundleAndRun.cs @@ -41,8 +41,7 @@ private void BundleRun(TestProjectFixture fixture, string publishPath, string si RunTheApp(Path.Combine(publishPath, hostName)); // Bundle to a single-file - // Bundle all content, until the host can handle other scenarios. - Bundler bundler = new Bundler(hostName, singleFileDir, BundleOptions.BundleAllContent); + Bundler bundler = new Bundler(hostName, singleFileDir); string singleFile = BundleHelper.GenerateBundle(bundler, publishPath); // Run the extracted app diff --git a/src/installer/test/Microsoft.NET.HostModel.Tests/Microsoft.NET.HostModel.Bundle.Tests/BundlerConsistencyTests.cs b/src/installer/test/Microsoft.NET.HostModel.Tests/Microsoft.NET.HostModel.Bundle.Tests/BundlerConsistencyTests.cs index efa13283bd100f..ee60b1e85a08db 100644 --- a/src/installer/test/Microsoft.NET.HostModel.Tests/Microsoft.NET.HostModel.Bundle.Tests/BundlerConsistencyTests.cs +++ b/src/installer/test/Microsoft.NET.HostModel.Tests/Microsoft.NET.HostModel.Bundle.Tests/BundlerConsistencyTests.cs @@ -177,7 +177,7 @@ public void TestAssemblyAlignment() string publishPath = BundleHelper.GetPublishPath(fixture); var bundleDir = BundleHelper.GetBundleDir(fixture); - var bundler = new Bundler(hostName, bundleDir.FullName, BundleOptions.BundleAllContent); + var bundler = new Bundler(hostName, bundleDir.FullName); BundleHelper.GenerateBundle(bundler, publishPath); bundler.BundleManifest.Files.ForEach(file => @@ -188,13 +188,7 @@ public void TestAssemblyAlignment() public void TestWithAdditionalContentAfterBundleMetadata() { var fixture = sharedTestState.TestFixture.Copy(); - - var hostName = BundleHelper.GetHostName(fixture); - var bundleDir = BundleHelper.GetBundleDir(fixture); - string publishPath = BundleHelper.GetPublishPath(fixture); - - var bundler = new Bundler(hostName, bundleDir.FullName, BundleOptions.BundleAllContent); - string singleFile = BundleHelper.GenerateBundle(bundler, publishPath); + string singleFile = BundleHelper.BundleApp(fixture); using (var file = File.OpenWrite(singleFile)) { From 9982baaa7a63b893ee98f06fc1d78a008f3d4a43 Mon Sep 17 00:00:00 2001 From: Swaroop Sridhar Date: Tue, 31 Mar 2020 02:58:37 -0700 Subject: [PATCH 03/18] Tests: Publish files excluded from bundle to bundle directory --- .../Helpers/BundleHelper.cs | 33 +++++++++-- .../BundleAndRun.cs | 4 +- .../BundleLegacy.cs | 1 - .../BundlerConsistencyTests.cs | 57 +++++-------------- 4 files changed, 43 insertions(+), 52 deletions(-) diff --git a/src/installer/test/Microsoft.NET.HostModel.Tests/Helpers/BundleHelper.cs b/src/installer/test/Microsoft.NET.HostModel.Tests/Helpers/BundleHelper.cs index 15eeb2bf9b0116..0ab9b50d147600 100644 --- a/src/installer/test/Microsoft.NET.HostModel.Tests/Helpers/BundleHelper.cs +++ b/src/installer/test/Microsoft.NET.HostModel.Tests/Helpers/BundleHelper.cs @@ -88,7 +88,7 @@ public static DirectoryInfo GetExtractionDir(TestProjectFixture fixture, Bundler } /// Generate a bundle containind the (embeddable) files in sourceDir - public static string GenerateBundle(Bundler bundler, string sourceDir) + public static string GenerateBundle(Bundler bundler, string sourceDir, string outputDir, bool copyExludedFiles=true) { // Convert sourceDir to absolute path sourceDir = Path.GetFullPath(sourceDir); @@ -105,7 +105,23 @@ public static string GenerateBundle(Bundler bundler, string sourceDir) fileSpecs.Add(new FileSpec(file, Path.GetRelativePath(sourceDir, file))); } - return bundler.GenerateBundle(fileSpecs); + + var singleFile = bundler.GenerateBundle(fileSpecs); + + if (copyExludedFiles) + { + foreach (var spec in fileSpecs) + { + if (spec.Excluded) + { + var outputFilePath = Path.Combine(outputDir, spec.BundleRelativePath); + Directory.CreateDirectory(Path.GetDirectoryName(outputFilePath)); + File.Copy(spec.SourcePath, outputFilePath, true); + } + } + } + + return singleFile; } // Bundle to a single-file @@ -116,14 +132,16 @@ public static string GenerateBundle(Bundler bundler, string sourceDir) public static Bundler BundleApp(TestProjectFixture fixture, out string singleFile, BundleOptions options = BundleOptions.None, - Version targetFrameworkVersion = null) + Version targetFrameworkVersion = null, + bool copyExcludedFiles = true) { var hostName = GetHostName(fixture); string publishPath = GetPublishPath(fixture); var bundleDir = GetBundleDir(fixture); var bundler = new Bundler(hostName, bundleDir.FullName, options, targetFrameworkVersion: targetFrameworkVersion); - singleFile = GenerateBundle(bundler, publishPath); + singleFile = GenerateBundle(bundler, publishPath, bundleDir.FullName, copyExcludedFiles); + return bundler; } @@ -136,6 +154,13 @@ public static string BundleApp(TestProjectFixture fixture, return singleFile; } + public static Bundler Bundle(TestProjectFixture fixture, BundleOptions options = BundleOptions.None) + { + string singleFile; + return BundleApp(fixture, out singleFile, copyExcludedFiles:false); + } + + public static void AddLongNameContentToAppWithSubDirs(TestProjectFixture fixture) { // For tests using the AppWithSubDirs, One of the sub-directories with a really long name diff --git a/src/installer/test/Microsoft.NET.HostModel.Tests/Microsoft.NET.HostModel.Bundle.Tests/BundleAndRun.cs b/src/installer/test/Microsoft.NET.HostModel.Tests/Microsoft.NET.HostModel.Bundle.Tests/BundleAndRun.cs index 386c7ec0c5d12e..e4949b47750daf 100644 --- a/src/installer/test/Microsoft.NET.HostModel.Tests/Microsoft.NET.HostModel.Bundle.Tests/BundleAndRun.cs +++ b/src/installer/test/Microsoft.NET.HostModel.Tests/Microsoft.NET.HostModel.Bundle.Tests/BundleAndRun.cs @@ -7,7 +7,6 @@ using Xunit; using Microsoft.DotNet.Cli.Build.Framework; using Microsoft.DotNet.CoreSetup.Test; -using Microsoft.NET.HostModel.Bundle; using BundleTests.Helpers; namespace Microsoft.NET.HostModel.Tests @@ -41,8 +40,7 @@ private void BundleRun(TestProjectFixture fixture, string publishPath, string si RunTheApp(Path.Combine(publishPath, hostName)); // Bundle to a single-file - Bundler bundler = new Bundler(hostName, singleFileDir); - string singleFile = BundleHelper.GenerateBundle(bundler, publishPath); + string singleFile = BundleHelper.BundleApp(fixture); // Run the extracted app RunTheApp(singleFile); diff --git a/src/installer/test/Microsoft.NET.HostModel.Tests/Microsoft.NET.HostModel.Bundle.Tests/BundleLegacy.cs b/src/installer/test/Microsoft.NET.HostModel.Tests/Microsoft.NET.HostModel.Bundle.Tests/BundleLegacy.cs index a9c9b700064fa6..7b9d6863607d20 100644 --- a/src/installer/test/Microsoft.NET.HostModel.Tests/Microsoft.NET.HostModel.Bundle.Tests/BundleLegacy.cs +++ b/src/installer/test/Microsoft.NET.HostModel.Tests/Microsoft.NET.HostModel.Bundle.Tests/BundleLegacy.cs @@ -3,7 +3,6 @@ // See the LICENSE file in the project root for more information. using System; -using System.IO; using Xunit; using Microsoft.DotNet.Cli.Build.Framework; using Microsoft.DotNet.CoreSetup.Test; diff --git a/src/installer/test/Microsoft.NET.HostModel.Tests/Microsoft.NET.HostModel.Bundle.Tests/BundlerConsistencyTests.cs b/src/installer/test/Microsoft.NET.HostModel.Tests/Microsoft.NET.HostModel.Bundle.Tests/BundlerConsistencyTests.cs index ee60b1e85a08db..f96413c54de939 100644 --- a/src/installer/test/Microsoft.NET.HostModel.Tests/Microsoft.NET.HostModel.Bundle.Tests/BundlerConsistencyTests.cs +++ b/src/installer/test/Microsoft.NET.HostModel.Tests/Microsoft.NET.HostModel.Bundle.Tests/BundlerConsistencyTests.cs @@ -92,18 +92,10 @@ public void TestWithDuplicateEntriesFails() public void TestFilesAlwaysBundled(BundleOptions options) { var fixture = sharedTestState.TestFixture.Copy(); + var bundler = BundleHelper.Bundle(fixture, options); + var bundledFiles = BundleHelper.GetBundledFiles(fixture); - var hostName = BundleHelper.GetHostName(fixture); - var appName = Path.GetFileNameWithoutExtension(hostName); - string publishPath = BundleHelper.GetPublishPath(fixture); - var bundleDir = BundleHelper.GetBundleDir(fixture); - - var bundler = new Bundler(hostName, bundleDir.FullName, options); - BundleHelper.GenerateBundle(bundler, publishPath); - - bundler.BundleManifest.Contains($"{appName}.dll").Should().BeTrue(); - bundler.BundleManifest.Contains($"{appName}.deps.json").Should().BeTrue(); - bundler.BundleManifest.Contains($"{appName}.runtimeconfig.json").Should().BeTrue(); + Array.ForEach(bundledFiles, file => bundler.BundleManifest.Contains(file).Should().BeTrue()); } [InlineData(BundleOptions.None)] @@ -115,20 +107,16 @@ public void TestFilesAlwaysBundled(BundleOptions options) public void TestFilesNeverBundled(BundleOptions options) { var fixture = sharedTestState.TestFixture.Copy(); - - var hostName = BundleHelper.GetHostName(fixture); - var appName = Path.GetFileNameWithoutExtension(hostName); + var appBaseName = BundleHelper.GetAppBaseName(fixture); string publishPath = BundleHelper.GetPublishPath(fixture); - var bundleDir = BundleHelper.GetBundleDir(fixture); // Make up a app.runtimeconfig.dev.json file in the publish directory. - File.Copy(Path.Combine(publishPath, $"{appName}.runtimeconfig.json"), - Path.Combine(publishPath, $"{appName}.runtimeconfig.dev.json")); + File.Copy(Path.Combine(publishPath, $"{appBaseName}.runtimeconfig.json"), + Path.Combine(publishPath, $"{appBaseName}.runtimeconfig.dev.json")); - var bundler = new Bundler(hostName, bundleDir.FullName, options); - BundleHelper.GenerateBundle(bundler, publishPath); + var bundler = BundleHelper.Bundle(fixture, options); - bundler.BundleManifest.Contains($"{appName}.runtimeconfig.dev.json").Should().BeFalse(); + bundler.BundleManifest.Contains($"{appBaseName}.runtimeconfig.dev.json").Should().BeFalse(); } [InlineData(BundleOptions.None)] @@ -137,16 +125,10 @@ public void TestFilesNeverBundled(BundleOptions options) public void TestBundlingSymbols(BundleOptions options) { var fixture = sharedTestState.TestFixture.Copy(); + var appBaseName = BundleHelper.GetAppBaseName(fixture); + var bundler = BundleHelper.Bundle(fixture, options); - var hostName = BundleHelper.GetHostName(fixture); - var appName = Path.GetFileNameWithoutExtension(hostName); - string publishPath = BundleHelper.GetPublishPath(fixture); - var bundleDir = BundleHelper.GetBundleDir(fixture); - - var bundler = new Bundler(hostName, bundleDir.FullName, options); - BundleHelper.GenerateBundle(bundler, publishPath); - - bundler.BundleManifest.Contains($"{appName}.pdb").Should().Be(options.HasFlag(BundleOptions.BundleSymbolFiles)); + bundler.BundleManifest.Contains($"{appBaseName}.pdb").Should().Be(options.HasFlag(BundleOptions.BundleSymbolFiles)); } [InlineData(BundleOptions.None)] @@ -155,14 +137,8 @@ public void TestBundlingSymbols(BundleOptions options) public void TestBundlingNativeBinaries(BundleOptions options) { var fixture = sharedTestState.TestFixture.Copy(); - - var hostName = BundleHelper.GetHostName(fixture); var hostfxr = Path.GetFileName(fixture.TestProject.HostFxrDll); - string publishPath = BundleHelper.GetPublishPath(fixture); - var bundleDir = BundleHelper.GetBundleDir(fixture); - - var bundler = new Bundler(hostName, bundleDir.FullName, options); - BundleHelper.GenerateBundle(bundler, publishPath); + var bundler = BundleHelper.Bundle(fixture, options); bundler.BundleManifest.Contains($"{hostfxr}").Should().Be(options.HasFlag(BundleOptions.BundleNativeBinaries)); } @@ -171,14 +147,7 @@ public void TestBundlingNativeBinaries(BundleOptions options) public void TestAssemblyAlignment() { var fixture = sharedTestState.TestFixture.Copy(); - - var hostName = BundleHelper.GetHostName(fixture); - var appName = Path.GetFileNameWithoutExtension(hostName); - string publishPath = BundleHelper.GetPublishPath(fixture); - var bundleDir = BundleHelper.GetBundleDir(fixture); - - var bundler = new Bundler(hostName, bundleDir.FullName); - BundleHelper.GenerateBundle(bundler, publishPath); + var bundler = BundleHelper.Bundle(fixture); bundler.BundleManifest.Files.ForEach(file => Assert.True((file.Type != FileType.Assembly) || (file.Offset % Bundler.AssemblyAlignment == 0))); From 3ef4372707010c274e570410cb412093fbb54aed Mon Sep 17 00:00:00 2001 From: Swaroop Sridhar Date: Tue, 31 Mar 2020 23:55:50 -0700 Subject: [PATCH 04/18] Fix a few test leg failures --- src/installer/corehost/cli/bundle/info.h | 4 ++-- src/installer/corehost/cli/deps_entry.cpp | 9 ++++---- .../corehost/cli/hostpolicy/args.cpp | 2 +- .../cli/hostpolicy/hostpolicy_init.cpp | 7 ++++-- .../Microsoft.NET.HostModel/Bundle/Bundler.cs | 22 ++++++++++++------- .../Bundle/TargetInfo.cs | 3 +++ .../BundleExtractToSpecificPath.cs | 15 +++++-------- .../BundledAppWithSubDirs.cs | 6 ++--- .../Helpers/BundleHelper.cs | 8 ++++++- 9 files changed, 44 insertions(+), 32 deletions(-) diff --git a/src/installer/corehost/cli/bundle/info.h b/src/installer/corehost/cli/bundle/info.h index 60982315160df0..fbb3ee6d4c19bf 100644 --- a/src/installer/corehost/cli/bundle/info.h +++ b/src/installer/corehost/cli/bundle/info.h @@ -40,8 +40,8 @@ namespace bundle static bool probe(const pal::string_t& path) { - return bundle::info_t::the_app->m_deps_json.matches(path) || - bundle::info_t::the_app->m_runtimeconfig_json.matches(path); + return is_single_file_bundle() && + (the_app->m_deps_json.matches(path) || the_app->m_runtimeconfig_json.matches(path)); } static const int8_t* map(const pal::string_t& path, const location_t* &location); diff --git a/src/installer/corehost/cli/deps_entry.cpp b/src/installer/corehost/cli/deps_entry.cpp index dd25735811d6af..2d7a6be20de665 100644 --- a/src/installer/corehost/cli/deps_entry.cpp +++ b/src/installer/corehost/cli/deps_entry.cpp @@ -192,17 +192,16 @@ bool deps_entry_t::to_bundle_path(const pal::string_t& base, pal::string_t* str) pal::string_t& candidate = *str; candidate.clear(); - pal::string_t pal_relative_path = normalize_dir_separator(asset.relative_path); - pal::string_t bundle_path; + pal::string_t file_name = get_filename(normalize_dir_separator(asset.relative_path)); pal::string_t sub_path; if (asset_type == asset_types::resources) { - append_resource_path(bundle_path); - sub_path = get_filename(pal_relative_path); + append_resource_path(sub_path); + sub_path.append(file_name); } else { - sub_path = pal_relative_path; + sub_path = file_name; } bool exists = app->locate(sub_path, candidate); diff --git a/src/installer/corehost/cli/hostpolicy/args.cpp b/src/installer/corehost/cli/hostpolicy/args.cpp index 8c9b7912ca093a..e5886962043d20 100644 --- a/src/installer/corehost/cli/hostpolicy/args.cpp +++ b/src/installer/corehost/cli/hostpolicy/args.cpp @@ -139,7 +139,7 @@ bool init_arguments( } else { - args.app_root = get_directory(args.managed_application); + args.app_root = get_directory(managed_application_path); args.managed_application = managed_application_path; if (!pal::realpath(&args.managed_application)) { diff --git a/src/installer/corehost/cli/hostpolicy/hostpolicy_init.cpp b/src/installer/corehost/cli/hostpolicy/hostpolicy_init.cpp index 18e839eca6b451..a496defb5ab55c 100644 --- a/src/installer/corehost/cli/hostpolicy/hostpolicy_init.cpp +++ b/src/installer/corehost/cli/hostpolicy/hostpolicy_init.cpp @@ -129,8 +129,11 @@ bool hostpolicy_init_t::init(host_interface_t* input, hostpolicy_init_t* init) if (input->version_lo >= offsetof(host_interface_t, app_bundle) + sizeof(input->app_bundle)) { - static bundle::runner_t bundle_runner(input->app_bundle); - bundle::info_t::the_app = &bundle_runner; + if (input->app_bundle != nullptr) + { + static bundle::runner_t bundle_runner(input->app_bundle); + bundle::info_t::the_app = &bundle_runner; + } } return true; diff --git a/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs b/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs index 320969c101dfde..1cdd9187eda592 100644 --- a/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs +++ b/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs @@ -90,7 +90,7 @@ bool ShouldIgnore(string fileRelativePath) return fileRelativePath.Equals(RuntimeConfigDevJson); } - bool ShouldExclude(FileType type) + bool ShouldExclude(FileType type, string relativePath) { switch (type) { @@ -100,7 +100,11 @@ bool ShouldExclude(FileType type) return false; case FileType.NativeBinary: - return !Options.HasFlag(BundleOptions.BundleNativeBinaries); + // Don't bundle hostfxr and hostpolicy because the bundle handling code is actually within these libraries. + // Until a statically-linked apphost is available, these libraries will live next to the single-file app. + return !Options.HasFlag(BundleOptions.BundleNativeBinaries) || + relativePath.Equals(Target.HostFxr) || + relativePath.Equals(Target.HostPolicy); case FileType.Symbols: return !Options.HasFlag(BundleOptions.BundleSymbolFiles); @@ -229,22 +233,24 @@ public string GenerateBundle(IReadOnlyList fileSpecs) foreach (var fileSpec in fileSpecs) { - if (IsHost(fileSpec.BundleRelativePath)) + string relativePath = fileSpec.BundleRelativePath; + + if (IsHost(relativePath)) { continue; } - if (ShouldIgnore(fileSpec.BundleRelativePath)) + if (ShouldIgnore(relativePath)) { - Tracer.Log($"Ignore: {fileSpec.BundleRelativePath}"); + Tracer.Log($"Ignore: {relativePath}"); continue; } FileType type = InferType(fileSpec); - if (ShouldExclude(type)) + if (ShouldExclude(type, relativePath)) { - Tracer.Log($"Exclude [{type}]: {fileSpec.BundleRelativePath}"); + Tracer.Log($"Exclude [{type}]: {relativePath}"); fileSpec.Excluded = true; continue; } @@ -253,7 +259,7 @@ public string GenerateBundle(IReadOnlyList fileSpecs) { FileType targetType = Target.TargetSpecificFileType(type); long startOffset = AddToBundle(bundle, file, targetType); - FileEntry entry = BundleManifest.AddEntry(targetType, fileSpec.BundleRelativePath, startOffset, file.Length); + FileEntry entry = BundleManifest.AddEntry(targetType, relativePath, startOffset, file.Length); Tracer.Log($"Embed: {entry}"); } } diff --git a/src/installer/managed/Microsoft.NET.HostModel/Bundle/TargetInfo.cs b/src/installer/managed/Microsoft.NET.HostModel/Bundle/TargetInfo.cs index 1f6f61a7eee982..6045e84b127c46 100644 --- a/src/installer/managed/Microsoft.NET.HostModel/Bundle/TargetInfo.cs +++ b/src/installer/managed/Microsoft.NET.HostModel/Bundle/TargetInfo.cs @@ -68,6 +68,9 @@ public override string ToString() public bool IsOSX => OS.Equals(OSPlatform.OSX); public bool IsWindows => OS.Equals(OSPlatform.Windows); + public string HostFxr => IsWindows ? "hostfxr.dll" : IsLinux ? "libhostfxr.so" : "libhostfxr.dylib"; + public string HostPolicy => IsWindows ? "hostpolicy.dll" : IsLinux ? "libhostpolicy.so" : "libhostpolicy.dylib"; + // The .net core 3 apphost doesn't care about semantics of FileType -- all files are extracted at startup. // However, the apphost checks that the FileType value is within expected bounds, so set it to the first enumeration. public FileType TargetSpecificFileType(FileType fileType) => (BundleVersion == 1) ? FileType.Unknown : fileType; diff --git a/src/installer/test/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleExtractToSpecificPath.cs b/src/installer/test/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleExtractToSpecificPath.cs index c2b844e59731f5..e8abaa8c34a189 100644 --- a/src/installer/test/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleExtractToSpecificPath.cs +++ b/src/installer/test/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleExtractToSpecificPath.cs @@ -13,11 +13,6 @@ namespace AppHost.Bundle.Tests { - // Once the static-host (AppHost, HostFxr, HostPolicy linked together) is available, - // this test should test with /p:IncludeNativeBinariesInSingleFile=true. - // This testing is currently not possible because the two native DLLs HostFxr.dll and HostPolicy.dll - // must remain adjacent to AppHost (since the bundle processing is dong within these binaries). - public class BundleExtractToSpecificPath : IClassFixture { private SharedTestState sharedTestState; @@ -35,7 +30,7 @@ private void Bundle_Extraction_To_Specific_Path_Succeeds() // Publish the bundle string singleFile; - Bundler bundler = BundleHelper.BundleApp(fixture, out singleFile); + Bundler bundler = BundleHelper.BundleApp(fixture, out singleFile, BundleOptions.BundleNativeBinaries); // Verify expected files in the bundle directory var bundleDir = BundleHelper.GetBundleDir(fixture); @@ -60,7 +55,7 @@ private void Bundle_Extraction_To_Specific_Path_Succeeds() var extractDir = BundleHelper.GetExtractionDir(fixture, bundler); extractDir.Should().OnlyHaveFiles(BundleHelper.GetExtractedFiles(fixture)); - extractDir.Should().NotHaveFile(hostName); + extractDir.Should().NotHaveFiles(BundleHelper.GetFilesNeverExtracted(fixture)); } [Fact] @@ -70,7 +65,7 @@ private void Bundle_extraction_is_reused() // Publish the bundle string singleFile; - Bundler bundler = BundleHelper.BundleApp(fixture, out singleFile); + Bundler bundler = BundleHelper.BundleApp(fixture, out singleFile, BundleOptions.BundleNativeBinaries); // Create a directory for extraction. var extractBaseDir = BundleHelper.GetExtractionRootDir(fixture); @@ -121,7 +116,7 @@ private void Bundle_extraction_can_recover_missing_files() // Publish the bundle string singleFile; - Bundler bundler = BundleHelper.BundleApp(fixture, out singleFile); + Bundler bundler = BundleHelper.BundleApp(fixture, out singleFile, BundleOptions.BundleNativeBinaries); // Create a directory for extraction. var extractBaseDir = BundleHelper.GetExtractionRootDir(fixture); @@ -180,7 +175,7 @@ public SharedTestState() public void Dispose() { - TestFixture.Dispose(); + // TestFixture.Dispose(); } } } diff --git a/src/installer/test/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundledAppWithSubDirs.cs b/src/installer/test/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundledAppWithSubDirs.cs index 59360c09f9909d..c9c47c0269831c 100644 --- a/src/installer/test/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundledAppWithSubDirs.cs +++ b/src/installer/test/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundledAppWithSubDirs.cs @@ -103,9 +103,9 @@ public SharedTestState() public void Dispose() { - TestFrameworkDependentFixture.Dispose(); - TestSelfContainedFixture.Dispose(); - TestAppWithEmptyFileFixture.Dispose(); + // TestFrameworkDependentFixture.Dispose(); + // TestSelfContainedFixture.Dispose(); + // TestAppWithEmptyFileFixture.Dispose(); } } } diff --git a/src/installer/test/Microsoft.NET.HostModel.Tests/Helpers/BundleHelper.cs b/src/installer/test/Microsoft.NET.HostModel.Tests/Helpers/BundleHelper.cs index 0ab9b50d147600..5fb2970cd9d2f4 100644 --- a/src/installer/test/Microsoft.NET.HostModel.Tests/Helpers/BundleHelper.cs +++ b/src/installer/test/Microsoft.NET.HostModel.Tests/Helpers/BundleHelper.cs @@ -57,6 +57,13 @@ public static string[] GetExtractedFiles(TestProjectFixture fixture) return new string[] { $"{appBaseName}.dll" }; } + public static string[] GetFilesNeverExtracted(TestProjectFixture fixture) + { + string appBaseName = GetAppBaseName(fixture); + return new string[] { $"{appBaseName}.deps.json", $"{appBaseName}.runtimeconfig.json", + fixture.TestProject.HostFxrDll, fixture.TestProject.HostPolicyDll }; + } + public static string GetPublishPath(TestProjectFixture fixture) { return Path.Combine(fixture.TestProject.ProjectDirectory, "publish"); @@ -105,7 +112,6 @@ public static string GenerateBundle(Bundler bundler, string sourceDir, string ou fileSpecs.Add(new FileSpec(file, Path.GetRelativePath(sourceDir, file))); } - var singleFile = bundler.GenerateBundle(fileSpecs); if (copyExludedFiles) From b6d278615ea3f64f35288d54fb25bfde6d4d22c4 Mon Sep 17 00:00:00 2001 From: Swaroop Sridhar Date: Wed, 1 Apr 2020 20:51:25 -0700 Subject: [PATCH 05/18] Address feedback from @lpereira --- src/installer/corehost/cli/bundle/info.cpp | 8 +++-- src/installer/corehost/cli/bundle/info.h | 7 +++-- .../corehost/cli/bundle/manifest.cpp | 2 +- src/installer/corehost/cli/deps_entry.cpp | 6 +--- src/installer/corehost/cli/hostmisc/pal.h | 1 + .../corehost/cli/hostmisc/pal.unix.cpp | 19 ++++++++---- .../corehost/cli/hostmisc/pal.windows.cpp | 16 ++++++++-- .../cli/json/rapidjson/cursorstreamwrapper.h | 2 +- src/installer/corehost/cli/json_parser.cpp | 30 ++++--------------- src/installer/corehost/cli/json_parser.h | 7 +---- src/installer/corehost/cli/runtime_config.cpp | 2 +- 11 files changed, 47 insertions(+), 53 deletions(-) diff --git a/src/installer/corehost/cli/bundle/info.cpp b/src/installer/corehost/cli/bundle/info.cpp index 706f9f79d4e042..5a88cbacbcc4b1 100644 --- a/src/installer/corehost/cli/bundle/info.cpp +++ b/src/installer/corehost/cli/bundle/info.cpp @@ -76,10 +76,11 @@ void info_t::init_config(const pal::string_t& app_path) m_runtimeconfig_json = config_t(runtimeconfig_json_name, &m_header.runtimeconfig_json_location()); } -const int8_t* info_t::config_t::map(const pal::string_t& path, const location_t* &location) +int8_t* info_t::config_t::map(const pal::string_t& path, const location_t* &location) { - const bundle::info_t* app = bundle::info_t::the_app; + assert(is_single_file_bundle()); + const bundle::info_t* app = bundle::info_t::the_app; if (app->m_deps_json.matches(path)) { location = app->m_deps_json.m_location; @@ -101,8 +102,9 @@ const int8_t* info_t::config_t::map(const pal::string_t& path, const location_t* // Therefore, mapping only portions of the bundle will involve align-down/round-up calculations, and associated offset adjustments. // We choose the simpler approach of rounding to the whole file // * There is no performance limitation due to a larger sized mapping, since we actually only read the pages with relevant contents. + // * Files that are too large to be mapped (ex: that exhaust 32-bit virtual address space) are not supported. - const int8_t* addr = (const int8_t*)pal::mmap_read(app->m_bundle_path); + int8_t* addr = (int8_t*)pal::mmap_read(app->m_bundle_path); if (addr == nullptr) { trace::error(_X("Failure processing application bundle.")); diff --git a/src/installer/corehost/cli/bundle/info.h b/src/installer/corehost/cli/bundle/info.h index fbb3ee6d4c19bf..11f5c7c949d59a 100644 --- a/src/installer/corehost/cli/bundle/info.h +++ b/src/installer/corehost/cli/bundle/info.h @@ -19,7 +19,8 @@ namespace bundle { struct config_t { - config_t() {} + config_t() + : m_location(nullptr) {} config_t(const config_t& config) { @@ -44,13 +45,13 @@ namespace bundle (the_app->m_deps_json.matches(path) || the_app->m_runtimeconfig_json.matches(path)); } - static const int8_t* map(const pal::string_t& path, const location_t* &location); + static int8_t* map(const pal::string_t& path, const location_t* &location); static void unmap(const int8_t* addr, const location_t* location); private: pal::string_t m_path; const location_t *m_location; - } json_info; + }; static StatusCode process_bundle(const pal::char_t* bundle_path, const pal::char_t *app_path, int64_t header_offset); static bool is_single_file_bundle() { return the_app != nullptr; } diff --git a/src/installer/corehost/cli/bundle/manifest.cpp b/src/installer/corehost/cli/bundle/manifest.cpp index 991c8e8408ca65..55f1ff1e639272 100644 --- a/src/installer/corehost/cli/bundle/manifest.cpp +++ b/src/installer/corehost/cli/bundle/manifest.cpp @@ -13,7 +13,7 @@ manifest_t manifest_t::read(reader_t& reader, int32_t num_files) for (int32_t i = 0; i < num_files; i++) { file_entry_t entry = file_entry_t::read(reader); - manifest.files.emplace_back(entry); + manifest.files.push_back(std::move(entry)); manifest.m_need_extraction |= entry.needs_extraction(); } diff --git a/src/installer/corehost/cli/deps_entry.cpp b/src/installer/corehost/cli/deps_entry.cpp index 2d7a6be20de665..3cfb0832ba5993 100644 --- a/src/installer/corehost/cli/deps_entry.cpp +++ b/src/installer/corehost/cli/deps_entry.cpp @@ -25,11 +25,7 @@ void deps_entry_t::append_resource_path(pal::string_t& base) const { assert(asset_type == asset_types::resources); - pal::string_t pal_relative_path = asset.relative_path; - if (_X('/') != DIR_SEPARATOR) - { - replace_char(&pal_relative_path, _X('/'), DIR_SEPARATOR); - } + pal::string_t pal_relative_path = normalize_dir_separator(asset.relative_path); // Resources are represented as "lib///" in the deps.json. // The is the "directory" in the pal_relative_path below, so extract it. diff --git a/src/installer/corehost/cli/hostmisc/pal.h b/src/installer/corehost/cli/hostmisc/pal.h index 007d76c5044fe7..8f7a7566189d8e 100644 --- a/src/installer/corehost/cli/hostmisc/pal.h +++ b/src/installer/corehost/cli/hostmisc/pal.h @@ -257,6 +257,7 @@ namespace pal } void* mmap_read(const string_t& path, size_t* length = nullptr); + void* mmap_copy_on_write(const string_t& path, size_t* length = nullptr); bool touch_file(const string_t& path); bool realpath(string_t* path, bool skip_error_logging = false); diff --git a/src/installer/corehost/cli/hostmisc/pal.unix.cpp b/src/installer/corehost/cli/hostmisc/pal.unix.cpp index b04802a3c61dbc..b4a0742ce15130 100644 --- a/src/installer/corehost/cli/hostmisc/pal.unix.cpp +++ b/src/installer/corehost/cli/hostmisc/pal.unix.cpp @@ -58,7 +58,7 @@ bool pal::touch_file(const pal::string_t& path) return true; } -void* pal::mmap_read(const string_t& path, size_t* length) +static void* map_file(const pal::string_t& path, size_t* length, int flags) { int fd = open(path.c_str(), O_RDONLY); if (fd == -1) @@ -81,19 +81,28 @@ void* pal::mmap_read(const string_t& path, size_t* length) *length = size; } - void* address = mmap(nullptr, size, PROT_READ, MAP_SHARED, fd, 0); + void* address = mmap(nullptr, size, PROT_READ, flags, fd, 0); - if (address == nullptr) + if (address == MAP_FAILED) { trace::error(_X("Failed to map file. mmap(%s) failed with error %d"), path.c_str(), errno); - close(fd); - return nullptr; + address = nullptr; } close(fd); return address; } +void* pal::mmap_read(const string_t& path, size_t* length) +{ + return mmap(path, length, MAP_SHARED); +} + +void* pal::mmap_copy_on_write(const string_t& path, size_t* length) +{ + return mmap(path, length, MAP_PRIVATE); +} + bool pal::getcwd(pal::string_t* recv) { recv->clear(); diff --git a/src/installer/corehost/cli/hostmisc/pal.windows.cpp b/src/installer/corehost/cli/hostmisc/pal.windows.cpp index 7ef85cc40d6d32..65d79355bd80e0 100644 --- a/src/installer/corehost/cli/hostmisc/pal.windows.cpp +++ b/src/installer/corehost/cli/hostmisc/pal.windows.cpp @@ -76,7 +76,7 @@ bool pal::touch_file(const pal::string_t& path) return true; } -void* pal::mmap_read(const string_t& path, size_t *length) +static void* mmap(const pal::string_t& path, size_t *length, DWORD mapping_protect, DWORD view_desired_access) { HANDLE file = CreateFileW(path.c_str(), GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); @@ -98,7 +98,7 @@ void* pal::mmap_read(const string_t& path, size_t *length) *length = (size_t)fileSize.QuadPart; } - HANDLE map = CreateFileMappingW(file, NULL, PAGE_READONLY, 0, 0, NULL); + HANDLE map = CreateFileMappingW(file, NULL, mapping_protect, 0, 0, NULL); if (map == NULL) { @@ -107,7 +107,7 @@ void* pal::mmap_read(const string_t& path, size_t *length) return nullptr; } - void *address = MapViewOfFile(map, FILE_MAP_READ, 0, 0, 0); + void *address = MapViewOfFile(map, view_desired_access, 0, 0, 0); if (address == NULL) { @@ -123,6 +123,16 @@ void* pal::mmap_read(const string_t& path, size_t *length) return address; } +void* pal::mmap_read(const string_t& path, size_t* length) +{ + return mmap(path, length, PAGE_READONLY, FILE_MAP_READ); +} + +void* pal::mmap_copy_on_write(const string_t& path, size_t* length) +{ + return mmap(path, length, PAGE_WRITECOPY, FILE_MAP_READ | FILE_MAP_COPY); +} + bool pal::getcwd(pal::string_t* recv) { recv->clear(); diff --git a/src/installer/corehost/cli/json/rapidjson/cursorstreamwrapper.h b/src/installer/corehost/cli/json/rapidjson/cursorstreamwrapper.h index be470fbc278b13..52c11a7c01d7d9 100644 --- a/src/installer/corehost/cli/json/rapidjson/cursorstreamwrapper.h +++ b/src/installer/corehost/cli/json/rapidjson/cursorstreamwrapper.h @@ -46,7 +46,7 @@ class CursorStreamWrapper : public GenericStreamWrapper { // counting line and column number Ch Take() { Ch ch = this->is_.Take(); - if (ch == '\n') { + if(ch == '\n') { line_ ++; col_ = 0; } else { diff --git a/src/installer/corehost/cli/json_parser.cpp b/src/installer/corehost/cli/json_parser.cpp index c02f07142cf5fa..8b7d8e456c70b9 100644 --- a/src/installer/corehost/cli/json_parser.cpp +++ b/src/installer/corehost/cli/json_parser.cpp @@ -76,9 +76,9 @@ void json_parser_t::realloc_buffer(size_t size) m_json[size] = '\0'; } -#ifdef _WIN32 -void json_parser_t::parse_json(const char* data) +bool json_parser_t::parse_json(char* data, int64_t size, const pal::string_t& context) { +#ifdef _WIN32 // Can't use in-situ parsing on Windows, as JSON data is encoded in // UTF-8 and the host expects wide strings. m_document will store // data in UTF-16 (with pal::char_t as the character type), but it @@ -86,16 +86,10 @@ void json_parser_t::parse_json(const char* data) constexpr auto flags = rapidjson::ParseFlag::kParseStopWhenDoneFlag | rapidjson::ParseFlag::kParseCommentsFlag; m_document.Parse>(data); -} #else // _WIN32 -void json_parser_t::parse_json(char* data) -{ m_document.ParseInsitu(data); -} #endif // _WIN32 -bool json_parser_t::validate_json(const char* data, int64_t size, const pal::string_t& context) -{ if (m_document.HasParseError()) { int line, column; @@ -135,8 +129,7 @@ bool json_parser_t::parse_stream(pal::istream_t& stream, realloc_buffer(stream_size - current_pos); stream.read(m_json.data(), stream_size - current_pos); - parse_json(m_json.data()); - return validate_json(m_json.data(), m_json.size(), context); + return parse_json(m_json.data(), m_json.size(), context); } bool json_parser_t::parse_file(const pal::string_t& path) @@ -147,24 +140,11 @@ bool json_parser_t::parse_file(const pal::string_t& path) if (bundle::info_t::is_single_file_bundle()) { const bundle::location_t *location = nullptr; - const char* data = (const char*) bundle::info_t::config_t::map(path, location); + char* data = (char*) bundle::info_t::config_t::map(path, location); if (data != nullptr) { -#ifdef _WIN32 // _WIN32 - // Windows doesn't use in-situ parsing. Therefore, pass the memory mapped data directly. - // The parser will then copy/convert the data appropriately. - parse_json(data); - bool result = validate_json(data, location->size, path); -#else // _WIN32 - // On Linux, copy the input data, since m_document.ParseInsitu - // requires a mutable data input. - realloc_buffer(location->size); - memcpy(m_json.data(), data, location->size); - parse_json(m_json.data()); - bool result = validate_json(m_json.data(), location->size, path); -#endif // _WIN32 - + bool result = parse_json(data, location->size, path); bundle::info_t::config_t::unmap((const int8_t*)data, location); return result; } diff --git a/src/installer/corehost/cli/json_parser.h b/src/installer/corehost/cli/json_parser.h index aa81879ff26ffb..04e995a0a9e076 100644 --- a/src/installer/corehost/cli/json_parser.h +++ b/src/installer/corehost/cli/json_parser.h @@ -34,12 +34,7 @@ class json_parser_t { document_t m_document; void realloc_buffer(size_t size); -#ifdef _WIN32 - void parse_json(const char* data); -#else // _WIN32 - void parse_json(char* data); -#endif // _WIN32 - bool validate_json(const char* data, int64_t size, const pal::string_t& context); + bool parse_json(char* data, int64_t size, const pal::string_t& context); }; #endif // __JSON_PARSER_H__ diff --git a/src/installer/corehost/cli/runtime_config.cpp b/src/installer/corehost/cli/runtime_config.cpp index 4dc26e58368f39..d41aa2a05739ac 100644 --- a/src/installer/corehost/cli/runtime_config.cpp +++ b/src/installer/corehost/cli/runtime_config.cpp @@ -401,13 +401,13 @@ bool runtime_config_t::ensure_parsed() trace::verbose(_X("Did not successfully parse the runtimeconfig.dev.json")); } - json_parser_t json; if (!bundle::info_t::config_t::probe(m_path) && !pal::file_exists(m_path)) { // Not existing is not an error. return true; } + json_parser_t json; if (!json.parse_file(m_path)) { return false; From b71feeee12f4eb36b801c03fb89a0435eaf8ba00 Mon Sep 17 00:00:00 2001 From: Swaroop Sridhar Date: Wed, 1 Apr 2020 20:57:20 -0700 Subject: [PATCH 06/18] Undo temporary test changes --- .../AppHost.Bundle.Tests/BundleExtractToSpecificPath.cs | 2 +- .../AppHost.Bundle.Tests/BundledAppWithSubDirs.cs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/installer/test/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleExtractToSpecificPath.cs b/src/installer/test/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleExtractToSpecificPath.cs index e8abaa8c34a189..f08f03fcce43c4 100644 --- a/src/installer/test/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleExtractToSpecificPath.cs +++ b/src/installer/test/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleExtractToSpecificPath.cs @@ -175,7 +175,7 @@ public SharedTestState() public void Dispose() { - // TestFixture.Dispose(); + TestFixture.Dispose(); } } } diff --git a/src/installer/test/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundledAppWithSubDirs.cs b/src/installer/test/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundledAppWithSubDirs.cs index c9c47c0269831c..59360c09f9909d 100644 --- a/src/installer/test/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundledAppWithSubDirs.cs +++ b/src/installer/test/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundledAppWithSubDirs.cs @@ -103,9 +103,9 @@ public SharedTestState() public void Dispose() { - // TestFrameworkDependentFixture.Dispose(); - // TestSelfContainedFixture.Dispose(); - // TestAppWithEmptyFileFixture.Dispose(); + TestFrameworkDependentFixture.Dispose(); + TestSelfContainedFixture.Dispose(); + TestAppWithEmptyFileFixture.Dispose(); } } } From 090daf049644f189c563e6689f40a2ae2d695c42 Mon Sep 17 00:00:00 2001 From: Swaroop Sridhar Date: Wed, 1 Apr 2020 23:48:15 -0700 Subject: [PATCH 07/18] Fix a rename bug --- src/installer/corehost/cli/hostmisc/pal.unix.cpp | 4 ++-- src/installer/corehost/cli/hostmisc/pal.windows.cpp | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/installer/corehost/cli/hostmisc/pal.unix.cpp b/src/installer/corehost/cli/hostmisc/pal.unix.cpp index b4a0742ce15130..9f145f765762ce 100644 --- a/src/installer/corehost/cli/hostmisc/pal.unix.cpp +++ b/src/installer/corehost/cli/hostmisc/pal.unix.cpp @@ -95,12 +95,12 @@ static void* map_file(const pal::string_t& path, size_t* length, int flags) void* pal::mmap_read(const string_t& path, size_t* length) { - return mmap(path, length, MAP_SHARED); + return map_file(path, length, MAP_SHARED); } void* pal::mmap_copy_on_write(const string_t& path, size_t* length) { - return mmap(path, length, MAP_PRIVATE); + return map_file(path, length, MAP_PRIVATE); } bool pal::getcwd(pal::string_t* recv) diff --git a/src/installer/corehost/cli/hostmisc/pal.windows.cpp b/src/installer/corehost/cli/hostmisc/pal.windows.cpp index 65d79355bd80e0..0cfe036de38254 100644 --- a/src/installer/corehost/cli/hostmisc/pal.windows.cpp +++ b/src/installer/corehost/cli/hostmisc/pal.windows.cpp @@ -76,7 +76,7 @@ bool pal::touch_file(const pal::string_t& path) return true; } -static void* mmap(const pal::string_t& path, size_t *length, DWORD mapping_protect, DWORD view_desired_access) +static void* map_file(const pal::string_t& path, size_t *length, DWORD mapping_protect, DWORD view_desired_access) { HANDLE file = CreateFileW(path.c_str(), GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); @@ -125,12 +125,12 @@ static void* mmap(const pal::string_t& path, size_t *length, DWORD mapping_prote void* pal::mmap_read(const string_t& path, size_t* length) { - return mmap(path, length, PAGE_READONLY, FILE_MAP_READ); + return map_file(path, length, PAGE_READONLY, FILE_MAP_READ); } void* pal::mmap_copy_on_write(const string_t& path, size_t* length) { - return mmap(path, length, PAGE_WRITECOPY, FILE_MAP_READ | FILE_MAP_COPY); + return map_file(path, length, PAGE_WRITECOPY, FILE_MAP_READ | FILE_MAP_COPY); } bool pal::getcwd(pal::string_t* recv) From 5c321526dce1f3d0188c8ac0092953272f88dce4 Mon Sep 17 00:00:00 2001 From: Swaroop Sridhar Date: Wed, 1 Apr 2020 23:59:53 -0700 Subject: [PATCH 08/18] Improve a test The rename test would hang forever if AppWithWait() fails before creating the resume file. So, while waiting, check that the app-process is still running. --- .../AppHost.Bundle.Tests/BundleRename.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/installer/test/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleRename.cs b/src/installer/test/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleRename.cs index 5466743fc5d24a..581a1155c98f98 100644 --- a/src/installer/test/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleRename.cs +++ b/src/installer/test/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleRename.cs @@ -51,11 +51,13 @@ private void Bundle_can_be_renamed_while_running(bool renameFirstRun) .CaptureStdOut() .Start(); - while (!File.Exists(waitFile)) + while (!File.Exists(waitFile) && !singleExe.Process.HasExited) { Thread.Sleep(100); } + Assert.True(File.Exists(waitFile)); + File.Move(singleFile, renameFile); File.Create(resumeFile).Close(); From e8703c32b52af5f2c1e5610a43c07f321565c51c Mon Sep 17 00:00:00 2001 From: Swaroop Sridhar Date: Thu, 2 Apr 2020 17:16:47 -0700 Subject: [PATCH 09/18] Address feedback from vitek-karas * Fix HostFxr startup function naming in CoreHost * deps_resolver: Lookup up bundle even for rid-specific assets within the app * Ensure CUI/GUI error output when expected framework is not found for single file apps. * Other minor fixes --- src/installer/corehost/cli/bundle/info.cpp | 2 +- src/installer/corehost/cli/bundle/runner.cpp | 3 - src/installer/corehost/cli/deps_entry.cpp | 164 ++++++++---------- src/installer/corehost/cli/deps_entry.h | 15 +- src/installer/corehost/cli/fxr/hostfxr.cpp | 4 +- .../corehost/cli/fxr/hostpolicy_resolver.cpp | 3 +- src/installer/corehost/cli/hostfxr.h | 2 +- .../corehost/cli/hostpolicy/deps_resolver.cpp | 12 +- src/installer/corehost/corehost.cpp | 84 +++++---- 9 files changed, 124 insertions(+), 165 deletions(-) diff --git a/src/installer/corehost/cli/bundle/info.cpp b/src/installer/corehost/cli/bundle/info.cpp index 5a88cbacbcc4b1..f505606ca12bcd 100644 --- a/src/installer/corehost/cli/bundle/info.cpp +++ b/src/installer/corehost/cli/bundle/info.cpp @@ -97,7 +97,7 @@ int8_t* info_t::config_t::map(const pal::string_t& path, const location_t* &loca // When necessary to map the deps.json or runtimeconfig.json files, we map the whole single-file bundle, // and return the address at the appropriate offset. // This is because: - // * The host is the only code that is currently running, and attempting to map the bundle at this time + // * The host is the only code that is currently running and trying to map the bundle. // * Files can only be memory mapped at page-aligned offsets, and in whole page units. // Therefore, mapping only portions of the bundle will involve align-down/round-up calculations, and associated offset adjustments. // We choose the simpler approach of rounding to the whole file diff --git a/src/installer/corehost/cli/bundle/runner.cpp b/src/installer/corehost/cli/bundle/runner.cpp index 8b8fd58a1d51f0..89e07353ac1ee9 100644 --- a/src/installer/corehost/cli/bundle/runner.cpp +++ b/src/installer/corehost/cli/bundle/runner.cpp @@ -69,9 +69,6 @@ bool runner_t::locate(const pal::string_t& relative_path, pal::string_t& full_pa bool needs_extraction = entry->needs_extraction(); pal::string_t file_base = entry->needs_extraction() ? app->extraction_path() : app->base_path(); - // Reserve space for the path below - full_path.reserve(file_base.length() + relative_path.length() + 3); - full_path.assign(file_base); append_path(&full_path, relative_path.c_str()); diff --git a/src/installer/corehost/cli/deps_entry.cpp b/src/installer/corehost/cli/deps_entry.cpp index 3cfb0832ba5993..95be42c2226b64 100644 --- a/src/installer/corehost/cli/deps_entry.cpp +++ b/src/installer/corehost/cli/deps_entry.cpp @@ -21,28 +21,29 @@ static pal::string_t normalize_dir_separator(const pal::string_t& path) return normalized_path; } -void deps_entry_t::append_resource_path(pal::string_t& base) const -{ - assert(asset_type == asset_types::resources); - - pal::string_t pal_relative_path = normalize_dir_separator(asset.relative_path); - - // Resources are represented as "lib///" in the deps.json. - // The is the "directory" in the pal_relative_path below, so extract it. - pal::string_t ietf_dir = get_directory(pal_relative_path); - pal::string_t ietf = ietf_dir; - - // get_directory returns with DIR_SEPARATOR appended that we need to remove. - remove_trailing_dir_seperator(&ietf); - - // Extract IETF code from "lib//" - ietf = get_filename(ietf); - - append_path(&base, ietf.c_str()); - trace::verbose(_X("Detected a resource asset, will query dir/ietf-tag/resource base: %s asset: %s"), base.c_str(), asset.name.c_str()); -} - -bool deps_entry_t::to_path(const pal::string_t& base, bool look_in_base, pal::string_t* str) const +// ----------------------------------------------------------------------------- +// Given a "base" directory, determine the resolved path this this file. +// +// If this file exists within the single-file bundle candidate is: +// * If the file was extracted to disk, the full-path to the extracted file. +// * Otherwise, the path within the bundle, relative to the "base" directory. +// The runtime expects the entries in the TPA, NativeDllSearchDirectories, etc +// to be absolute paths. Therefore, the relative-paths within the bundle are +// expressed as absolute paths with respect to the location of the bundle-file. +// Otherwise, candidate is: +// * The full local path of the file. +// +// Parameters: +// base - The base directory to look for the relative path of this entry +// ietf_dir - If this is a resource asset, the IETF intermediate directory +// look_in_base - Whether to search as a relative path +// look_in_bundle - Whether to look within the single-file bundle +// str - If the method returns true, contains the file path for this deps entry +// +// Returns: +// If the file exists in the path relative to the "base" directory within the +// single-file or on disk. +bool deps_entry_t::to_path(const pal::string_t& base, const pal::string_t& ietf_dir, bool look_in_base, bool look_in_bundle, pal::string_t* str) const { pal::string_t& candidate = *str; @@ -54,13 +55,37 @@ bool deps_entry_t::to_path(const pal::string_t& base, bool look_in_base, pal::st return false; } - pal::string_t pal_relative_path = normalize_dir_separator(asset.relative_path); + pal::string_t normalized_path = normalize_dir_separator(asset.relative_path); // Reserve space for the path below - candidate.reserve(base.length() + pal_relative_path.length() + 3); + candidate.reserve(base.length() + ietf_dir.length() + normalized_path.length() + 3); + + pal::string_t file_path = look_in_base ? get_filename(normalized_path) : normalized_path; + pal::string_t sub_path = ietf_dir + file_path; + + if (look_in_bundle && bundle::info_t::is_single_file_bundle()) + { + const bundle::runner_t* app = bundle::runner_t::app(); + + if (base.compare(app->base_path()) == 0) + { + if (app->locate(sub_path, candidate)) + { + trace::verbose(_X(" %s found in bundle [%s]"), sub_path.c_str(), candidate.c_str()); + } + else + { + trace::verbose(_X(" %s not found in bundle"), sub_path.c_str()); + } + } + else + { + trace::verbose(_X(" %s not searched in bundle base path %s doesn't match bundle base %s."), + sub_path.c_str(), base.c_str(), app->base_path().c_str()); + } + } candidate.assign(base); - pal::string_t sub_path = look_in_base ? get_filename(pal_relative_path) : pal_relative_path; append_path(&candidate, sub_path.c_str()); bool exists = pal::file_exists(candidate); @@ -88,17 +113,29 @@ bool deps_entry_t::to_path(const pal::string_t& base, bool look_in_base, pal::st // Returns: // If the file exists in the path relative to the "base" directory. // -bool deps_entry_t::to_dir_path(const pal::string_t& base, pal::string_t* str) const +bool deps_entry_t::to_dir_path(const pal::string_t& base, bool look_in_bundle, pal::string_t* str) const { + pal::string_t ietf_dir; + if (asset_type == asset_types::resources) { - pal::string_t base_ietf_dir = base; - append_resource_path(base_ietf_dir); + pal::string_t pal_relative_path = normalize_dir_separator(asset.relative_path); + + // Resources are represented as "lib///" in the deps.json. + // The is the "directory" in the pal_relative_path below, so extract it. + ietf_dir = get_directory(pal_relative_path); + + // get_directory returns with DIR_SEPARATOR appended that we need to remove. + remove_trailing_dir_seperator(&ietf_dir); - return to_path(base_ietf_dir, true, str); + // Extract IETF code from "lib//" + ietf_dir = get_filename(ietf_dir); + + trace::verbose(_X("Detected a resource asset, will query dir/ietf-tag/resource base: %s ietf: %s asset: %s"), + base.c_str(), ietf_dir.c_str(), asset.name.c_str()); } - return to_path(base, true, str); + return to_path(base, ietf_dir, true, look_in_bundle, str); } // ----------------------------------------------------------------------------- @@ -112,9 +149,9 @@ bool deps_entry_t::to_dir_path(const pal::string_t& base, pal::string_t* str) co // Returns: // If the file exists in the path relative to the "base" directory. // -bool deps_entry_t::to_rel_path(const pal::string_t& base, pal::string_t* str) const +bool deps_entry_t::to_rel_path(const pal::string_t& base, bool look_in_bundle, pal::string_t* str) const { - return to_path(base, false, str); + return to_path(base, _X(""), false, look_in_bundle, str); } // ----------------------------------------------------------------------------- @@ -150,66 +187,5 @@ bool deps_entry_t::to_full_path(const pal::string_t& base, pal::string_t* str) c append_path(&new_base, library_path.c_str()); } - return to_rel_path(new_base, str); -} - -// ----------------------------------------------------------------------------- -// Given a "base" directory, if this file exists within the single-file bundle, -// return -// * If the file was extracted to disk, the full-path to the extracted file. -// * Otherwise, the path within the bundle, relative to the "base" directory. -// The runtime expects the entries in the TPA, NativeDllSearchDirectories, etc -// to be absolute paths. Therefore, the relative-paths within the bundle are -// expressed as absolute paths with respect to the location of the bundle-file. -// -// Parameters: -// base - The directory containing the single-file bundle. -// str - If the method returns true, contains the file path for this deps entry -// -// Returns: -// If the file exists in the single-file bundle -// -bool deps_entry_t::to_bundle_path(const pal::string_t& base, pal::string_t* str) const -{ - if (!bundle::info_t::is_single_file_bundle()) - { - return false; - } - - const bundle::runner_t* app = bundle::runner_t::app(); - - // Bundled files are only searched relative to the app-directory. - if (base.compare(app->base_path()) != 0) - { - trace::verbose(_X(" Base directory %s is different from bundle-base %s"), base.c_str(), app->base_path().c_str()); - return false; - } - - pal::string_t& candidate = *str; - candidate.clear(); - - pal::string_t file_name = get_filename(normalize_dir_separator(asset.relative_path)); - pal::string_t sub_path; - if (asset_type == asset_types::resources) - { - append_resource_path(sub_path); - sub_path.append(file_name); - } - else - { - sub_path = file_name; - } - - bool exists = app->locate(sub_path, candidate); - - if (exists) - { - trace::verbose(_X(" %s found in bundle [%s]"), sub_path.c_str(), candidate.c_str()); - } - else - { - trace::verbose(_X(" %s not found in bundle"), sub_path.c_str()); - } - - return exists; + return to_rel_path(new_base, false, str); } diff --git a/src/installer/corehost/cli/deps_entry.h b/src/installer/corehost/cli/deps_entry.h index 1e0fa05e2e51d4..2c66b1f0552732 100644 --- a/src/installer/corehost/cli/deps_entry.h +++ b/src/installer/corehost/cli/deps_entry.h @@ -52,24 +52,19 @@ struct deps_entry_t bool is_serviceable; bool is_rid_specific; - // Given a "base" dir, yield the file path within this directory. - bool to_dir_path(const pal::string_t& base, pal::string_t* str) const; + // Given a "base" dir, yield the file path within this directory or single-file bundle. + bool to_dir_path(const pal::string_t& base, bool look_in_bundle, pal::string_t* str) const; // Given a "base" dir, yield the relative path in the package layout. - bool to_rel_path(const pal::string_t& base, pal::string_t* str) const; + bool to_rel_path(const pal::string_t& base, bool look_in_bundle, pal::string_t* str) const; // Given a "base" dir, yield the relative path with package name, version in the package layout. bool to_full_path(const pal::string_t& root, pal::string_t* str) const; - // Given a "base" dir, yield the file path within the single-file bundle. - bool to_bundle_path(const pal::string_t& base, pal::string_t* str) const; - private: // Given a "base" dir, yield the filepath within this directory or relative to this directory based on "look_in_base" - bool to_path(const pal::string_t& base, bool look_in_base, pal::string_t* str) const; - - // For resource assets, append the IETF code to the base path - void append_resource_path(pal::string_t& base) const; + // Returns a path within the single-file bundle, or a file on disk, + bool to_path(const pal::string_t& base, const pal::string_t& ietf_code, bool look_in_base, bool look_in_bundle, pal::string_t* str) const; }; #endif // __DEPS_ENTRY_H_ diff --git a/src/installer/corehost/cli/fxr/hostfxr.cpp b/src/installer/corehost/cli/fxr/hostfxr.cpp index 1d165b949d963f..1fdfa210489358 100644 --- a/src/installer/corehost/cli/fxr/hostfxr.cpp +++ b/src/installer/corehost/cli/fxr/hostfxr.cpp @@ -25,9 +25,9 @@ namespace } } -SHARED_API int HOSTFXR_CALLTYPE hostfxr_main_bundle_startup_info(const int argc, const pal::char_t* argv[], const pal::char_t* host_path, const pal::char_t* dotnet_root, const pal::char_t* app_path, int64_t bundle_header_offset) +SHARED_API int HOSTFXR_CALLTYPE hostfxr_main_bundle_startupinfo(const int argc, const pal::char_t* argv[], const pal::char_t* host_path, const pal::char_t* dotnet_root, const pal::char_t* app_path, int64_t bundle_header_offset) { - trace_hostfxr_entry_point(_X("hostfxr_main_bundle_startup_info")); + trace_hostfxr_entry_point(_X("hostfxr_main_bundle_startupinfo")); StatusCode bundleStatus = bundle::info_t::process_bundle(host_path, app_path, bundle_header_offset); if (bundleStatus != StatusCode::Success) diff --git a/src/installer/corehost/cli/fxr/hostpolicy_resolver.cpp b/src/installer/corehost/cli/fxr/hostpolicy_resolver.cpp index 12a04aa1a3a58b..c4616646bc05d2 100644 --- a/src/installer/corehost/cli/fxr/hostpolicy_resolver.cpp +++ b/src/installer/corehost/cli/fxr/hostpolicy_resolver.cpp @@ -238,8 +238,7 @@ bool hostpolicy_resolver::try_get_dir( // Resolve hostpolicy version out of the deps file. pal::string_t version = resolve_hostpolicy_version_from_deps(resolved_deps); - if (trace::is_enabled() && version.empty() && - (bundle::info_t::config_t::probe(resolved_deps)|| pal::file_exists(resolved_deps))) + if (trace::is_enabled() && version.empty() && pal::file_exists(resolved_deps)) { trace::warning(_X("Dependency manifest %s does not contain an entry for %s"), resolved_deps.c_str(), _STRINGIFY(HOST_POLICY_PKG_NAME)); diff --git a/src/installer/corehost/cli/hostfxr.h b/src/installer/corehost/cli/hostfxr.h index 30a7e4a8b4e920..f0bc0a53e22088 100644 --- a/src/installer/corehost/cli/hostfxr.h +++ b/src/installer/corehost/cli/hostfxr.h @@ -37,7 +37,7 @@ typedef int32_t(HOSTFXR_CALLTYPE *hostfxr_main_startupinfo_fn)( const char_t *host_path, const char_t *dotnet_root, const char_t *app_path); -typedef int32_t(HOSTFXR_CALLTYPE* hostfxr_main_bundle_startup_info_fn)( +typedef int32_t(HOSTFXR_CALLTYPE* hostfxr_main_bundle_startupinfo_fn)( const int argc, const char_t** argv, const char_t* host_path, diff --git a/src/installer/corehost/cli/hostpolicy/deps_resolver.cpp b/src/installer/corehost/cli/hostpolicy/deps_resolver.cpp index e96ed836070042..06ededa7e60aab 100644 --- a/src/installer/corehost/cli/hostpolicy/deps_resolver.cpp +++ b/src/installer/corehost/cli/hostpolicy/deps_resolver.cpp @@ -316,7 +316,7 @@ bool deps_resolver_t::probe_deps_entry(const deps_entry_t& entry, const pal::str // If the deps json has the package name and version, then someone has already done rid selection and // put the right asset in the dir. So checking just package name and version would suffice. // No need to check further for the exact asset relative sub path. - if (config.probe_deps_json->has_package(entry.library_name, entry.library_version) && entry.to_dir_path(probe_dir, candidate)) + if (config.probe_deps_json->has_package(entry.library_name, entry.library_version) && entry.to_dir_path(probe_dir, false, candidate)) { trace::verbose(_X(" Probed deps json and matched '%s'"), candidate->c_str()); return true; @@ -334,7 +334,7 @@ bool deps_resolver_t::probe_deps_entry(const deps_entry_t& entry, const pal::str { if (entry.is_rid_specific) { - if (entry.to_rel_path(deps_dir, candidate)) + if (entry.to_rel_path(deps_dir, true, candidate)) { trace::verbose(_X(" Probed deps dir and matched '%s'"), candidate->c_str()); return true; @@ -343,13 +343,7 @@ bool deps_resolver_t::probe_deps_entry(const deps_entry_t& entry, const pal::str else { // Non-rid assets, lookup in the published dir. - if (entry.to_bundle_path(deps_dir, candidate)) - { - trace::verbose(_X(" Probed bundle and matched '%s'"), candidate->c_str()); - return true; - } - - if (entry.to_dir_path(deps_dir, candidate)) + if (entry.to_dir_path(deps_dir, true, candidate)) { trace::verbose(_X(" Probed deps dir and matched '%s'"), candidate->c_str()); return true; diff --git a/src/installer/corehost/corehost.cpp b/src/installer/corehost/corehost.cpp index 623dba8e98711f..9d17d825b6e5d8 100644 --- a/src/installer/corehost/corehost.cpp +++ b/src/installer/corehost/corehost.cpp @@ -77,6 +77,14 @@ bool is_exe_enabled_for_execution(pal::string_t* app_dll) return true; } +void need_newer_framework_error() +{ + pal::string_t url = get_download_url(); + trace::error(_X(" _ To run this application, you need to install a newer version of .NET Core.")); + trace::error(_X("")); + trace::error(_X(" - %s"), url.c_str()); +} + #elif !defined(FEATURE_LIBHOST) #define CURHOST_TYPE _X("dotnet") #define CUREXE_PKG_VER HOST_PKG_VER @@ -96,7 +104,7 @@ int exe_start(const int argc, const pal::char_t* argv[]) pal::string_t app_path; pal::string_t app_root; - bool requires_v2_hostfxr_interface = false; + bool requires_hostfxr_startupinfo_interface = false; #if defined(FEATURE_APPHOST) pal::string_t embedded_app_name; @@ -114,7 +122,7 @@ int exe_start(const int argc, const pal::char_t* argv[]) auto pos_path_char = embedded_app_name.find(DIR_SEPARATOR); if (pos_path_char != pal::string_t::npos) { - requires_v2_hostfxr_interface = true; + requires_hostfxr_startupinfo_interface = true; } app_path.assign(get_directory(host_path)); @@ -190,72 +198,62 @@ int exe_start(const int argc, const pal::char_t* argv[]) #if defined(FEATURE_APPHOST) if (bundle_marker_t::is_bundle()) { - hostfxr_main_bundle_startup_info_fn main_fn_v3 = reinterpret_cast(pal::get_symbol(fxr, "hostfxr_main_bundle_startup_info")); - if (main_fn_v3 == nullptr) + hostfxr_main_bundle_startupinfo_fn hostfxr_main_bundle_startupinfo = reinterpret_cast(pal::get_symbol(fxr, "hostfxr_main_bundle_startupinfo")); + if (hostfxr_main_bundle_startupinfo != nullptr) { - trace::error(_X("The required library %s does not support single-file apps."), fxr_path.c_str()); - rc = StatusCode::CoreHostEntryPointFailure; - } - - const pal::char_t* host_path_cstr = host_path.c_str(); - const pal::char_t* dotnet_root_cstr = dotnet_root.empty() ? nullptr : dotnet_root.c_str(); - const pal::char_t* app_path_cstr = app_path.empty() ? nullptr : app_path.c_str(); - int64_t bundle_header_offset = bundle_marker_t::header_offset(); + const pal::char_t* host_path_cstr = host_path.c_str(); + const pal::char_t* dotnet_root_cstr = dotnet_root.empty() ? nullptr : dotnet_root.c_str(); + const pal::char_t* app_path_cstr = app_path.empty() ? nullptr : app_path.c_str(); + int64_t bundle_header_offset = bundle_marker_t::header_offset(); - trace::info(_X("Invoking fx resolver [%s] v3"), fxr_path.c_str()); - trace::info(_X("Host path: [%s]"), host_path.c_str()); - trace::info(_X("Dotnet path: [%s]"), dotnet_root.c_str()); - trace::info(_X("App path: [%s]"), app_path.c_str()); - trace::info(_X("Bundle Header Offset: [%lx]"), bundle_header_offset); + trace::info(_X("Invoking fx resolver [%s] hostfxr_main_bundle_startupinfo"), fxr_path.c_str()); + trace::info(_X("Host path: [%s]"), host_path.c_str()); + trace::info(_X("Dotnet path: [%s]"), dotnet_root.c_str()); + trace::info(_X("App path: [%s]"), app_path.c_str()); + trace::info(_X("Bundle Header Offset: [%lx]"), bundle_header_offset); - hostfxr_set_error_writer_fn set_error_writer_fn = reinterpret_cast(pal::get_symbol(fxr, "hostfxr_set_error_writer")); - { + hostfxr_set_error_writer_fn set_error_writer_fn = reinterpret_cast(pal::get_symbol(fxr, "hostfxr_set_error_writer")); propagate_error_writer_t propagate_error_writer_to_hostfxr(set_error_writer_fn); - - rc = main_fn_v3(argc, argv, host_path_cstr, dotnet_root_cstr, app_path_cstr,bundle_header_offset); - - if (trace::get_error_writer() != nullptr && rc == static_cast(StatusCode::FrameworkMissingFailure) && !set_error_writer_fn) - { - pal::string_t url = get_download_url(); - trace::error(_X(" _ To run this application, you need to install a newer version of .NET.")); - trace::error(_X("")); - trace::error(_X(" - %s"), url.c_str()); - } + rc = hostfxr_main_bundle_startupinfo(argc, argv, host_path_cstr, dotnet_root_cstr, app_path_cstr, bundle_header_offset); + } + else + { + // The host components will be statically linked with the app-host: https://github.com/dotnet/runtime/issues/32823 + // Once this work is completed, an outdated hostfxr can only be found for framework-related apps. + trace::error(_X("The required library %s does not support single-file apps."), fxr_path.c_str()); + need_newer_framework_error(); + rc = StatusCode::FrameworkMissingFailure; } } else #endif // defined(FEATURE_APPHOST) { - hostfxr_main_startupinfo_fn main_fn_v2 = reinterpret_cast(pal::get_symbol(fxr, "hostfxr_main_startupinfo")); - if (main_fn_v2 != nullptr) + hostfxr_main_startupinfo_fn hostfxr_main_startupinfo = reinterpret_cast(pal::get_symbol(fxr, "hostfxr_main_startupinfo")); + if (hostfxr_main_startupinfo != nullptr) { const pal::char_t* host_path_cstr = host_path.c_str(); const pal::char_t* dotnet_root_cstr = dotnet_root.empty() ? nullptr : dotnet_root.c_str(); const pal::char_t* app_path_cstr = app_path.empty() ? nullptr : app_path.c_str(); - trace::info(_X("Invoking fx resolver [%s] v2"), fxr_path.c_str()); + trace::info(_X("Invoking fx resolver [%s] hostfxr_main_startupinfo"), fxr_path.c_str()); trace::info(_X("Host path: [%s]"), host_path.c_str()); trace::info(_X("Dotnet path: [%s]"), dotnet_root.c_str()); trace::info(_X("App path: [%s]"), app_path.c_str()); hostfxr_set_error_writer_fn set_error_writer_fn = reinterpret_cast(pal::get_symbol(fxr, "hostfxr_set_error_writer")); - { - propagate_error_writer_t propagate_error_writer_to_hostfxr(set_error_writer_fn); + propagate_error_writer_t propagate_error_writer_to_hostfxr(set_error_writer_fn); - rc = main_fn_v2(argc, argv, host_path_cstr, dotnet_root_cstr, app_path_cstr); + rc = hostfxr_main_startupinfo(argc, argv, host_path_cstr, dotnet_root_cstr, app_path_cstr); - if (trace::get_error_writer() != nullptr && rc == static_cast(StatusCode::FrameworkMissingFailure) && !set_error_writer_fn) - { - pal::string_t url = get_download_url(); - trace::error(_X(" _ To run this application, you need to install a newer version of .NET Core.")); - trace::error(_X("")); - trace::error(_X(" - %s"), url.c_str()); - } + // This check exists to provide an error message for UI apps when running 3.0 apps on 2.0 only hostfxr, which doesn't support error writer redirection. + if (trace::get_error_writer() != nullptr && rc == static_cast(StatusCode::FrameworkMissingFailure) && !set_error_writer_fn) + { + need_newer_framework_error(); } } else { - if (requires_v2_hostfxr_interface) + if (requires_hostfxr_startupinfo_interface) { trace::error(_X("The required library %s does not support relative app dll paths."), fxr_path.c_str()); rc = StatusCode::CoreHostEntryPointFailure; From 4abf767e540fc705983bea3f61fc880114002342 Mon Sep 17 00:00:00 2001 From: Swaroop Sridhar Date: Fri, 3 Apr 2020 23:29:47 -0700 Subject: [PATCH 10/18] Fix tests failures --- .../Microsoft.NET.HostModel/Bundle/Bundler.cs | 6 +--- .../Bundle/TargetInfo.cs | 20 ++++++++--- .../TestProjects/AppWithSubDirs/Program.cs | 6 ++-- .../BundleExtractToSpecificPath.cs | 4 +-- .../BundledAppWithSubDirs.cs | 34 ++++++++++++------- .../Helpers/BundleHelper.cs | 16 +++++---- .../BundlerConsistencyTests.cs | 4 +-- src/installer/test/TestUtils/TestApp.cs | 2 ++ src/installer/test/TestUtils/TestProject.cs | 1 + 9 files changed, 57 insertions(+), 36 deletions(-) diff --git a/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs b/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs index 1cdd9187eda592..29e9635d4fd94a 100644 --- a/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs +++ b/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs @@ -100,11 +100,7 @@ bool ShouldExclude(FileType type, string relativePath) return false; case FileType.NativeBinary: - // Don't bundle hostfxr and hostpolicy because the bundle handling code is actually within these libraries. - // Until a statically-linked apphost is available, these libraries will live next to the single-file app. - return !Options.HasFlag(BundleOptions.BundleNativeBinaries) || - relativePath.Equals(Target.HostFxr) || - relativePath.Equals(Target.HostPolicy); + return !Options.HasFlag(BundleOptions.BundleNativeBinaries) || Target.ShouldExclude(relativePath); case FileType.Symbols: return !Options.HasFlag(BundleOptions.BundleSymbolFiles); diff --git a/src/installer/managed/Microsoft.NET.HostModel/Bundle/TargetInfo.cs b/src/installer/managed/Microsoft.NET.HostModel/Bundle/TargetInfo.cs index 6045e84b127c46..87b1c65202ed87 100644 --- a/src/installer/managed/Microsoft.NET.HostModel/Bundle/TargetInfo.cs +++ b/src/installer/managed/Microsoft.NET.HostModel/Bundle/TargetInfo.cs @@ -27,8 +27,6 @@ public class TargetInfo public TargetInfo(OSPlatform? os, Version targetFrameworkVersion) { - Version net50 = new Version(5, 0); - OS = os ?? HostOS; FrameworkVersion = targetFrameworkVersion ?? net50; @@ -68,12 +66,24 @@ public override string ToString() public bool IsOSX => OS.Equals(OSPlatform.OSX); public bool IsWindows => OS.Equals(OSPlatform.Windows); - public string HostFxr => IsWindows ? "hostfxr.dll" : IsLinux ? "libhostfxr.so" : "libhostfxr.dylib"; - public string HostPolicy => IsWindows ? "hostpolicy.dll" : IsLinux ? "libhostpolicy.so" : "libhostpolicy.dylib"; - // The .net core 3 apphost doesn't care about semantics of FileType -- all files are extracted at startup. // However, the apphost checks that the FileType value is within expected bounds, so set it to the first enumeration. public FileType TargetSpecificFileType(FileType fileType) => (BundleVersion == 1) ? FileType.Unknown : fileType; + + // In .net core 3.x, bundle processing happens within the AppHost. + // Therefore HostFxr and HostPolicy can be bundled within the single-file app. + // In .net 5, bundle processing happens in HostFxr and HostPolicy libraries. + // Therefore, these libraries themselves cannot be bundled into the single-file app. + // This problem is mitigated by statically linking these host components with the AppHost. + // https://github.com/dotnet/runtime/issues/32823 + public bool ShouldExclude(string relativePath) => + (FrameworkVersion.Major != 3) && (relativePath.Equals(HostFxr) || relativePath.Equals(HostPolicy)); + + readonly Version net50 = new Version(5, 0); + string HostFxr => IsWindows ? "hostfxr.dll" : IsLinux ? "libhostfxr.so" : "libhostfxr.dylib"; + string HostPolicy => IsWindows ? "hostpolicy.dll" : IsLinux ? "libhostpolicy.so" : "libhostpolicy.dylib"; + + } } diff --git a/src/installer/test/Assets/TestProjects/AppWithSubDirs/Program.cs b/src/installer/test/Assets/TestProjects/AppWithSubDirs/Program.cs index 20c24839186eb6..041ab2b670dde5 100644 --- a/src/installer/test/Assets/TestProjects/AppWithSubDirs/Program.cs +++ b/src/installer/test/Assets/TestProjects/AppWithSubDirs/Program.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using System; +using System.Diagnostics; using System.IO; using System.Reflection; @@ -12,10 +13,7 @@ public static class Program { public static void Main(string[] args) { - string baseDir = - Path.Combine( - Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), - "Sentence"); + string baseDir = Path.Combine(AppContext.BaseDirectory, "Sentence"); string Part(string dir="", string subdir="", string subsubdir="") { diff --git a/src/installer/test/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleExtractToSpecificPath.cs b/src/installer/test/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleExtractToSpecificPath.cs index f08f03fcce43c4..26dbc3a00ef832 100644 --- a/src/installer/test/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleExtractToSpecificPath.cs +++ b/src/installer/test/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleExtractToSpecificPath.cs @@ -30,7 +30,7 @@ private void Bundle_Extraction_To_Specific_Path_Succeeds() // Publish the bundle string singleFile; - Bundler bundler = BundleHelper.BundleApp(fixture, out singleFile, BundleOptions.BundleNativeBinaries); + Bundler bundler = BundleHelper.BundleApp(fixture, out singleFile); // Verify expected files in the bundle directory var bundleDir = BundleHelper.GetBundleDir(fixture); @@ -54,7 +54,7 @@ private void Bundle_Extraction_To_Specific_Path_Succeeds() .HaveStdOutContaining("Hello World"); var extractDir = BundleHelper.GetExtractionDir(fixture, bundler); - extractDir.Should().OnlyHaveFiles(BundleHelper.GetExtractedFiles(fixture)); + extractDir.Should().HaveFiles(BundleHelper.GetExtractedFiles(fixture)); extractDir.Should().NotHaveFiles(BundleHelper.GetFilesNeverExtracted(fixture)); } diff --git a/src/installer/test/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundledAppWithSubDirs.cs b/src/installer/test/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundledAppWithSubDirs.cs index 59360c09f9909d..33e05b5c0504dc 100644 --- a/src/installer/test/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundledAppWithSubDirs.cs +++ b/src/installer/test/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundledAppWithSubDirs.cs @@ -2,11 +2,12 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using System; -using Xunit; -using Microsoft.DotNet.Cli.Build.Framework; using BundleTests.Helpers; +using Microsoft.DotNet.Cli.Build.Framework; +using Microsoft.NET.HostModel.Bundle; using Microsoft.DotNet.CoreSetup.Test; +using System; +using Xunit; namespace AppHost.Bundle.Tests { @@ -31,11 +32,16 @@ private void RunTheApp(string path) .HaveStdOutContaining("Wow! We now say hello to the big world and you."); } - [Fact] - public void Bundled_Framework_dependent_App_Run_Succeeds() + // BundleOptions.BundleNativeBinaries: Test when the payload data files are unbundled, and beside the single-file app. + // BundleOptions.BundleAllContent: Test when the payload data files are bundled and extracted to temporary directory. + // Once the runtime can load assemblies from the bundle, BundleOptions.None can be used in place of BundleOptions.BundleNativeBinaries. + [InlineData(BundleOptions.BundleNativeBinaries)] + [InlineData(BundleOptions.BundleAllContent)] + [Theory] + public void Bundled_Framework_dependent_App_Run_Succeeds(BundleOptions options) { var fixture = sharedTestState.TestFrameworkDependentFixture.Copy(); - var singleFile = BundleHelper.BundleApp(fixture); + var singleFile = BundleHelper.BundleApp(fixture, options); // Run the bundled app (extract files) RunTheApp(singleFile); @@ -44,11 +50,13 @@ public void Bundled_Framework_dependent_App_Run_Succeeds() RunTheApp(singleFile); } - [Fact] - public void Bundled_Self_Contained_App_Run_Succeeds() + [InlineData(BundleOptions.BundleNativeBinaries)] + [InlineData(BundleOptions.BundleAllContent)] + [Theory] + public void Bundled_Self_Contained_App_Run_Succeeds(BundleOptions options) { var fixture = sharedTestState.TestSelfContainedFixture.Copy(); - var singleFile = BundleHelper.BundleApp(fixture); + var singleFile = BundleHelper.BundleApp(fixture, options); // Run the bundled app (extract files) RunTheApp(singleFile); @@ -57,11 +65,13 @@ public void Bundled_Self_Contained_App_Run_Succeeds() RunTheApp(singleFile); } - [Fact] - public void Bundled_With_Empty_File_Succeeds() + [InlineData(BundleOptions.BundleNativeBinaries)] + [InlineData(BundleOptions.BundleAllContent)] + [Theory] + public void Bundled_With_Empty_File_Succeeds(BundleOptions options) { var fixture = sharedTestState.TestAppWithEmptyFileFixture.Copy(); - var singleFile = BundleHelper.BundleApp(fixture); + var singleFile = BundleHelper.BundleApp(fixture, options); // Run the app RunTheApp(singleFile); diff --git a/src/installer/test/Microsoft.NET.HostModel.Tests/Helpers/BundleHelper.cs b/src/installer/test/Microsoft.NET.HostModel.Tests/Helpers/BundleHelper.cs index 5fb2970cd9d2f4..c5f77179436768 100644 --- a/src/installer/test/Microsoft.NET.HostModel.Tests/Helpers/BundleHelper.cs +++ b/src/installer/test/Microsoft.NET.HostModel.Tests/Helpers/BundleHelper.cs @@ -60,8 +60,10 @@ public static string[] GetExtractedFiles(TestProjectFixture fixture) public static string[] GetFilesNeverExtracted(TestProjectFixture fixture) { string appBaseName = GetAppBaseName(fixture); - return new string[] { $"{appBaseName}.deps.json", $"{appBaseName}.runtimeconfig.json", - fixture.TestProject.HostFxrDll, fixture.TestProject.HostPolicyDll }; + return new string[] { $"{appBaseName}.deps.json", + $"{appBaseName}.runtimeconfig.json", + Path.GetFileName(fixture.TestProject.HostFxrDll), + Path.GetFileName(fixture.TestProject.HostPolicyDll) }; } public static string GetPublishPath(TestProjectFixture fixture) @@ -137,7 +139,7 @@ public static string GenerateBundle(Bundler bundler, string sourceDir, string ou // which may not (yet) be available in the SDK. public static Bundler BundleApp(TestProjectFixture fixture, out string singleFile, - BundleOptions options = BundleOptions.None, + BundleOptions options = BundleOptions.BundleNativeBinaries, Version targetFrameworkVersion = null, bool copyExcludedFiles = true) { @@ -151,8 +153,11 @@ public static Bundler BundleApp(TestProjectFixture fixture, return bundler; } + // The defaut option for Bundling apps is BundleOptions.BundleNativeBinaries + // Until CoreCLR runtime can learn how to process assemblies from the bundle. + // This is because CoreCLR expects System.Private.Corelib.dll to be beside CoreCLR.dll public static string BundleApp(TestProjectFixture fixture, - BundleOptions options = BundleOptions.None, + BundleOptions options = BundleOptions.BundleNativeBinaries, Version targetFrameworkVersion = null) { string singleFile; @@ -163,10 +168,9 @@ public static string BundleApp(TestProjectFixture fixture, public static Bundler Bundle(TestProjectFixture fixture, BundleOptions options = BundleOptions.None) { string singleFile; - return BundleApp(fixture, out singleFile, copyExcludedFiles:false); + return BundleApp(fixture, out singleFile, options, copyExcludedFiles:false); } - public static void AddLongNameContentToAppWithSubDirs(TestProjectFixture fixture) { // For tests using the AppWithSubDirs, One of the sub-directories with a really long name diff --git a/src/installer/test/Microsoft.NET.HostModel.Tests/Microsoft.NET.HostModel.Bundle.Tests/BundlerConsistencyTests.cs b/src/installer/test/Microsoft.NET.HostModel.Tests/Microsoft.NET.HostModel.Bundle.Tests/BundlerConsistencyTests.cs index f96413c54de939..2e0bda9e45ed32 100644 --- a/src/installer/test/Microsoft.NET.HostModel.Tests/Microsoft.NET.HostModel.Bundle.Tests/BundlerConsistencyTests.cs +++ b/src/installer/test/Microsoft.NET.HostModel.Tests/Microsoft.NET.HostModel.Bundle.Tests/BundlerConsistencyTests.cs @@ -137,10 +137,10 @@ public void TestBundlingSymbols(BundleOptions options) public void TestBundlingNativeBinaries(BundleOptions options) { var fixture = sharedTestState.TestFixture.Copy(); - var hostfxr = Path.GetFileName(fixture.TestProject.HostFxrDll); + var coreclr = Path.GetFileName(fixture.TestProject.CoreClrDll); var bundler = BundleHelper.Bundle(fixture, options); - bundler.BundleManifest.Contains($"{hostfxr}").Should().Be(options.HasFlag(BundleOptions.BundleNativeBinaries)); + bundler.BundleManifest.Contains($"{coreclr}").Should().Be(options.HasFlag(BundleOptions.BundleNativeBinaries)); } [Fact] diff --git a/src/installer/test/TestUtils/TestApp.cs b/src/installer/test/TestUtils/TestApp.cs index 3e4e72f1fe705a..a1ff67d95c82ed 100644 --- a/src/installer/test/TestUtils/TestApp.cs +++ b/src/installer/test/TestUtils/TestApp.cs @@ -16,6 +16,7 @@ public class TestApp : TestArtifact public string RuntimeDevConfigJson { get; private set; } public string HostPolicyDll { get; private set; } public string HostFxrDll { get; private set; } + public string CoreClrDll { get; private set; } public string AssemblyName { get; } @@ -55,6 +56,7 @@ private void LoadAssets() RuntimeDevConfigJson = Path.Combine(Location, $"{AssemblyName}.runtimeconfig.dev.json"); HostPolicyDll = Path.Combine(Location, RuntimeInformationExtensions.GetSharedLibraryFileNameForCurrentPlatform("hostpolicy")); HostFxrDll = Path.Combine(Location, RuntimeInformationExtensions.GetSharedLibraryFileNameForCurrentPlatform("hostfxr")); + CoreClrDll = Path.Combine(Location, RuntimeInformationExtensions.GetSharedLibraryFileNameForCurrentPlatform("coreclr")); } } } diff --git a/src/installer/test/TestUtils/TestProject.cs b/src/installer/test/TestUtils/TestProject.cs index 5fd9135beec2f7..b512f4c1e6bc12 100644 --- a/src/installer/test/TestUtils/TestProject.cs +++ b/src/installer/test/TestUtils/TestProject.cs @@ -22,6 +22,7 @@ public class TestProject : TestArtifact public string AppExe { get => BuiltApp?.AppExe; } public string HostPolicyDll { get => BuiltApp?.HostPolicyDll; } public string HostFxrDll { get => BuiltApp?.HostFxrDll; } + public string CoreClrDll { get => BuiltApp?.CoreClrDll; } public TestApp BuiltApp { get; private set; } From 1a86437f5b4fd43871e27d24188881368060d4ac Mon Sep 17 00:00:00 2001 From: Swaroop Sridhar Date: Sat, 4 Apr 2020 00:10:12 -0700 Subject: [PATCH 11/18] Fix a non-apphost scenario --- src/installer/corehost/corehost.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/installer/corehost/corehost.cpp b/src/installer/corehost/corehost.cpp index 9d17d825b6e5d8..c49e00dba1ee23 100644 --- a/src/installer/corehost/corehost.cpp +++ b/src/installer/corehost/corehost.cpp @@ -77,6 +77,12 @@ bool is_exe_enabled_for_execution(pal::string_t* app_dll) return true; } +#elif !defined(FEATURE_LIBHOST) +#define CURHOST_TYPE _X("dotnet") +#define CUREXE_PKG_VER HOST_PKG_VER +#define CURHOST_EXE +#endif + void need_newer_framework_error() { pal::string_t url = get_download_url(); @@ -85,12 +91,6 @@ void need_newer_framework_error() trace::error(_X(" - %s"), url.c_str()); } -#elif !defined(FEATURE_LIBHOST) -#define CURHOST_TYPE _X("dotnet") -#define CUREXE_PKG_VER HOST_PKG_VER -#define CURHOST_EXE -#endif - #if defined(CURHOST_EXE) int exe_start(const int argc, const pal::char_t* argv[]) From 4f604110d084f3c8c5ed7e8335642d5921971fc6 Mon Sep 17 00:00:00 2001 From: Swaroop Sridhar Date: Sun, 5 Apr 2020 14:51:03 -0700 Subject: [PATCH 12/18] Fix a bug in deps resolution --- src/installer/corehost/cli/deps_entry.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/installer/corehost/cli/deps_entry.cpp b/src/installer/corehost/cli/deps_entry.cpp index 95be42c2226b64..45d70fee3840ec 100644 --- a/src/installer/corehost/cli/deps_entry.cpp +++ b/src/installer/corehost/cli/deps_entry.cpp @@ -72,6 +72,7 @@ bool deps_entry_t::to_path(const pal::string_t& base, const pal::string_t& ietf_ if (app->locate(sub_path, candidate)) { trace::verbose(_X(" %s found in bundle [%s]"), sub_path.c_str(), candidate.c_str()); + return true; } else { From 755b27a727cd0639075c4cd0b7dcc71c8207f156 Mon Sep 17 00:00:00 2001 From: Swaroop Sridhar Date: Mon, 6 Apr 2020 01:01:19 -0700 Subject: [PATCH 13/18] Fix resource path --- src/installer/corehost/cli/deps_entry.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/installer/corehost/cli/deps_entry.cpp b/src/installer/corehost/cli/deps_entry.cpp index 45d70fee3840ec..9d3cbc5b6dc7c7 100644 --- a/src/installer/corehost/cli/deps_entry.cpp +++ b/src/installer/corehost/cli/deps_entry.cpp @@ -61,7 +61,8 @@ bool deps_entry_t::to_path(const pal::string_t& base, const pal::string_t& ietf_ candidate.reserve(base.length() + ietf_dir.length() + normalized_path.length() + 3); pal::string_t file_path = look_in_base ? get_filename(normalized_path) : normalized_path; - pal::string_t sub_path = ietf_dir + file_path; + pal::string_t sub_path = ietf_dir; + append_path(&sub_path, file_path.c_str()); if (look_in_bundle && bundle::info_t::is_single_file_bundle()) { From ff561f64d4568a4b76286d118867c1b3fd63c750 Mon Sep 17 00:00:00 2001 From: Swaroop Sridhar Date: Mon, 6 Apr 2020 15:32:54 -0700 Subject: [PATCH 14/18] Add apphost version to framework_missing URL Also update a few comments to clarify that assemblies are currently extracted to disk. --- src/installer/corehost/cli/bundle/runner.cpp | 8 +++++--- src/installer/corehost/cli/deps_entry.cpp | 15 ++++++--------- src/installer/corehost/corehost.cpp | 2 +- .../Assets/TestProjects/AppWithSubDirs/Program.cs | 2 -- 4 files changed, 12 insertions(+), 15 deletions(-) diff --git a/src/installer/corehost/cli/bundle/runner.cpp b/src/installer/corehost/cli/bundle/runner.cpp index 89e07353ac1ee9..514e5abf754549 100644 --- a/src/installer/corehost/cli/bundle/runner.cpp +++ b/src/installer/corehost/cli/bundle/runner.cpp @@ -63,13 +63,15 @@ bool runner_t::locate(const pal::string_t& relative_path, pal::string_t& full_pa if (entry == nullptr) { + full_path.clear(); return false; } - bool needs_extraction = entry->needs_extraction(); - pal::string_t file_base = entry->needs_extraction() ? app->extraction_path() : app->base_path(); + // Currently, all files except deps.json and runtimeconfig.json are extracted to disk. + // The json files are not queried by the host using this method. + assert(entry->needs_extraction()); - full_path.assign(file_base); + full_path.assign(app->extraction_path()); append_path(&full_path, relative_path.c_str()); return true; diff --git a/src/installer/corehost/cli/deps_entry.cpp b/src/installer/corehost/cli/deps_entry.cpp index 9d3cbc5b6dc7c7..449403fa038e2f 100644 --- a/src/installer/corehost/cli/deps_entry.cpp +++ b/src/installer/corehost/cli/deps_entry.cpp @@ -22,16 +22,11 @@ static pal::string_t normalize_dir_separator(const pal::string_t& path) } // ----------------------------------------------------------------------------- -// Given a "base" directory, determine the resolved path this this file. +// Given a "base" directory, determine the resolved path for this file. // -// If this file exists within the single-file bundle candidate is: -// * If the file was extracted to disk, the full-path to the extracted file. -// * Otherwise, the path within the bundle, relative to the "base" directory. -// The runtime expects the entries in the TPA, NativeDllSearchDirectories, etc -// to be absolute paths. Therefore, the relative-paths within the bundle are -// expressed as absolute paths with respect to the location of the bundle-file. -// Otherwise, candidate is: -// * The full local path of the file. +// * If this file exists within the single-file bundle candidate is +// the full-path to the extracted file. +// * Otherwise, candidate is the full local path of the file. // // Parameters: // base - The base directory to look for the relative path of this entry @@ -70,6 +65,8 @@ bool deps_entry_t::to_path(const pal::string_t& base, const pal::string_t& ietf_ if (base.compare(app->base_path()) == 0) { + // If sub_path is found in the single-file bundle, + // app::locate() will set candidate to the full-path to the assembly extracted out to disk. if (app->locate(sub_path, candidate)) { trace::verbose(_X(" %s found in bundle [%s]"), sub_path.c_str(), candidate.c_str()); diff --git a/src/installer/corehost/corehost.cpp b/src/installer/corehost/corehost.cpp index c49e00dba1ee23..55b8ed629827ce 100644 --- a/src/installer/corehost/corehost.cpp +++ b/src/installer/corehost/corehost.cpp @@ -88,7 +88,7 @@ void need_newer_framework_error() pal::string_t url = get_download_url(); trace::error(_X(" _ To run this application, you need to install a newer version of .NET Core.")); trace::error(_X("")); - trace::error(_X(" - %s"), url.c_str()); + trace::error(_X(" - %s&apphost_version=%s"), url.c_str(), _STRINGIFY(COMMON_HOST_PKG_VER)); } #if defined(CURHOST_EXE) diff --git a/src/installer/test/Assets/TestProjects/AppWithSubDirs/Program.cs b/src/installer/test/Assets/TestProjects/AppWithSubDirs/Program.cs index 041ab2b670dde5..5b51c37991b79e 100644 --- a/src/installer/test/Assets/TestProjects/AppWithSubDirs/Program.cs +++ b/src/installer/test/Assets/TestProjects/AppWithSubDirs/Program.cs @@ -3,9 +3,7 @@ // See the LICENSE file in the project root for more information. using System; -using System.Diagnostics; using System.IO; -using System.Reflection; namespace AppWithSubDirs { From 3268b2967c0083d209c5e17adaf04595adb5ad21 Mon Sep 17 00:00:00 2001 From: Swaroop Sridhar Date: Mon, 6 Apr 2020 20:38:49 -0700 Subject: [PATCH 15/18] Fix a bug in hostpolicy initialization for nativehost --- .../corehost/cli/hostpolicy/args.cpp | 75 +++++++++++-------- 1 file changed, 42 insertions(+), 33 deletions(-) diff --git a/src/installer/corehost/cli/hostpolicy/args.cpp b/src/installer/corehost/cli/hostpolicy/args.cpp index e5886962043d20..b846d76142bd86 100644 --- a/src/installer/corehost/cli/hostpolicy/args.cpp +++ b/src/installer/corehost/cli/hostpolicy/args.cpp @@ -102,6 +102,47 @@ bool parse_arguments( args); } +bool set_root_from_app(const pal::string_t& managed_application_path, + arguments_t& args) +{ + args.managed_application = managed_application_path; + + if (args.managed_application.empty()) + { + // Managed app being empty by itself is not a failure. Host may be initialized from a config file. + assert(args.host_mode != host_mode_t::apphost); + return true; + } + + if (bundle::info_t::is_single_file_bundle()) + { + const bundle::runner_t* app = bundle::runner_t::app(); + args.app_root = app->base_path(); + + // Check for the main app within the bundle. + // locate() sets args.managed_application to the full path of the app extracted to disk. + pal::string_t managed_application_name = get_filename(managed_application_path); + if (app->locate(managed_application_name, args.managed_application)) + { + return true; + } + + trace::info(_X("Managed application [%s] not found in single-file bundle"), managed_application_name.c_str()); + + // If the main assembly is not found in the bundle, continue checking on disk + // for very unlikely case where the main app.dll was itself excluded from the app bundle. + return pal::realpath(&args.managed_application); + } + + if (pal::realpath(&args.managed_application)) + { + args.app_root = get_directory(managed_application_path); + return true; + } + + return false; +} + bool init_arguments( const pal::string_t& managed_application_path, const host_startup_info_t& host_info, @@ -116,39 +157,7 @@ bool init_arguments( args.host_path = host_info.host_path; args.additional_deps_serialized = additional_deps_serialized; - if (!managed_application_path.empty()) - { - pal::string_t managed_application_name = get_filename(managed_application_path); - if (bundle::info_t::is_single_file_bundle()) - { - args.app_root = bundle::runner_t::app()->base_path(); - const bundle::runner_t* app = bundle::runner_t::app(); - - // Check for the main app within the bundle. - if (!app->locate(managed_application_name, args.managed_application)) - { - // If the main app is not found in the bundle, check beside the bundle - // for very unlikely case where the main app.dll was itself excluded from the app bundle. - args.managed_application = managed_application_path; - if (!pal::realpath(&args.managed_application)) - { - trace::error(_X("Failed to locate managed application [%s]"), args.managed_application.c_str()); - return false; - } - } - } - else - { - args.app_root = get_directory(managed_application_path); - args.managed_application = managed_application_path; - if (!pal::realpath(&args.managed_application)) - { - trace::error(_X("Failed to locate managed application [%s]"), args.managed_application.c_str()); - return false; - } - } - } - else + if (!set_root_from_app(managed_application_path, args)) { trace::error(_X("Failed to locate managed application [%s]"), args.managed_application.c_str()); return false; From 7d2c74eb91bf301ec5be20b45b02f03bf893bcff Mon Sep 17 00:00:00 2001 From: Swaroop Sridhar Date: Tue, 7 Apr 2020 01:31:58 -0700 Subject: [PATCH 16/18] HostInterface: Don't pass host::info pointer across hostfxr/hostpolicy. Host::info* is not POD, so pass the bundle-header offset instead. The deps-json and runtime-config json locations are recomputed in hostpolicy. Since hostpolicy needs to re-map the bundle, recomputing these locations is not expensive than passing them through the interface. This also keeps the host_interface cleaner. --- src/installer/corehost/cli/bundle/header.cpp | 4 +- src/installer/corehost/cli/bundle/header.h | 4 -- src/installer/corehost/cli/bundle/info.cpp | 43 ++++++++++--------- src/installer/corehost/cli/bundle/info.h | 30 +++++-------- src/installer/corehost/cli/bundle/runner.cpp | 9 +++- src/installer/corehost/cli/bundle/runner.h | 10 ++--- .../corehost/cli/fxr/corehost_init.cpp | 2 +- .../corehost/cli/fxr/hostpolicy_resolver.cpp | 1 - src/installer/corehost/cli/host_interface.h | 4 +- .../corehost/cli/hostpolicy/args.cpp | 2 +- .../cli/hostpolicy/hostpolicy_init.cpp | 6 +-- .../test/mockhostpolicy/mockhostpolicy.cpp | 2 +- 12 files changed, 53 insertions(+), 64 deletions(-) diff --git a/src/installer/corehost/cli/bundle/header.cpp b/src/installer/corehost/cli/bundle/header.cpp index 73c166232c9acd..7e6f778464da4e 100644 --- a/src/installer/corehost/cli/bundle/header.cpp +++ b/src/installer/corehost/cli/bundle/header.cpp @@ -36,12 +36,10 @@ header_t header_t::read(reader_t& reader) header_t header(fixed_header->num_embedded_files); // bundle_id is a component of the extraction path - size_t bundle_id_size = reader.read_path_string(header.m_bundle_id); + reader.read_path_string(header.m_bundle_id); const header_fixed_v2_t *v2_header = reinterpret_cast(reader.read_direct(sizeof(header_fixed_v2_t))); header.m_v2_header = *v2_header; - header.m_header_size = sizeof(header_fixed_t) + bundle_id_size + sizeof(header_fixed_v2_t); - return header; } diff --git a/src/installer/corehost/cli/bundle/header.h b/src/installer/corehost/cli/bundle/header.h index f9dbe6b94b0852..f9ec85f42213cf 100644 --- a/src/installer/corehost/cli/bundle/header.h +++ b/src/installer/corehost/cli/bundle/header.h @@ -77,7 +77,6 @@ namespace bundle : m_num_embedded_files(num_embedded_files) , m_bundle_id() , m_v2_header() - , m_header_size(0) { } @@ -89,8 +88,6 @@ namespace bundle const location_t& runtimeconfig_json_location() const { return m_v2_header.runtimeconfig_json_location; } bool is_netcoreapp3_compat_mode() const { return m_v2_header.is_netcoreapp3_compat_mode(); } - size_t size() const { return m_header_size; } - static const uint32_t major_version = 2; static const uint32_t minor_version = 0; @@ -98,7 +95,6 @@ namespace bundle int32_t m_num_embedded_files; pal::string_t m_bundle_id; header_fixed_v2_t m_v2_header; - size_t m_header_size; }; } #endif // __HEADER_H__ diff --git a/src/installer/corehost/cli/bundle/info.cpp b/src/installer/corehost/cli/bundle/info.cpp index f505606ca12bcd..cf2b387775d39b 100644 --- a/src/installer/corehost/cli/bundle/info.cpp +++ b/src/installer/corehost/cli/bundle/info.cpp @@ -11,6 +11,26 @@ using namespace bundle; // Global single-file bundle information, if any const info_t* info_t::the_app = nullptr; +info_t::info_t(const pal::char_t* bundle_path, + const pal::char_t* app_path, + int64_t header_offset) + : m_bundle_path(bundle_path) + , m_bundle_size(0) + , m_header_offset(header_offset) +{ + m_base_path = get_directory(m_bundle_path); + + // Single-file bundles currently only support deps/runtime config json files + // named based on the app.dll. Any other name for these configuration files + // mentioned via the command line are assumed to be actual files on disk. + // + // Supporting custom names for these config files is straightforward (with associated changes in bundler and SDK). + // There is no known use-case for it yet, and the facility is TBD. + + m_deps_json = config_t(get_deps_from_app_binary(m_base_path, app_path)); + m_runtimeconfig_json = config_t(get_runtime_config_path(m_base_path, get_filename_without_ext(app_path))); +} + StatusCode info_t::process_bundle(const pal::char_t* bundle_path, const pal::char_t* app_path, int64_t header_offset) { if (header_offset == 0) @@ -19,7 +39,7 @@ StatusCode info_t::process_bundle(const pal::char_t* bundle_path, const pal::cha return StatusCode::Success; } - static info_t info(bundle_path, header_offset); + static info_t info(bundle_path, app_path, header_offset); StatusCode status = info.process_header(); if (status != StatusCode::Success) @@ -27,8 +47,6 @@ StatusCode info_t::process_bundle(const pal::char_t* bundle_path, const pal::cha return status; } - info.init_config(app_path); - trace::info(_X("Single-File bundle details:")); trace::info(_X("DepsJson Offset:[%lx] Size[%lx]"), info.m_header.deps_json_location().offset, info.m_header.deps_json_location().size); trace::info(_X("RuntimeConfigJson Offset:[%lx] Size[%lx]"), info.m_header.runtimeconfig_json_location().offset, info.m_header.runtimeconfig_json_location().size); @@ -48,6 +66,8 @@ StatusCode info_t::process_header() reader_t reader(addr, m_bundle_size, m_header_offset); m_header = header_t::read(reader); + m_deps_json.set_location(&m_header.deps_json_location()); + m_runtimeconfig_json.set_location(&m_header.runtimeconfig_json_location()); unmap_bundle(addr); @@ -59,23 +79,6 @@ StatusCode info_t::process_header() } } -void info_t::init_config(const pal::string_t& app_path) -{ - // Single-file bundles currently only support deps/runtime config json files - // named based on the app.dll. Any other name for these configuration files - // mentioned via the command line are assumed to be actual files on disk. - // - // Supporting custom names for these config files is straightforward (with associated changes in bundler and SDK). - // There is no known use-case for it yet, and the facility is TBD. - - m_base_path = get_directory(m_bundle_path); - pal::string_t deps_json_name = get_deps_from_app_binary(m_base_path, app_path); - pal::string_t runtimeconfig_json_name = get_runtime_config_path(m_base_path, get_filename_without_ext(app_path)); - - m_deps_json = config_t(deps_json_name, &m_header.deps_json_location()); - m_runtimeconfig_json = config_t(runtimeconfig_json_name, &m_header.runtimeconfig_json_location()); -} - int8_t* info_t::config_t::map(const pal::string_t& path, const location_t* &location) { assert(is_single_file_bundle()); diff --git a/src/installer/corehost/cli/bundle/info.h b/src/installer/corehost/cli/bundle/info.h index 11f5c7c949d59a..e3cae564b128f7 100644 --- a/src/installer/corehost/cli/bundle/info.h +++ b/src/installer/corehost/cli/bundle/info.h @@ -28,7 +28,7 @@ namespace bundle m_location = config.m_location; } - config_t(const pal::string_t& path, const location_t *location) + config_t(const pal::string_t& path, const location_t *location=nullptr) { m_path = path; m_location = location; @@ -45,6 +45,11 @@ namespace bundle (the_app->m_deps_json.matches(path) || the_app->m_runtimeconfig_json.matches(path)); } + void set_location(const location_t* location) + { + m_location = location; + } + static int8_t* map(const pal::string_t& path, const location_t* &location); static void unmap(const int8_t* addr, const location_t* location); @@ -58,29 +63,15 @@ namespace bundle bool is_netcoreapp3_compat_mode() const { return m_header.is_netcoreapp3_compat_mode(); } const pal::string_t& base_path() const { return m_base_path; } + int64_t header_offset() const { return m_header_offset; } // Global single-file info object static const info_t* the_app; protected: - info_t(const pal::char_t* bundle_path_value, - int64_t header_offset_value) - : m_bundle_path(bundle_path_value) - , m_bundle_size(0) - , m_header_offset(header_offset_value) - , m_deps_json() - , m_runtimeconfig_json() {} - - info_t(const info_t* info) - { - m_bundle_path = info->m_bundle_path; - m_base_path = info->m_base_path; - m_bundle_size = info->m_bundle_size; - m_header_offset = info->m_header_offset; - m_header = info->m_header; - m_deps_json = info->m_deps_json; - m_runtimeconfig_json = info->m_runtimeconfig_json; - } + info_t(const pal::char_t* bundle_path, + const pal::char_t* app_path, + int64_t header_offset); const int8_t* map_bundle(); void unmap_bundle(const int8_t* addr) const; @@ -94,7 +85,6 @@ namespace bundle config_t m_runtimeconfig_json; private: - void init_config(const pal::string_t& app_path); StatusCode process_header(); }; } diff --git a/src/installer/corehost/cli/bundle/runner.cpp b/src/installer/corehost/cli/bundle/runner.cpp index 514e5abf754549..92526c122938cc 100644 --- a/src/installer/corehost/cli/bundle/runner.cpp +++ b/src/installer/corehost/cli/bundle/runner.cpp @@ -20,8 +20,13 @@ StatusCode runner_t::extract() { const int8_t* addr = map_bundle(); - // Set the Reader offset to read post the bundle header - reader_t reader(addr, m_bundle_size, m_header_offset + m_header.size()); + // Set the Reader at header_offset + reader_t reader(addr, m_bundle_size, m_header_offset); + + // Read the bundle header + m_header = header_t::read(reader); + m_deps_json.set_location(&m_header.deps_json_location()); + m_runtimeconfig_json.set_location(&m_header.runtimeconfig_json_location()); // Read the bundle manifest m_manifest = manifest_t::read(reader, m_header.num_embedded_files()); diff --git a/src/installer/corehost/cli/bundle/runner.h b/src/installer/corehost/cli/bundle/runner.h index a3873c33410b1a..fb4d74bf877a2f 100644 --- a/src/installer/corehost/cli/bundle/runner.h +++ b/src/installer/corehost/cli/bundle/runner.h @@ -20,12 +20,10 @@ namespace bundle class runner_t : public info_t { public: - runner_t(const pal::char_t* bundle_path_value, - int64_t header_offset_value) - : info_t(bundle_path_value, header_offset_value) {} - - runner_t(const bundle::info_t* info) - : info_t(info) {} + runner_t(const pal::char_t* bundle_path, + const pal::char_t *app_path, + int64_t header_offset) + : info_t(bundle_path, app_path, header_offset) {} const pal::string_t& extraction_path() const { return m_extraction_path; } diff --git a/src/installer/corehost/cli/fxr/corehost_init.cpp b/src/installer/corehost/cli/fxr/corehost_init.cpp index c388c77e03ddeb..d8a5f2498297ba 100644 --- a/src/installer/corehost/cli/fxr/corehost_init.cpp +++ b/src/installer/corehost/cli/fxr/corehost_init.cpp @@ -133,7 +133,7 @@ const host_interface_t& corehost_init_t::get_host_init_data() hi.host_info_dotnet_root = m_host_info_dotnet_root.c_str(); hi.host_info_app_path = m_host_info_app_path.c_str(); - hi.app_bundle = bundle::info_t::the_app; + hi.single_file_bundle_header_offset = bundle::info_t::is_single_file_bundle() ? bundle::info_t::the_app->header_offset() : 0; return hi; } diff --git a/src/installer/corehost/cli/fxr/hostpolicy_resolver.cpp b/src/installer/corehost/cli/fxr/hostpolicy_resolver.cpp index c4616646bc05d2..de3291fecc5b53 100644 --- a/src/installer/corehost/cli/fxr/hostpolicy_resolver.cpp +++ b/src/installer/corehost/cli/fxr/hostpolicy_resolver.cpp @@ -12,7 +12,6 @@ #include #include "json_parser.h" -#include "bundle/info.h" namespace { diff --git a/src/installer/corehost/cli/host_interface.h b/src/installer/corehost/cli/host_interface.h index 34894b0b698665..72295aee99e06b 100644 --- a/src/installer/corehost/cli/host_interface.h +++ b/src/installer/corehost/cli/host_interface.h @@ -59,7 +59,7 @@ struct host_interface_t const pal::char_t* host_info_host_path; const pal::char_t* host_info_dotnet_root; const pal::char_t* host_info_app_path; - const bundle::info_t *app_bundle; + size_t single_file_bundle_header_offset; // !! WARNING / WARNING / WARNING / WARNING / WARNING / WARNING / WARNING / WARNING / WARNING // !! 1. Only append to this structure to maintain compat. // !! 2. Any nested structs should not use compiler specific padding (pack with _HOST_INTERFACE_PACK) @@ -93,7 +93,7 @@ static_assert(offsetof(host_interface_t, host_command) == 26 * sizeof(size_t), " static_assert(offsetof(host_interface_t, host_info_host_path) == 27 * sizeof(size_t), "Struct offset breaks backwards compatibility"); static_assert(offsetof(host_interface_t, host_info_dotnet_root) == 28 * sizeof(size_t), "Struct offset breaks backwards compatibility"); static_assert(offsetof(host_interface_t, host_info_app_path) == 29 * sizeof(size_t), "Struct offset breaks backwards compatibility"); -static_assert(offsetof(host_interface_t, app_bundle) == 30 * sizeof(size_t), "Struct offset breaks backwards compatibility"); +static_assert(offsetof(host_interface_t, single_file_bundle_header_offset) == 30 * sizeof(size_t), "Struct offset breaks backwards compatibility"); static_assert(sizeof(host_interface_t) == 31 * sizeof(size_t), "Did you add static asserts for the newly added fields?"); #define HOST_INTERFACE_LAYOUT_VERSION_HI 0x16041101 // YYMMDD:nn always increases when layout breaks compat. diff --git a/src/installer/corehost/cli/hostpolicy/args.cpp b/src/installer/corehost/cli/hostpolicy/args.cpp index b846d76142bd86..f022d23ae5f3c3 100644 --- a/src/installer/corehost/cli/hostpolicy/args.cpp +++ b/src/installer/corehost/cli/hostpolicy/args.cpp @@ -136,7 +136,7 @@ bool set_root_from_app(const pal::string_t& managed_application_path, if (pal::realpath(&args.managed_application)) { - args.app_root = get_directory(managed_application_path); + args.app_root = get_directory(args.managed_application); return true; } diff --git a/src/installer/corehost/cli/hostpolicy/hostpolicy_init.cpp b/src/installer/corehost/cli/hostpolicy/hostpolicy_init.cpp index a496defb5ab55c..3e0d02a17c5db3 100644 --- a/src/installer/corehost/cli/hostpolicy/hostpolicy_init.cpp +++ b/src/installer/corehost/cli/hostpolicy/hostpolicy_init.cpp @@ -127,11 +127,11 @@ bool hostpolicy_init_t::init(host_interface_t* input, hostpolicy_init_t* init) // For the backwards compat case, this will be later initialized with argv[0] } - if (input->version_lo >= offsetof(host_interface_t, app_bundle) + sizeof(input->app_bundle)) + if (input->version_lo >= offsetof(host_interface_t, single_file_bundle_header_offset) + sizeof(input->single_file_bundle_header_offset)) { - if (input->app_bundle != nullptr) + if (input->single_file_bundle_header_offset != 0) { - static bundle::runner_t bundle_runner(input->app_bundle); + static bundle::runner_t bundle_runner(input->host_info_host_path, input->host_info_app_path, input->single_file_bundle_header_offset); bundle::info_t::the_app = &bundle_runner; } } diff --git a/src/installer/corehost/cli/test/mockhostpolicy/mockhostpolicy.cpp b/src/installer/corehost/cli/test/mockhostpolicy/mockhostpolicy.cpp index 9f5386982d24ff..2130fe7b2f9393 100644 --- a/src/installer/corehost/cli/test/mockhostpolicy/mockhostpolicy.cpp +++ b/src/installer/corehost/cli/test/mockhostpolicy/mockhostpolicy.cpp @@ -66,7 +66,7 @@ SHARED_API int HOSTPOLICY_CALLTYPE corehost_load(host_interface_t* init) std::cout << "mock host_info_host_path:" << tostr(init->host_info_host_path).data() << std::endl; std::cout << "mock host_info_dotnet_root:" << tostr(init->host_info_dotnet_root).data() << std::endl; std::cout << "mock host_info_app_path:" << tostr(init->host_info_app_path).data() << std::endl; - std::cout << "mock app_bundle:" << std::hex << init->app_bundle << std::endl; + std::cout << "mock single_file_bundle_header_offset:" << std::hex << init->single_file_bundle_header_offset << std::endl; if (init->fx_names.len == 0) { From 6fdc8431e68b702b4a7df766604a9a39c953e900 Mon Sep 17 00:00:00 2001 From: Swaroop Sridhar Date: Tue, 7 Apr 2020 15:32:29 -0700 Subject: [PATCH 17/18] Fix an mmap() call --- src/installer/corehost/cli/bundle/info.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/installer/corehost/cli/bundle/info.cpp b/src/installer/corehost/cli/bundle/info.cpp index cf2b387775d39b..0208484c9acd69 100644 --- a/src/installer/corehost/cli/bundle/info.cpp +++ b/src/installer/corehost/cli/bundle/info.cpp @@ -107,7 +107,7 @@ int8_t* info_t::config_t::map(const pal::string_t& path, const location_t* &loca // * There is no performance limitation due to a larger sized mapping, since we actually only read the pages with relevant contents. // * Files that are too large to be mapped (ex: that exhaust 32-bit virtual address space) are not supported. - int8_t* addr = (int8_t*)pal::mmap_read(app->m_bundle_path); + int8_t* addr = (int8_t*)pal::mmap_copy_on_write(app->m_bundle_path); if (addr == nullptr) { trace::error(_X("Failure processing application bundle.")); From 9aaffc409ea9340107cd317fedc3c5ff5f96600f Mon Sep 17 00:00:00 2001 From: Swaroop Sridhar Date: Tue, 7 Apr 2020 18:09:56 -0700 Subject: [PATCH 18/18] Fix json_parser: Unmap only during destruction. --- src/installer/corehost/cli/bundle/info.cpp | 25 ++++++++++++------- src/installer/corehost/cli/bundle/info.h | 8 +++--- src/installer/corehost/cli/bundle/reader.cpp | 8 +++--- src/installer/corehost/cli/bundle/reader.h | 16 ++++++------ src/installer/corehost/cli/bundle/runner.cpp | 2 +- src/installer/corehost/cli/hostmisc/pal.h | 2 +- .../corehost/cli/hostmisc/pal.unix.cpp | 10 ++++---- .../corehost/cli/hostmisc/pal.windows.cpp | 2 +- src/installer/corehost/cli/json_parser.cpp | 21 +++++++++++----- src/installer/corehost/cli/json_parser.h | 11 ++++++++ 10 files changed, 66 insertions(+), 39 deletions(-) diff --git a/src/installer/corehost/cli/bundle/info.cpp b/src/installer/corehost/cli/bundle/info.cpp index 0208484c9acd69..3fef4220890668 100644 --- a/src/installer/corehost/cli/bundle/info.cpp +++ b/src/installer/corehost/cli/bundle/info.cpp @@ -61,7 +61,7 @@ StatusCode info_t::process_header() { try { - const int8_t* addr = map_bundle(); + const char* addr = map_bundle(); reader_t reader(addr, m_bundle_size, m_header_offset); @@ -79,7 +79,7 @@ StatusCode info_t::process_header() } } -int8_t* info_t::config_t::map(const pal::string_t& path, const location_t* &location) +char* info_t::config_t::map(const pal::string_t& path, const location_t* &location) { assert(is_single_file_bundle()); @@ -107,27 +107,28 @@ int8_t* info_t::config_t::map(const pal::string_t& path, const location_t* &loca // * There is no performance limitation due to a larger sized mapping, since we actually only read the pages with relevant contents. // * Files that are too large to be mapped (ex: that exhaust 32-bit virtual address space) are not supported. - int8_t* addr = (int8_t*)pal::mmap_copy_on_write(app->m_bundle_path); + char* addr = (char*)pal::mmap_copy_on_write(app->m_bundle_path); if (addr == nullptr) { trace::error(_X("Failure processing application bundle.")); trace::error(_X("Failed to map bundle file [%s]"), path.c_str()); } + trace::info(_X("Mapped bundle for [%s]"), path.c_str()); + return addr + location->offset; } -void info_t::config_t::unmap(const int8_t* addr, const location_t* location) +void info_t::config_t::unmap(const char* addr, const location_t* location) { // Adjust to the beginning of the bundle. addr -= location->offset; - bundle::info_t::the_app->unmap_bundle(addr); } -const int8_t* info_t::map_bundle() +const char* info_t::map_bundle() { - void *addr = pal::mmap_read(m_bundle_path, &m_bundle_size); + const void *addr = pal::mmap_read(m_bundle_path, &m_bundle_size); if (addr == nullptr) { @@ -136,15 +137,21 @@ const int8_t* info_t::map_bundle() throw StatusCode::BundleExtractionIOError; } - return (int8_t *)addr; + trace::info(_X("Mapped application bundle")); + + return (const char *)addr; } -void info_t::unmap_bundle(const int8_t* addr) const +void info_t::unmap_bundle(const char* addr) const { if (!pal::munmap((void*)addr, m_bundle_size)) { trace::warning(_X("Failed to unmap bundle after extraction.")); } + else + { + trace::info(_X("Unmapped application bundle")); + } } diff --git a/src/installer/corehost/cli/bundle/info.h b/src/installer/corehost/cli/bundle/info.h index e3cae564b128f7..0f3db559fc2221 100644 --- a/src/installer/corehost/cli/bundle/info.h +++ b/src/installer/corehost/cli/bundle/info.h @@ -50,8 +50,8 @@ namespace bundle m_location = location; } - static int8_t* map(const pal::string_t& path, const location_t* &location); - static void unmap(const int8_t* addr, const location_t* location); + static char* map(const pal::string_t& path, const location_t* &location); + static void unmap(const char* addr, const location_t* location); private: pal::string_t m_path; @@ -73,8 +73,8 @@ namespace bundle const pal::char_t* app_path, int64_t header_offset); - const int8_t* map_bundle(); - void unmap_bundle(const int8_t* addr) const; + const char* map_bundle(); + void unmap_bundle(const char* addr) const; pal::string_t m_bundle_path; pal::string_t m_base_path; diff --git a/src/installer/corehost/cli/bundle/reader.cpp b/src/installer/corehost/cli/bundle/reader.cpp index 26c306d60c4daa..2e5135da564632 100644 --- a/src/installer/corehost/cli/bundle/reader.cpp +++ b/src/installer/corehost/cli/bundle/reader.cpp @@ -8,9 +8,9 @@ using namespace bundle; -const int8_t* reader_t::add_without_overflow(const int8_t* ptr, int64_t len) +const char* reader_t::add_without_overflow(const char* ptr, int64_t len) { - const int8_t* new_ptr = ptr + len; + const char* new_ptr = ptr + len; // The following check will fail in case len < 0 (which is also an error while reading) // even if the actual arthmetic didn't overflow. @@ -38,7 +38,7 @@ void reader_t::set_offset(int64_t offset) void reader_t::bounds_check(int64_t len) { - const int8_t* post_read_ptr = add_without_overflow(m_ptr, len); + const char* post_read_ptr = add_without_overflow(m_ptr, len); // It is legal for post_read_ptr == m_bound_ptr after reading the last byte. if (m_ptr < m_base_ptr || post_read_ptr > m_bound_ptr) @@ -90,7 +90,7 @@ size_t reader_t::read_path_length() size_t reader_t::read_path_string(pal::string_t &str) { - const int8_t* start_ptr = m_ptr; + const char* start_ptr = m_ptr; size_t size = read_path_length(); std::unique_ptr buffer{ new uint8_t[size + 1] }; read(buffer.get(), size); diff --git a/src/installer/corehost/cli/bundle/reader.h b/src/installer/corehost/cli/bundle/reader.h index 38eaf37561ec4f..8a1212d94b8249 100644 --- a/src/installer/corehost/cli/bundle/reader.h +++ b/src/installer/corehost/cli/bundle/reader.h @@ -13,7 +13,7 @@ namespace bundle // Helper class for reading sequentially from the memory-mapped bundle file. struct reader_t { - reader_t(const int8_t* base_ptr, int64_t bound, int64_t start_offset = 0) + reader_t(const char* base_ptr, int64_t bound, int64_t start_offset = 0) : m_base_ptr(base_ptr) , m_ptr(base_ptr) , m_bound(bound) @@ -26,7 +26,7 @@ namespace bundle void set_offset(int64_t offset); - operator const int8_t*() const + operator const char*() const { return m_ptr; } @@ -47,10 +47,10 @@ namespace bundle // Return a pointer to the requested bytes within the memory-mapped file. // Skip over len bytes. - const int8_t* read_direct(int64_t len) + const char* read_direct(int64_t len) { bounds_check(len); - const int8_t *ptr = m_ptr; + const char *ptr = m_ptr; m_ptr += len; return ptr; } @@ -61,12 +61,12 @@ namespace bundle private: void bounds_check(int64_t len = 1); - static const int8_t* add_without_overflow(const int8_t* ptr, int64_t len); + static const char* add_without_overflow(const char* ptr, int64_t len); - const int8_t* const m_base_ptr; - const int8_t* m_ptr; + const char* const m_base_ptr; + const char* m_ptr; const int64_t m_bound; - const int8_t* const m_bound_ptr; + const char* const m_bound_ptr; }; } diff --git a/src/installer/corehost/cli/bundle/runner.cpp b/src/installer/corehost/cli/bundle/runner.cpp index 92526c122938cc..58d10a817855f9 100644 --- a/src/installer/corehost/cli/bundle/runner.cpp +++ b/src/installer/corehost/cli/bundle/runner.cpp @@ -18,7 +18,7 @@ StatusCode runner_t::extract() { try { - const int8_t* addr = map_bundle(); + const char* addr = map_bundle(); // Set the Reader at header_offset reader_t reader(addr, m_bundle_size, m_header_offset); diff --git a/src/installer/corehost/cli/hostmisc/pal.h b/src/installer/corehost/cli/hostmisc/pal.h index 8f7a7566189d8e..c35d34c3c5c4d8 100644 --- a/src/installer/corehost/cli/hostmisc/pal.h +++ b/src/installer/corehost/cli/hostmisc/pal.h @@ -256,7 +256,7 @@ namespace pal return fallbackRid; } - void* mmap_read(const string_t& path, size_t* length = nullptr); + const void* mmap_read(const string_t& path, size_t* length = nullptr); void* mmap_copy_on_write(const string_t& path, size_t* length = nullptr); bool touch_file(const string_t& path); diff --git a/src/installer/corehost/cli/hostmisc/pal.unix.cpp b/src/installer/corehost/cli/hostmisc/pal.unix.cpp index 9f145f765762ce..dac760d8959346 100644 --- a/src/installer/corehost/cli/hostmisc/pal.unix.cpp +++ b/src/installer/corehost/cli/hostmisc/pal.unix.cpp @@ -58,7 +58,7 @@ bool pal::touch_file(const pal::string_t& path) return true; } -static void* map_file(const pal::string_t& path, size_t* length, int flags) +static void* map_file(const pal::string_t& path, size_t* length, int prot, int flags) { int fd = open(path.c_str(), O_RDONLY); if (fd == -1) @@ -81,7 +81,7 @@ static void* map_file(const pal::string_t& path, size_t* length, int flags) *length = size; } - void* address = mmap(nullptr, size, PROT_READ, flags, fd, 0); + void* address = mmap(nullptr, size, prot, flags, fd, 0); if (address == MAP_FAILED) { @@ -93,14 +93,14 @@ static void* map_file(const pal::string_t& path, size_t* length, int flags) return address; } -void* pal::mmap_read(const string_t& path, size_t* length) +const void* pal::mmap_read(const string_t& path, size_t* length) { - return map_file(path, length, MAP_SHARED); + return map_file(path, length, PROT_READ, MAP_SHARED); } void* pal::mmap_copy_on_write(const string_t& path, size_t* length) { - return map_file(path, length, MAP_PRIVATE); + return map_file(path, length, PROT_READ | PROT_WRITE, MAP_PRIVATE); } bool pal::getcwd(pal::string_t* recv) diff --git a/src/installer/corehost/cli/hostmisc/pal.windows.cpp b/src/installer/corehost/cli/hostmisc/pal.windows.cpp index 0cfe036de38254..f8348a05d167b0 100644 --- a/src/installer/corehost/cli/hostmisc/pal.windows.cpp +++ b/src/installer/corehost/cli/hostmisc/pal.windows.cpp @@ -123,7 +123,7 @@ static void* map_file(const pal::string_t& path, size_t *length, DWORD mapping_p return address; } -void* pal::mmap_read(const string_t& path, size_t* length) +const void* pal::mmap_read(const string_t& path, size_t* length) { return map_file(path, length, PAGE_READONLY, FILE_MAP_READ); } diff --git a/src/installer/corehost/cli/json_parser.cpp b/src/installer/corehost/cli/json_parser.cpp index 8b7d8e456c70b9..c6f0cbbf0c9127 100644 --- a/src/installer/corehost/cli/json_parser.cpp +++ b/src/installer/corehost/cli/json_parser.cpp @@ -15,7 +15,6 @@ #include "utils.h" #include #include -#include "bundle/info.h" namespace { @@ -136,16 +135,18 @@ bool json_parser_t::parse_file(const pal::string_t& path) { // This code assumes that the caller has checked that the file `path` exists // either within the bundle, or as a real file on disk. + assert(m_bundle_data == nullptr); + assert(m_bundle_location == nullptr); if (bundle::info_t::is_single_file_bundle()) { - const bundle::location_t *location = nullptr; - char* data = (char*) bundle::info_t::config_t::map(path, location); + m_bundle_data = bundle::info_t::config_t::map(path, m_bundle_location); + // The mapping will be unmapped by the json_parser destructor. + // The mapping cannot be immediately released due to in-situ parsing on Linux. - if (data != nullptr) + if (m_bundle_data != nullptr) { - bool result = parse_json(data, location->size, path); - bundle::info_t::config_t::unmap((const int8_t*)data, location); + bool result = parse_json(m_bundle_data, m_bundle_location->size, path); return result; } } @@ -153,3 +154,11 @@ bool json_parser_t::parse_file(const pal::string_t& path) pal::ifstream_t file{ path }; return parse_stream(file, path); } + +json_parser_t::~json_parser_t() +{ + if (m_bundle_data != nullptr) + { + bundle::info_t::config_t::unmap(m_bundle_data, m_bundle_location); + } +} diff --git a/src/installer/corehost/cli/json_parser.h b/src/installer/corehost/cli/json_parser.h index 04e995a0a9e076..16ed21bd033072 100644 --- a/src/installer/corehost/cli/json_parser.h +++ b/src/installer/corehost/cli/json_parser.h @@ -9,6 +9,7 @@ #include "rapidjson/document.h" #include "rapidjson/fwd.h" #include +#include "bundle/info.h" class json_parser_t { public: @@ -25,6 +26,12 @@ class json_parser_t { bool parse_stream(pal::istream_t& stream, const pal::string_t& context); bool parse_file(const pal::string_t& path); + json_parser_t() + : m_bundle_data(nullptr) + , m_bundle_location(nullptr) {} + + ~json_parser_t(); + private: // This is a vector of char and not pal::char_t because JSON data // parsed by this class is always encoded in UTF-8. On Windows, @@ -33,6 +40,10 @@ class json_parser_t { std::vector m_json; document_t m_document; + // If a json file is parsed from a single-file bundle, the following two fields represent: + char* m_bundle_data; // The memory mapped bytes of the application bundle. + const bundle::location_t* m_bundle_location; // Location of this json file within the bundle. + void realloc_buffer(size_t size); bool parse_json(char* data, int64_t size, const pal::string_t& context); };