diff --git a/Documentation/coding-guidelines/interop-guidelines.md b/Documentation/coding-guidelines/interop-guidelines.md index 64250da4186b..80ffe1ead09b 100644 --- a/Documentation/coding-guidelines/interop-guidelines.md +++ b/Documentation/coding-guidelines/interop-guidelines.md @@ -18,6 +18,10 @@ We have the following goals related to interop code being used in CoreFX: - This is both for good hygiene and to help keep platform-specific code separated from platform-neutral code, which is important for maximizing reusable code above PAL layers. +- Ensure maximal managed code reuse across different OS flavors which have + the same API but not the same ABI. + - This is the case for UNIX and addressing it is a work-in-progress (see issue + #2137 and section on "shims" below.) ## Approach @@ -244,3 +248,43 @@ rather than the underlying integral type. should be named with the most discoverable name possible; if that name is a concept (e.g. Errors), it can be named using managed naming guidelines. + + +## UNIX shims + +Often, various UNIX flavors offer the same API from the point-of-view of compatibility +with C/C++ source code, but they do not have the same ABI. e.g. Fields can be laid out +differently, constants cn have different numeric values, exports can +be named differently, etc. There are not only differences between operating systems +(Mac OS X vs. Ubuntu vs. FreeBSD), but also differences related to the underlying +processor architecture (x64 vs. x86 vs. ARM). + +This leaves us with a situation where we can't write portable P/Invoke declarations +that will work on all flavors, and writing separate declarations per flavor is quite +fragile and won't scale. + +To address this, we're moving to a model where all UNIX interop from corefx starts with +a P/Invoke to a C++ lib written specifically for corefx. These libs -- System.*.Native.so +(aka "shims") -- are intended to be very thin layers over underlying platform libraries. +Generally, they are not there to add any significant abstraction, but to create a +stable ABI such that the same IL assembly can work across UNIX flavors. + +Guidelines for shim C++ API: + +- Keep them as "thin"/1:1 as possible. + - We want to write the majority of code in C#. +- Never skip the shim and P/Invoke directly to the underlying platform API. It's +easy to assume something is safe/guaranteed when it isn't. +- Don't cheat and take advantage of coincidental agreement between +one flavor's ABI and the shim's ABI. +- Use PascalCase in a style closer to Win32 than libc. + - If an export point has a 1:1 correspondence to the platform API, then name + it after the platform API in PascalCase (e.g. stat -> Stat, fstat -> FStat). + - If an export is not 1:1, then spell things out as we typically would in + CoreFX code (i.e. don't use abbreviations unless they come from the underlying + API. + - At first, it seemed that we'd want to use 1:1 names throughout, but it + turns out there are many cases where being strictly 1:1 isn't practical. +- Stick to data types which are guaranteed not to vary in size across flavors. + - e.g. use int32_t, int64_t from stdint.h and not int, long. + diff --git a/run-test.sh b/run-test.sh index edc7f919ab00..0410f48db807 100755 --- a/run-test.sh +++ b/run-test.sh @@ -128,6 +128,12 @@ create_test_overlay() exit 1 fi find $CoreFxBins -name '*.dll' -exec cp '{}' "$OverlayDir" ";" + + # Then the native CoreFX binaries + # + # TODO: Currently, CI does not build the native CoreFX components so build them here + # in the test phase for now. + ( $ProjectRoot/src/Native/build.sh && cp $ProjectRoot/bin/$OS.x64.$Configuration/Native/* $OverlayDir ) || exit 1 } copy_test_overlay() diff --git a/src/Common/src/Interop/Unix/Interop.Libraries.cs b/src/Common/src/Interop/Unix/Interop.Libraries.cs index 8f6fb30dc6c9..b8464795ba4c 100644 --- a/src/Common/src/Interop/Unix/Interop.Libraries.cs +++ b/src/Common/src/Interop/Unix/Interop.Libraries.cs @@ -9,6 +9,7 @@ private static partial class Libraries internal const string LibCoreClr= "libcoreclr"; // CoreCLR runtime internal const string LibCrypto = "libcrypto"; // OpenSSL crypto library internal const string Zlib = "libz"; // zlib compression library + internal const string IOInterop = "System.IO.Native"; internal const string CryptoInterop = "System.Security.Cryptography.Native"; } } diff --git a/src/Common/src/Interop/Unix/System.IO.Native/Interop.NativeIO.cs b/src/Common/src/Interop/Unix/System.IO.Native/Interop.NativeIO.cs new file mode 100644 index 000000000000..903e18809f50 --- /dev/null +++ b/src/Common/src/Interop/Unix/System.IO.Native/Interop.NativeIO.cs @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Runtime.InteropServices; + +internal static partial class Interop +{ + internal static partial class NativeIO + { + internal struct FileStats + { + private FileStatsFlags Flags; + internal int Mode; + internal int Uid; + internal int Gid; + internal int Size; + internal int AccessTime; + internal int ModificationTime; + internal int StatusChangeTime; + internal int CreationTime; + } + + internal static class FileTypes + { + internal const int S_IFMT = 0xF000; + internal const int S_IFIFO = 0x1000; + internal const int S_IFCHR = 0x2000; + internal const int S_IFDIR = 0x4000; + internal const int S_IFREG = 0x8000; + internal const int S_IFLNK = 0xA000; + } + + [Flags] + internal enum FileStatsFlags + { + None = 0, + HasCreationTime = 1, + } + + [DllImport(Libraries.IOInterop, SetLastError = true)] + internal static extern int FStat(int fileDescriptor, out FileStats output); + + [DllImport(Libraries.IOInterop, SetLastError = true)] + internal static extern int Stat(string path, out FileStats output); + } +} diff --git a/src/Native/CMakeLists.txt b/src/Native/CMakeLists.txt new file mode 100644 index 000000000000..b3a4657e6f47 --- /dev/null +++ b/src/Native/CMakeLists.txt @@ -0,0 +1,26 @@ +cmake_minimum_required(VERSION 2.8.12) +project(CoreFX) + +set(CMAKE_INSTALL_PREFIX $ENV{__CMakeBinDir}) +set(CMAKE_INCLUDE_CURRENT_DIR ON) +set(CMAKE_C_FLAGS "-std=c11") +set(CMAKE_CXX_FLAGS "-std=c++11") +set(CMAKE_SHARED_LIBRARY_PREFIX "") +add_compile_options(-Wall -Werror -fPIC) + +if (CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64 OR CMAKE_SYSTEM_PROCESSOR STREQUAL amd64) + add_definitions(-DBIT64=1) +endif () + +string(TOUPPER ${CMAKE_BUILD_TYPE} UPPERCASE_CMAKE_BUILD_TYPE) +if (UPPERCASE_CMAKE_BUILD_TYPE STREQUAL DEBUG) + add_compile_options(-g -O0) + add_definitions(-DDEBUG) +elseif (UPPERCASE_CMAKE_BUILD_TYPE STREQUAL RELEASE) + add_compile_options (-O3) + add_definitions(-DNDEBUG) +else () + message(FATAL_ERROR "Unknown build type. Set CMAKE_BUILD_TYPE to DEBUG or RELEASE.") +endif () + +add_subdirectory(System.IO.Native) \ No newline at end of file diff --git a/src/Native/System.IO.Native/CMakeLists.txt b/src/Native/System.IO.Native/CMakeLists.txt new file mode 100644 index 000000000000..defb98e1a4a7 --- /dev/null +++ b/src/Native/System.IO.Native/CMakeLists.txt @@ -0,0 +1,15 @@ + +project(System.IO.Native) + +set(NATIVEIO_SOURCES + nativeio.h + nativeio.cpp +) + +add_library(System.IO.Native + SHARED + ${NATIVEIO_SOURCES} +) + +install (TARGETS System.IO.Native DESTINATION .) + diff --git a/src/Native/System.IO.Native/nativeio.cpp b/src/Native/System.IO.Native/nativeio.cpp new file mode 100755 index 000000000000..b7a66a39332a --- /dev/null +++ b/src/Native/System.IO.Native/nativeio.cpp @@ -0,0 +1,61 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +#include "nativeio.h" +#include + +#if HAVE_STAT64 && !(defined(__APPLE__) && defined(_AMD64_)) +# define stat_ stat64 +# define fstat_ fstat64 +#else +# define stat_ stat +# define fstat_ fstat +#endif + +static void ConvertFileStats(const struct stat_& src, FileStats* dst) +{ + dst->Flags = FILESTATS_FLAGS_NONE; + dst->Mode = src.st_mode; + dst->Uid = src.st_uid; + dst->Gid = src.st_gid; + dst->Size = src.st_size; + dst->AccessTime = src.st_atime; + dst->ModificationTime = src.st_mtime; + dst->StatusChangeTime = src.st_ctime; + +#if HAVE_STAT_BIRTHTIME + dst->CreationTime = src->st_birthtime; + dst->Flags |= FILESTATS_FLAGS_HAS_CREATION_TIME; +#endif +} + +extern "C" +{ + int32_t Stat(const char* path, struct FileStats* output) + { + struct stat_ result; + int ret = stat_(path, &result); + + if (ret == 0) + { + ConvertFileStats(result, output); + } + + return ret; // TODO: errno conversion + } + + int32_t FStat(int32_t fileDescriptor, FileStats* output) + { + struct stat_ result; + int ret = fstat_(fileDescriptor, &result); + + if (ret == 0) + { + ConvertFileStats(result, output); + } + + return ret; // TODO: errno conversion + } +} diff --git a/src/Native/System.IO.Native/nativeio.h b/src/Native/System.IO.Native/nativeio.h new file mode 100755 index 000000000000..b33466ca0057 --- /dev/null +++ b/src/Native/System.IO.Native/nativeio.h @@ -0,0 +1,45 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +#pragma once + +#include + +extern "C" +{ + struct FileStats + { + int32_t Flags; // flags for testing if some members are present (see values below) + int32_t Mode; // protection + int32_t Uid; // user ID of owner + int32_t Gid; // group ID of owner + int64_t Size; // total size, in bytes + int64_t AccessTime; // time of last access (atime) + int64_t ModificationTime; // time of last modification (mtime) + int64_t StatusChangeTime; // time of last status change (ctime) + int64_t CreationTime; // time the file was created (birthtime) + }; + + enum + { + FILESTATS_FLAGS_NONE = 0, + FILESTATS_FLAGS_HAS_CREATION_TIME = 1, + }; + + /** + * Get file stats from a decriptor. Implemented as shim to fstat(2). + * + * Returns 0 for success, -1 for failure. Sets errno on failure. + */ + int32_t FStat(int32_t fileDescriptor, FileStats* output); + + /** + * Get file stats from a full path. Implemented as shim to stat(2). + * + * Returns 0 for success, -1 for failure. Sets errno on failure. + */ + int32_t Stat(const char* path, FileStats* output); +} + diff --git a/src/Native/build.sh b/src/Native/build.sh new file mode 100755 index 000000000000..a2e77911b701 --- /dev/null +++ b/src/Native/build.sh @@ -0,0 +1,215 @@ +#!/usr/bin/env bash + +usage() +{ + echo "Usage: $0 [BuildArch] [BuildType] [clean] [verbose] [clangx.y]" + echo "BuildArch can be: x64" + echo "BuildType can be: Debug, Release" + echo "clean - optional argument to force a clean build." + echo "verbose - optional argument to enable verbose build output." + echo "clangx.y - optional argument to build using clang version x.y." + + exit 1 +} + +setup_dirs() +{ + echo Setting up directories for build + + mkdir -p "$__BinDir" + mkdir -p "$__IntermediatesDir" +} + +# Performs "clean build" type actions (deleting and remaking directories) + +clean() +{ + echo Cleaning previous output for the selected configuration + rm -rf "$__BinDir" + rm -rf "$__IntermediatesDir" +} + +# Check the system to ensure the right pre-reqs are in place + +check_prereqs() +{ + echo "Checking pre-requisites..." + + # Check presence of CMake on the path + hash cmake 2>/dev/null || { echo >&2 "Please install cmake before running this script"; exit 1; } + + # Check for clang + hash clang-$__ClangMajorVersion.$__ClangMinorVersion 2>/dev/null || hash clang$__ClangMajorVersion$__ClangMinorVersion 2>/dev/null || hash clang 2>/dev/null || { echo >&2 "Please install clang before running this script"; exit 1; } +} + +build_corefx_native() +{ + # All set to commence the build + + echo "Commencing build of corefx native components for $__BuildOS.$__BuildArch.$__BuildType" + cd "$__IntermediatesDir" + + # Regenerate the CMake solution + echo "Invoking cmake with arguments: \"$__ScriptDir\" $__CMakeArgs" + "$__ScriptDir/gen-buildsys-clang.sh" "$__ScriptDir" $__ClangMajorVersion $__ClangMinorVersion $__CMakeArgs + + # Check that the makefiles were created. + + if [ ! -f "$__IntermediatesDir/Makefile" ]; then + echo "Failed to generate native component build project!" + exit 1 + fi + + # Get the number of processors available to the scheduler + # Other techniques such as `nproc` only get the number of + # processors available to a single process. + if [ `uname` = "FreeBSD" ]; then + NumProc=`sysctl hw.ncpu | awk '{ print $2+1 }'` + else + NumProc=$(($(getconf _NPROCESSORS_ONLN)+1)) + fi + + # Build + + echo "Executing make install -j $NumProc $__UnprocessedBuildArgs" + + make install -j $NumProc $__UnprocessedBuildArgs + if [ $? != 0 ]; then + echo "Failed to build corefx native components." + exit 1 + fi +} + +echo "Commencing CoreFX Native build" + +# Argument types supported by this script: +# +# Build architecture - valid value is: x64. +# Build Type - valid values are: Debug, Release +# +# Set the default arguments for build + +# Obtain the location of the bash script to figure out whether the root of the repo is. + +__ScriptDir="$( dirname "${BASH_SOURCE[0]}" )" +__ScriptDir="$( cd $__ScriptDir && pwd )" +__ProjectRoot="$( cd $__ScriptDir && cd ../.. && pwd )" +__BuildArch=x64 +# Use uname to determine what the OS is. +OSName=$(uname -s) +case $OSName in + Linux) + __BuildOS=Linux + ;; + + Darwin) + __BuildOS=OSX + ;; + + FreeBSD) + __BuildOS=FreeBSD + ;; + + OpenBSD) + __BuildOS=OpenBSD + ;; + + NetBSD) + __BuildOS=NetBSD + ;; + + *) + echo "Unsupported OS $OSName detected, configuring as if for Linux" + __BuildOS=Linux + ;; +esac +__BuildType=Debug +__CMakeArgs=DEBUG + +# Set the various build properties here so that CMake and MSBuild can pick them up +__ProjectDir="$__ProjectRoot" +__SourceDir="$__ScriptDir" +__RootBinDir="$__ProjectDir/bin" +__UnprocessedBuildArgs= +__CleanBuild=false +__VerboseBuild=false +__ClangMajorVersion=3 +__ClangMinorVersion=5 + +for i in "$@" + do + lowerI="$(echo $i | awk '{print tolower($0)}')" + case $lowerI in + -?|-h|--help) + usage + exit 1 + ;; + x64) + __BuildArch=x64 + __MSBuildBuildArch=x64 + ;; + debug) + __BuildType=Debug + ;; + release) + __BuildType=Release + __CMakeArgs=RELEASE + ;; + clean) + __CleanBuild=1 + ;; + verbose) + __VerboseBuild=1 + ;; + clang3.5) + __ClangMajorVersion=3 + __ClangMinorVersion=5 + ;; + clang3.6) + __ClangMajorVersion=3 + __ClangMinorVersion=6 + ;; + clang3.7) + __ClangMajorVersion=3 + __ClangMinorVersion=7 + ;; + *) + __UnprocessedBuildArgs="$__UnprocessedBuildArgs $i" + esac +done + +# Set the remaining variables based upon the determined build configuration +__IntermediatesDir="$__RootBinDir/obj/$__BuildOS.$__BuildArch.$__BuildType/Native" +__BinDir="$__RootBinDir/$__BuildOS.$__BuildArch.$__BuildType/Native" + +# Specify path to be set for CMAKE_INSTALL_PREFIX. +# This is where all built CoreClr libraries will copied to. +export __CMakeBinDir="$__BinDir" + +# Configure environment if we are doing a clean build. +if [ $__CleanBuild == 1 ]; then + clean +fi + +# Configure environment if we are doing a verbose build +if [ $__VerboseBuild == 1 ]; then + export VERBOSE=1 +fi + +# Make the directories necessary for build if they don't exist + +setup_dirs + +# Check prereqs. + +check_prereqs + +# Build the corefx native components. + +build_corefx_native + +# Build complete + +echo "CoreFX native components successfully built." +echo "Product binaries are available at $__BinDir" +exit 0 diff --git a/src/Native/gen-buildsys-clang.sh b/src/Native/gen-buildsys-clang.sh new file mode 100755 index 000000000000..85160845f6ae --- /dev/null +++ b/src/Native/gen-buildsys-clang.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# +# This file invokes cmake and generates the build system for gcc. +# + +if [ $# -lt 3 -o $# -gt 4 ] +then + echo "Usage..." + echo "gen-buildsys-clang.sh [build flavor]" + echo "Specify the path to the top level CMake file - /src/Native" + echo "Specify the clang version to use, split into major and minor version" + echo "Optionally specify the build configuration (flavor.) Defaults to DEBUG." + exit 1 +fi + +# Set up the environment to be used for building with clang. +if which "clang-$2.$3" > /dev/null 2>&1 + then + export CC="$(which clang-$2.$3)" + export CXX="$(which clang++-$2.$3)" +elif which "clang$2$3" > /dev/null 2>&1 + then + export CC="$(which clang$2$3)" + export CXX="$(which clang++$2$3)" +elif which clang > /dev/null 2>&1 + then + export CC="$(which clang)" + export CXX="$(which clang++)" +else + echo "Unable to find Clang Compiler" + exit 1 +fi + +# Possible build types are DEBUG, RELEASE, RELWITHDEBINFO, MINSIZEREL. +# Default to DEBUG +if [ -z "$4" ] +then + echo "Defaulting to DEBUG build." + buildtype="DEBUG" +else + buildtype="$4" +fi + +OS=`uname` + +# Locate llvm +# This can be a little complicated, because the common use-case of Ubuntu with +# llvm-3.5 installed uses a rather unusual llvm installation with the version +# number postfixed (i.e. llvm-ar-3.5), so we check for that first. +# On FreeBSD the version number is appended without point and dash (i.e. +# llvm-ar35). +# Additionally, OSX doesn't use the llvm- prefix. +if [ $OS = "Linux" -o $OS = "FreeBSD" -o $OS = "OpenBSD" -o $OS = "NetBSD" ]; then + llvm_prefix="llvm-" +elif [ $OS = "Darwin" ]; then + llvm_prefix="" +else + echo "Unable to determine build platform" + exit 1 +fi + +desired_llvm_major_version=$2 +desired_llvm_minor_version=$3 +if [ $OS = "FreeBSD" ]; then + desired_llvm_version="$desired_llvm_major_version$desired_llvm_minor_version" +elif [ $OS = "OpenBSD" ]; then + desired_llvm_version="" +elif [ $OS = "NetBSD" ]; then + desired_llvm_version="" +else + desired_llvm_version="-$desired_llvm_major_version.$desired_llvm_minor_version" +fi +locate_llvm_exec() { + if which "$llvm_prefix$1$desired_llvm_version" > /dev/null 2>&1 + then + echo "$(which $llvm_prefix$1$desired_llvm_version)" + elif which "$llvm_prefix$1" > /dev/null 2>&1 + then + echo "$(which $llvm_prefix$1)" + else + exit 1 + fi +} + +llvm_ar="$(locate_llvm_exec ar)" +[[ $? -eq 0 ]] || { echo "Unable to locate llvm-ar"; exit 1; } +llvm_link="$(locate_llvm_exec link)" +[[ $? -eq 0 ]] || { echo "Unable to locate llvm-link"; exit 1; } +llvm_nm="$(locate_llvm_exec nm)" +[[ $? -eq 0 ]] || { echo "Unable to locate llvm-nm"; exit 1; } +llvm_ranlib="$(locate_llvm_exec ranlib)" +[[ $? -eq 0 ]] || { echo "Unable to locate llvm-ranlib"; exit 1; } +if [ $OS = "Linux" -o $OS = "FreeBSD" -o $OS = "OpenBSD" -o $OS = "NetBSD" ]; then + llvm_objdump="$(locate_llvm_exec objdump)" + [[ $? -eq 0 ]] || { echo "Unable to locate llvm-objdump"; exit 1; } +fi + +cmake_extra_defines= +if [[ -n "$LLDB_LIB_DIR" ]]; then + cmake_extra_defines="$cmake_extra_defines -DWITH_LLDB_LIBS=$LLDB_LIB_DIR" +fi +if [[ -n "$LLDB_INCLUDE_DIR" ]]; then + cmake_extra_defines="$cmake_extra_defines -DWITH_LLDB_INCLUDES=$LLDB_INCLUDE_DIR" +fi + +cmake \ + "-DCMAKE_AR=$llvm_ar" \ + "-DCMAKE_LINKER=$llvm_link" \ + "-DCMAKE_NM=$llvm_nm" \ + "-DCMAKE_OBJDUMP=$llvm_objdump" \ + "-DCMAKE_RANLIB=$llvm_ranlib" \ + "-DCMAKE_BUILD_TYPE=$buildtype" \ + $cmake_extra_defines \ + "$1" diff --git a/src/System.Console/src/System.Console.csproj b/src/System.Console/src/System.Console.csproj index 813b8d393add..dc9c29008c6d 100644 --- a/src/System.Console/src/System.Console.csproj +++ b/src/System.Console/src/System.Console.csproj @@ -138,6 +138,9 @@ Common\Interop\Unix\Interop.GetFileInformation.cs" + + Common\Interop\Unix\Interop.NativeIO.cs" + Common\Interop\Unix\Interop.SetConsoleCtrlHandler.cs" diff --git a/src/System.Console/src/System/ConsolePal.Unix.cs b/src/System.Console/src/System/ConsolePal.Unix.cs index afad581dec29..d3c1251c8e6b 100644 --- a/src/System.Console/src/System/ConsolePal.Unix.cs +++ b/src/System.Console/src/System/ConsolePal.Unix.cs @@ -91,7 +91,7 @@ private static bool ConsoleOutIsTerminal UnixConsoleStream ucs = sw.BaseStream as UnixConsoleStream; if (ucs != null) { - return ucs._handleType == Interop.libcoreclr.FileTypes.S_IFCHR; + return ucs._handleType == Interop.NativeIO.FileTypes.S_IFCHR; } } } @@ -378,11 +378,11 @@ internal UnixConsoleStream(string devPath, FileAccess access) try { _handle.DangerousAddRef(ref gotFd); - Interop.libcoreclr.fileinfo buf; + Interop.NativeIO.FileStats buf; _handleType = - Interop.libcoreclr.GetFileInformationFromFd((int)_handle.DangerousGetHandle(), out buf) == 0 ? - (buf.mode & Interop.libcoreclr.FileTypes.S_IFMT) : - Interop.libcoreclr.FileTypes.S_IFREG; // if something goes wrong, don't fail, just say it's a regular file + Interop.NativeIO.FStat((int)_handle.DangerousGetHandle(), out buf) == 0 ? + (buf.Mode & Interop.NativeIO.FileTypes.S_IFMT) : + Interop.NativeIO.FileTypes.S_IFREG; // if something goes wrong, don't fail, just say it's a regular file } finally {