From 1f42d12ea7898b9dbd46bb454d05de0eed65e36b Mon Sep 17 00:00:00 2001 From: dotnet-bot Date: Tue, 21 Jul 2015 14:33:46 -0700 Subject: [PATCH 1/8] Copy build scripts from dotnet/coreclr --- src/Native/build.sh | 228 +++++++++++++++++++++++++++++++ src/Native/gen-buildsys-clang.sh | 115 ++++++++++++++++ 2 files changed, 343 insertions(+) create mode 100755 src/Native/build.sh create mode 100755 src/Native/gen-buildsys-clang.sh diff --git a/src/Native/build.sh b/src/Native/build.sh new file mode 100755 index 000000000000..fe5577cba487 --- /dev/null +++ b/src/Native/build.sh @@ -0,0 +1,228 @@ +#!/usr/bin/env bash + +usage() +{ + echo "Usage: $0 [BuildArch] [BuildType] [clean] [verbose] [clangx.y]" + echo "BuildArch can be: x64" + echo "BuildType can be: Debug, Release" + echo "clean - optional argument to force a clean build." + echo "verbose - optional argument to enable verbose build output." + echo "clangx.y - optional argument to build using clang version x.y." + + exit 1 +} + +setup_dirs() +{ + echo Setting up directories for build + + mkdir -p "$__RootBinDir" + mkdir -p "$__BinDir" + mkdir -p "$__LogsDir" + mkdir -p "$__IntermediatesDir" +} + +# Performs "clean build" type actions (deleting and remaking directories) + +clean() +{ + echo Cleaning previous output for the selected configuration + rm -rf "$__BinDir" + rm -rf "$__IntermediatesDir" + + rm -rf "$__TestWorkingDir" + rm -rf "$__TestIntermediatesDir" + + rm -rf "$__LogsDir/*_$__BuildOS__$__BuildArch__$__BuildType.*" +} + +# Check the system to ensure the right pre-reqs are in place + +check_prereqs() +{ + echo "Checking pre-requisites..." + + # Check presence of CMake on the path + hash cmake 2>/dev/null || { echo >&2 "Please install cmake before running this script"; exit 1; } + + # Check for clang + hash clang-$__ClangMajorVersion.$__ClangMinorVersion 2>/dev/null || hash clang$__ClangMajorVersion$__ClangMinorVersion 2>/dev/null || hash clang 2>/dev/null || { echo >&2 "Please install clang before running this script"; exit 1; } + +} + +build_coreclr() +{ + # All set to commence the build + + echo "Commencing build of native components for $__BuildOS.$__BuildArch.$__BuildType" + cd "$__IntermediatesDir" + + # Regenerate the CMake solution + echo "Invoking cmake with arguments: \"$__ProjectRoot\" $__CMakeArgs" + "$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh" "$__ProjectRoot" $__ClangMajorVersion $__ClangMinorVersion $__CMakeArgs + + # Check that the makefiles were created. + + if [ ! -f "$__IntermediatesDir/Makefile" ]; then + echo "Failed to generate native component build project!" + exit 1 + fi + + # Get the number of processors available to the scheduler + # Other techniques such as `nproc` only get the number of + # processors available to a single process. + if [ `uname` = "FreeBSD" ]; then + NumProc=`sysctl hw.ncpu | awk '{ print $2+1 }'` + else + NumProc=$(($(getconf _NPROCESSORS_ONLN)+1)) + fi + + # Build CoreCLR + + echo "Executing make install -j $NumProc $__UnprocessedBuildArgs" + + make install -j $NumProc $__UnprocessedBuildArgs + if [ $? != 0 ]; then + echo "Failed to build coreclr components." + exit 1 + fi +} + +echo "Commencing CoreCLR Repo build" + +# Argument types supported by this script: +# +# Build architecture - valid value is: x64. +# Build Type - valid values are: Debug, Release +# +# Set the default arguments for build + +# Obtain the location of the bash script to figure out whether the root of the repo is. +__ProjectRoot="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +__BuildArch=x64 +# Use uname to determine what the OS is. +OSName=$(uname -s) +case $OSName in + Linux) + __BuildOS=Linux + ;; + + Darwin) + __BuildOS=OSX + ;; + + FreeBSD) + __BuildOS=FreeBSD + ;; + + OpenBSD) + __BuildOS=OpenBSD + ;; + + NetBSD) + __BuildOS=NetBSD + ;; + + *) + echo "Unsupported OS $OSName detected, configuring as if for Linux" + __BuildOS=Linux + ;; +esac +__MSBuildBuildArch=x64 +__BuildType=Debug +__CMakeArgs=DEBUG + +# Set the various build properties here so that CMake and MSBuild can pick them up +__ProjectDir="$__ProjectRoot" +__SourceDir="$__ProjectDir/src" +__PackagesDir="$__ProjectDir/packages" +__RootBinDir="$__ProjectDir/bin" +__LogsDir="$__RootBinDir/Logs" +__UnprocessedBuildArgs= +__MSBCleanBuildArgs= +__CleanBuild=false +__VerboseBuild=false +__ClangMajorVersion=3 +__ClangMinorVersion=5 + +for i in "$@" + do + lowerI="$(echo $i | awk '{print tolower($0)}')" + case $lowerI in + -?|-h|--help) + usage + exit 1 + ;; + x64) + __BuildArch=x64 + __MSBuildBuildArch=x64 + ;; + debug) + __BuildType=Debug + ;; + release) + __BuildType=Release + __CMakeArgs=RELEASE + ;; + clean) + __CleanBuild=1 + ;; + verbose) + __VerboseBuild=1 + ;; + clang3.5) + __ClangMajorVersion=3 + __ClangMinorVersion=5 + ;; + clang3.6) + __ClangMajorVersion=3 + __ClangMinorVersion=6 + ;; + clang3.7) + __ClangMajorVersion=3 + __ClangMinorVersion=7 + ;; + *) + __UnprocessedBuildArgs="$__UnprocessedBuildArgs $i" + esac +done + +# Set the remaining variables based upon the determined build configuration +__BinDir="$__RootBinDir/Product/$__BuildOS.$__BuildArch.$__BuildType" +__PackagesBinDir="$__BinDir/.nuget" +__ToolsDir="$__RootBinDir/tools" +__TestWorkingDir="$__RootBinDir/tests/$__BuildOS.$__BuildArch.$__BuildType" +__IntermediatesDir="$__RootBinDir/obj/$__BuildOS.$__BuildArch.$__BuildType" +__TestIntermediatesDir="$__RootBinDir/tests/obj/$__BuildOS.$__BuildArch.$__BuildType" + +# Specify path to be set for CMAKE_INSTALL_PREFIX. +# This is where all built CoreClr libraries will copied to. +export __CMakeBinDir="$__BinDir" + +# Configure environment if we are doing a clean build. +if [ $__CleanBuild == 1 ]; then + clean +fi + +# Configure environment if we are doing a verbose build +if [ $__VerboseBuild == 1 ]; then + export VERBOSE=1 +fi + +# Make the directories necessary for build if they don't exist + +setup_dirs + +# Check prereqs. + +check_prereqs + +# Build the coreclr (native) components. + +build_coreclr + +# Build complete + +echo "Repo successfully built." +echo "Product binaries are available at $__BinDir" +exit 0 diff --git a/src/Native/gen-buildsys-clang.sh b/src/Native/gen-buildsys-clang.sh new file mode 100755 index 000000000000..91ac5c821a06 --- /dev/null +++ b/src/Native/gen-buildsys-clang.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash +# +# This file invokes cmake and generates the build system for gcc. +# + +if [ $# -lt 3 -o $# -gt 4 ] +then + echo "Usage..." + echo "gen-buildsys-clang.sh [build flavor]" + echo "Specify the path to the top level CMake file - /src/NDP" + echo "Specify the clang version to use, split into major and minor version" + echo "Optionally specify the build configuration (flavor.) Defaults to DEBUG." + exit 1 +fi + +# Set up the environment to be used for building with clang. +if which "clang-$2.$3" > /dev/null 2>&1 + then + export CC="$(which clang-$2.$3)" + export CXX="$(which clang++-$2.$3)" +elif which "clang$2$3" > /dev/null 2>&1 + then + export CC="$(which clang$2$3)" + export CXX="$(which clang++$2$3)" +elif which clang > /dev/null 2>&1 + then + export CC="$(which clang)" + export CXX="$(which clang++)" +else + echo "Unable to find Clang Compiler" + exit 1 +fi + +# Possible build types are DEBUG, RELEASE, RELWITHDEBINFO, MINSIZEREL. +# Default to DEBUG +if [ -z "$4" ] +then + echo "Defaulting to DEBUG build." + buildtype="DEBUG" +else + buildtype="$4" +fi + +OS=`uname` + +# Locate llvm +# This can be a little complicated, because the common use-case of Ubuntu with +# llvm-3.5 installed uses a rather unusual llvm installation with the version +# number postfixed (i.e. llvm-ar-3.5), so we check for that first. +# On FreeBSD the version number is appended without point and dash (i.e. +# llvm-ar35). +# Additionally, OSX doesn't use the llvm- prefix. +if [ $OS = "Linux" -o $OS = "FreeBSD" -o $OS = "OpenBSD" -o $OS = "NetBSD" ]; then + llvm_prefix="llvm-" +elif [ $OS = "Darwin" ]; then + llvm_prefix="" +else + echo "Unable to determine build platform" + exit 1 +fi + +desired_llvm_major_version=$2 +desired_llvm_minor_version=$3 +if [ $OS = "FreeBSD" ]; then + desired_llvm_version="$desired_llvm_major_version$desired_llvm_minor_version" +elif [ $OS = "OpenBSD" ]; then + desired_llvm_version="" +elif [ $OS = "NetBSD" ]; then + desired_llvm_version="" +else + desired_llvm_version="-$desired_llvm_major_version.$desired_llvm_minor_version" +fi +locate_llvm_exec() { + if which "$llvm_prefix$1$desired_llvm_version" > /dev/null 2>&1 + then + echo "$(which $llvm_prefix$1$desired_llvm_version)" + elif which "$llvm_prefix$1" > /dev/null 2>&1 + then + echo "$(which $llvm_prefix$1)" + else + exit 1 + fi +} + +llvm_ar="$(locate_llvm_exec ar)" +[[ $? -eq 0 ]] || { echo "Unable to locate llvm-ar"; exit 1; } +llvm_link="$(locate_llvm_exec link)" +[[ $? -eq 0 ]] || { echo "Unable to locate llvm-link"; exit 1; } +llvm_nm="$(locate_llvm_exec nm)" +[[ $? -eq 0 ]] || { echo "Unable to locate llvm-nm"; exit 1; } +llvm_ranlib="$(locate_llvm_exec ranlib)" +[[ $? -eq 0 ]] || { echo "Unable to locate llvm-ranlib"; exit 1; } +if [ $OS = "Linux" -o $OS = "FreeBSD" -o $OS = "OpenBSD" -o $OS = "NetBSD" ]; then + llvm_objdump="$(locate_llvm_exec objdump)" + [[ $? -eq 0 ]] || { echo "Unable to locate llvm-objdump"; exit 1; } +fi + +cmake_extra_defines= +if [[ -n "$LLDB_LIB_DIR" ]]; then + cmake_extra_defines="$cmake_extra_defines -DWITH_LLDB_LIBS=$LLDB_LIB_DIR" +fi +if [[ -n "$LLDB_INCLUDE_DIR" ]]; then + cmake_extra_defines="$cmake_extra_defines -DWITH_LLDB_INCLUDES=$LLDB_INCLUDE_DIR" +fi + +cmake \ + "-DCMAKE_USER_MAKE_RULES_OVERRIDE=$1/src/pal/tools/clang-compiler-override.txt" \ + "-DCMAKE_AR=$llvm_ar" \ + "-DCMAKE_LINKER=$llvm_link" \ + "-DCMAKE_NM=$llvm_nm" \ + "-DCMAKE_OBJDUMP=$llvm_objdump" \ + "-DCMAKE_RANLIB=$llvm_ranlib" \ + "-DCMAKE_BUILD_TYPE=$buildtype" \ + $cmake_extra_defines \ + "$1" From 314d081b33e00359247441cc11b37aa12de1895c Mon Sep 17 00:00:00 2001 From: Nick Guerrera Date: Tue, 21 Jul 2015 15:50:02 -0700 Subject: [PATCH 2/8] Clean up whitespace --- src/Native/build.sh | 61 ++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 31 deletions(-) diff --git a/src/Native/build.sh b/src/Native/build.sh index fe5577cba487..60d35bb2ba84 100755 --- a/src/Native/build.sh +++ b/src/Native/build.sh @@ -29,10 +29,10 @@ clean() echo Cleaning previous output for the selected configuration rm -rf "$__BinDir" rm -rf "$__IntermediatesDir" - + rm -rf "$__TestWorkingDir" rm -rf "$__TestIntermediatesDir" - + rm -rf "$__LogsDir/*_$__BuildOS__$__BuildArch__$__BuildType.*" } @@ -47,7 +47,6 @@ check_prereqs() # Check for clang hash clang-$__ClangMajorVersion.$__ClangMinorVersion 2>/dev/null || hash clang$__ClangMajorVersion$__ClangMinorVersion 2>/dev/null || hash clang 2>/dev/null || { echo >&2 "Please install clang before running this script"; exit 1; } - } build_coreclr() @@ -72,9 +71,9 @@ build_coreclr() # Other techniques such as `nproc` only get the number of # processors available to a single process. if [ `uname` = "FreeBSD" ]; then - NumProc=`sysctl hw.ncpu | awk '{ print $2+1 }'` + NumProc=`sysctl hw.ncpu | awk '{ print $2+1 }'` else - NumProc=$(($(getconf _NPROCESSORS_ONLN)+1)) + NumProc=$(($(getconf _NPROCESSORS_ONLN)+1)) fi # Build CoreCLR @@ -150,40 +149,40 @@ for i in "$@" lowerI="$(echo $i | awk '{print tolower($0)}')" case $lowerI in -?|-h|--help) - usage - exit 1 - ;; + usage + exit 1 + ;; x64) - __BuildArch=x64 - __MSBuildBuildArch=x64 - ;; + __BuildArch=x64 + __MSBuildBuildArch=x64 + ;; debug) - __BuildType=Debug - ;; + __BuildType=Debug + ;; release) - __BuildType=Release - __CMakeArgs=RELEASE - ;; + __BuildType=Release + __CMakeArgs=RELEASE + ;; clean) - __CleanBuild=1 - ;; + __CleanBuild=1 + ;; verbose) - __VerboseBuild=1 - ;; + __VerboseBuild=1 + ;; clang3.5) - __ClangMajorVersion=3 - __ClangMinorVersion=5 - ;; + __ClangMajorVersion=3 + __ClangMinorVersion=5 + ;; clang3.6) - __ClangMajorVersion=3 - __ClangMinorVersion=6 - ;; + __ClangMajorVersion=3 + __ClangMinorVersion=6 + ;; clang3.7) - __ClangMajorVersion=3 - __ClangMinorVersion=7 - ;; + __ClangMajorVersion=3 + __ClangMinorVersion=7 + ;; *) - __UnprocessedBuildArgs="$__UnprocessedBuildArgs $i" + __UnprocessedBuildArgs="$__UnprocessedBuildArgs $i" esac done @@ -206,7 +205,7 @@ fi # Configure environment if we are doing a verbose build if [ $__VerboseBuild == 1 ]; then - export VERBOSE=1 + export VERBOSE=1 fi # Make the directories necessary for build if they don't exist From 2715fe6ef61f04d26ed348c5a9663f7f181c8476 Mon Sep 17 00:00:00 2001 From: Nick Guerrera Date: Tue, 21 Jul 2015 15:52:51 -0700 Subject: [PATCH 3/8] Adjust copied scripts to be happy in their new corefx home --- src/Native/build.sh | 48 ++++++++++++-------------------- src/Native/gen-buildsys-clang.sh | 3 +- 2 files changed, 19 insertions(+), 32 deletions(-) diff --git a/src/Native/build.sh b/src/Native/build.sh index 60d35bb2ba84..a2e77911b701 100755 --- a/src/Native/build.sh +++ b/src/Native/build.sh @@ -16,9 +16,7 @@ setup_dirs() { echo Setting up directories for build - mkdir -p "$__RootBinDir" mkdir -p "$__BinDir" - mkdir -p "$__LogsDir" mkdir -p "$__IntermediatesDir" } @@ -29,11 +27,6 @@ clean() echo Cleaning previous output for the selected configuration rm -rf "$__BinDir" rm -rf "$__IntermediatesDir" - - rm -rf "$__TestWorkingDir" - rm -rf "$__TestIntermediatesDir" - - rm -rf "$__LogsDir/*_$__BuildOS__$__BuildArch__$__BuildType.*" } # Check the system to ensure the right pre-reqs are in place @@ -49,16 +42,16 @@ check_prereqs() hash clang-$__ClangMajorVersion.$__ClangMinorVersion 2>/dev/null || hash clang$__ClangMajorVersion$__ClangMinorVersion 2>/dev/null || hash clang 2>/dev/null || { echo >&2 "Please install clang before running this script"; exit 1; } } -build_coreclr() +build_corefx_native() { # All set to commence the build - echo "Commencing build of native components for $__BuildOS.$__BuildArch.$__BuildType" + echo "Commencing build of corefx native components for $__BuildOS.$__BuildArch.$__BuildType" cd "$__IntermediatesDir" # Regenerate the CMake solution - echo "Invoking cmake with arguments: \"$__ProjectRoot\" $__CMakeArgs" - "$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh" "$__ProjectRoot" $__ClangMajorVersion $__ClangMinorVersion $__CMakeArgs + echo "Invoking cmake with arguments: \"$__ScriptDir\" $__CMakeArgs" + "$__ScriptDir/gen-buildsys-clang.sh" "$__ScriptDir" $__ClangMajorVersion $__ClangMinorVersion $__CMakeArgs # Check that the makefiles were created. @@ -75,19 +68,19 @@ build_coreclr() else NumProc=$(($(getconf _NPROCESSORS_ONLN)+1)) fi - - # Build CoreCLR + + # Build echo "Executing make install -j $NumProc $__UnprocessedBuildArgs" make install -j $NumProc $__UnprocessedBuildArgs if [ $? != 0 ]; then - echo "Failed to build coreclr components." + echo "Failed to build corefx native components." exit 1 fi } -echo "Commencing CoreCLR Repo build" +echo "Commencing CoreFX Native build" # Argument types supported by this script: # @@ -97,7 +90,10 @@ echo "Commencing CoreCLR Repo build" # Set the default arguments for build # Obtain the location of the bash script to figure out whether the root of the repo is. -__ProjectRoot="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +__ScriptDir="$( dirname "${BASH_SOURCE[0]}" )" +__ScriptDir="$( cd $__ScriptDir && pwd )" +__ProjectRoot="$( cd $__ScriptDir && cd ../.. && pwd )" __BuildArch=x64 # Use uname to determine what the OS is. OSName=$(uname -s) @@ -127,18 +123,14 @@ case $OSName in __BuildOS=Linux ;; esac -__MSBuildBuildArch=x64 __BuildType=Debug __CMakeArgs=DEBUG # Set the various build properties here so that CMake and MSBuild can pick them up __ProjectDir="$__ProjectRoot" -__SourceDir="$__ProjectDir/src" -__PackagesDir="$__ProjectDir/packages" +__SourceDir="$__ScriptDir" __RootBinDir="$__ProjectDir/bin" -__LogsDir="$__RootBinDir/Logs" __UnprocessedBuildArgs= -__MSBCleanBuildArgs= __CleanBuild=false __VerboseBuild=false __ClangMajorVersion=3 @@ -187,12 +179,8 @@ for i in "$@" done # Set the remaining variables based upon the determined build configuration -__BinDir="$__RootBinDir/Product/$__BuildOS.$__BuildArch.$__BuildType" -__PackagesBinDir="$__BinDir/.nuget" -__ToolsDir="$__RootBinDir/tools" -__TestWorkingDir="$__RootBinDir/tests/$__BuildOS.$__BuildArch.$__BuildType" -__IntermediatesDir="$__RootBinDir/obj/$__BuildOS.$__BuildArch.$__BuildType" -__TestIntermediatesDir="$__RootBinDir/tests/obj/$__BuildOS.$__BuildArch.$__BuildType" +__IntermediatesDir="$__RootBinDir/obj/$__BuildOS.$__BuildArch.$__BuildType/Native" +__BinDir="$__RootBinDir/$__BuildOS.$__BuildArch.$__BuildType/Native" # Specify path to be set for CMAKE_INSTALL_PREFIX. # This is where all built CoreClr libraries will copied to. @@ -216,12 +204,12 @@ setup_dirs check_prereqs -# Build the coreclr (native) components. +# Build the corefx native components. -build_coreclr +build_corefx_native # Build complete -echo "Repo successfully built." +echo "CoreFX native components successfully built." echo "Product binaries are available at $__BinDir" exit 0 diff --git a/src/Native/gen-buildsys-clang.sh b/src/Native/gen-buildsys-clang.sh index 91ac5c821a06..85160845f6ae 100755 --- a/src/Native/gen-buildsys-clang.sh +++ b/src/Native/gen-buildsys-clang.sh @@ -7,7 +7,7 @@ if [ $# -lt 3 -o $# -gt 4 ] then echo "Usage..." echo "gen-buildsys-clang.sh [build flavor]" - echo "Specify the path to the top level CMake file - /src/NDP" + echo "Specify the path to the top level CMake file - /src/Native" echo "Specify the clang version to use, split into major and minor version" echo "Optionally specify the build configuration (flavor.) Defaults to DEBUG." exit 1 @@ -104,7 +104,6 @@ if [[ -n "$LLDB_INCLUDE_DIR" ]]; then fi cmake \ - "-DCMAKE_USER_MAKE_RULES_OVERRIDE=$1/src/pal/tools/clang-compiler-override.txt" \ "-DCMAKE_AR=$llvm_ar" \ "-DCMAKE_LINKER=$llvm_link" \ "-DCMAKE_NM=$llvm_nm" \ From f802208594818c56ee9432def4842dfdfa7e7aca Mon Sep 17 00:00:00 2001 From: Nick Guerrera Date: Tue, 21 Jul 2015 18:46:33 -0700 Subject: [PATCH 4/8] Initial commit of System.IO.Native with stat shims --- src/Native/CMakeLists.txt | 26 +++++++++ src/Native/System.IO.Native/CMakeLists.txt | 15 ++++++ src/Native/System.IO.Native/nativeio.cpp | 61 ++++++++++++++++++++++ src/Native/System.IO.Native/nativeio.h | 45 ++++++++++++++++ 4 files changed, 147 insertions(+) create mode 100644 src/Native/CMakeLists.txt create mode 100644 src/Native/System.IO.Native/CMakeLists.txt create mode 100755 src/Native/System.IO.Native/nativeio.cpp create mode 100755 src/Native/System.IO.Native/nativeio.h diff --git a/src/Native/CMakeLists.txt b/src/Native/CMakeLists.txt new file mode 100644 index 000000000000..b3a4657e6f47 --- /dev/null +++ b/src/Native/CMakeLists.txt @@ -0,0 +1,26 @@ +cmake_minimum_required(VERSION 2.8.12) +project(CoreFX) + +set(CMAKE_INSTALL_PREFIX $ENV{__CMakeBinDir}) +set(CMAKE_INCLUDE_CURRENT_DIR ON) +set(CMAKE_C_FLAGS "-std=c11") +set(CMAKE_CXX_FLAGS "-std=c++11") +set(CMAKE_SHARED_LIBRARY_PREFIX "") +add_compile_options(-Wall -Werror -fPIC) + +if (CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64 OR CMAKE_SYSTEM_PROCESSOR STREQUAL amd64) + add_definitions(-DBIT64=1) +endif () + +string(TOUPPER ${CMAKE_BUILD_TYPE} UPPERCASE_CMAKE_BUILD_TYPE) +if (UPPERCASE_CMAKE_BUILD_TYPE STREQUAL DEBUG) + add_compile_options(-g -O0) + add_definitions(-DDEBUG) +elseif (UPPERCASE_CMAKE_BUILD_TYPE STREQUAL RELEASE) + add_compile_options (-O3) + add_definitions(-DNDEBUG) +else () + message(FATAL_ERROR "Unknown build type. Set CMAKE_BUILD_TYPE to DEBUG or RELEASE.") +endif () + +add_subdirectory(System.IO.Native) \ No newline at end of file diff --git a/src/Native/System.IO.Native/CMakeLists.txt b/src/Native/System.IO.Native/CMakeLists.txt new file mode 100644 index 000000000000..defb98e1a4a7 --- /dev/null +++ b/src/Native/System.IO.Native/CMakeLists.txt @@ -0,0 +1,15 @@ + +project(System.IO.Native) + +set(NATIVEIO_SOURCES + nativeio.h + nativeio.cpp +) + +add_library(System.IO.Native + SHARED + ${NATIVEIO_SOURCES} +) + +install (TARGETS System.IO.Native DESTINATION .) + diff --git a/src/Native/System.IO.Native/nativeio.cpp b/src/Native/System.IO.Native/nativeio.cpp new file mode 100755 index 000000000000..34bc7ff472e0 --- /dev/null +++ b/src/Native/System.IO.Native/nativeio.cpp @@ -0,0 +1,61 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +#include "nativeio.h" +#include + +#if HAVE_STAT64 && !(defined(__APPLE__) && defined(_AMD64_)) +# define stat_ stat64 +# define fstat_ fstat64 +#else +# define stat_ stat +# define fstat_ fstat +#endif + +static void ConvertFileStats(const struct stat_& src, FileStats* dst) +{ + dst->Flags = FILESTATS_FLAGS_NONE; + dst->Mode = src.st_mode; + dst->Uid = src.st_uid; + dst->Gid = src.st_gid; + dst->Size = src.st_size; + dst->AccessTime = src.st_atime; + dst->ModificationTime = src.st_mtime; + dst->StatusChangeTime = src.st_ctime; + +#if HAVE_STAT_BIRTHTIME + dst->CreationTime = src->st_birthtime; + dst->Flags |= FILESTATS_FLAGS_HAS_CREATION_TIME; +#endif +} + +extern "C" +{ + int32_t GetFileStatsFromPath(const char* path, struct FileStats* output) + { + struct stat_ result; + int ret = stat_(path, &result); + + if (ret == 0) + { + ConvertFileStats(result, output); + } + + return ret; + } + + int32_t GetFileStatsFromDescriptor(int32_t fileDescriptor, FileStats* output) + { + struct stat_ result; + int ret = fstat_(fileDescriptor, &result); + + if (ret == 0) + { + ConvertFileStats(result, output); + } + + return ret; + } +} diff --git a/src/Native/System.IO.Native/nativeio.h b/src/Native/System.IO.Native/nativeio.h new file mode 100755 index 000000000000..524d772906b3 --- /dev/null +++ b/src/Native/System.IO.Native/nativeio.h @@ -0,0 +1,45 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +#pragma once + +#include + +extern "C" +{ + struct FileStats + { + int32_t Flags; // flags for testing if some members are present (see values below) + int32_t Mode; // protection + int32_t Uid; // user ID of owner + int32_t Gid; // group ID of owner + int64_t Size; // total size, in bytes + int64_t AccessTime; // time of last access (atime) + int64_t ModificationTime; // time of last modification (mtime) + int64_t StatusChangeTime; // time of last status change (ctime) + int64_t CreationTime; // time the file was created (birthtime) + }; + + enum + { + FILESTATS_FLAGS_NONE = 0, + FILESTATS_FLAGS_HAS_CREATION_TIME = 1, + }; + + /** + * Get file stats from a decriptor. Implemented as shim to fstat(2). + * + * Returns 0 for success, -1 for failure. Sets errno on failure. + */ + int32_t GetFileStatsFromDescriptor(int32_t fileDescriptor, FileStats* output); + + /** + * Get file stats from a full path. Implemented as shim to stat(2). + * + * Returns 0 for success, -1 for failure. Sets errno on failure. + */ + int32_t GetFileStatsFromPath(const char* path, FileStats* output); +} + From dfe7b802e9ab2e171cc5bb9419ff7bf37dce3b0e Mon Sep 17 00:00:00 2001 From: Nick Guerrera Date: Tue, 21 Jul 2015 19:42:41 -0700 Subject: [PATCH 5/8] Add some guidelines around unix shim design to docs --- .../coding-guidelines/interop-guidelines.md | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/Documentation/coding-guidelines/interop-guidelines.md b/Documentation/coding-guidelines/interop-guidelines.md index 64250da4186b..8f387b8bbfa7 100644 --- a/Documentation/coding-guidelines/interop-guidelines.md +++ b/Documentation/coding-guidelines/interop-guidelines.md @@ -18,6 +18,10 @@ We have the following goals related to interop code being used in CoreFX: - This is both for good hygiene and to help keep platform-specific code separated from platform-neutral code, which is important for maximizing reusable code above PAL layers. +- Ensure maximal managed code reuse across different OS flavors which have + the same API but not the same ABI. + - This is the case for UNIX and addressing it is a work-in-progress (see issue + #2137 and section on "shims" below.) ## Approach @@ -244,3 +248,43 @@ rather than the underlying integral type. should be named with the most discoverable name possible; if that name is a concept (e.g. Errors), it can be named using managed naming guidelines. + + +## UNIX shims + +Often, various UNIX flavors offer the same API from the point-of-view of compatibility +with C/C++ source code, but they do not have the same ABI. e.g. Fields can be laid out +differently, constants can have different numeric values, etc. Even the exports can +be named differently. + +This leaves us with a situation where we can't write portable P/Invoke declarations +that will work on all flavors, and writing separate declarations per flavor is quite +fragile and won't scale. + +To address this, we're moving to a model where all UNIX interop from corefx starts with +a P/Invoke to a C++ lib written specifically for corefx. These libs -- System.*.Native.so +(aka "shims") -- are intended to be very thin layers over underlying platform libraries. +Generally, they are not there to add any significant abstraction, but to create a +stable ABI such that the same IL assembly can work across UNIX flavors. + +At this time, these shims are compiled in the dotnet/coreclr repository under the corefx +folder. This is temporary (issue #2301) until we add necessary infrastructure to build them +in this repository. + +Guidelines for shim C++ API: + +- Keep them as "thin"/1:1 as possible. + - We want to write the majority of code in C#. +- Never skip the shim and P/Invoke directly to the underlying platform API. It's +easy to assume something is safe/guaranteed when it isn't. +- Don't cheat and take advantage of coincidental agreement between +one flavor's ABI and the shim's ABI. +- Use PascalCase and spell things out in a style closer to Win32 than libc. + - At first, it seemed that we'd want to use 1:1 names for the shims, but it + turns out there are many cases where being strictly 1:1 isn't practical. As such, + the libraries will end up looking more self-consistent if we give them their + own style with which to express themselves. +- Stick to data types which are guaranteed not to vary in size across flavors. (Pointers +and size_t variance across bitness is OK.) + - e.g. use int32_t, int64_t from stdint.h and not int, long. + From 4ae994a569fcf351808f6ef25301becf6bf4f09a Mon Sep 17 00:00:00 2001 From: Nick Guerrera Date: Tue, 21 Jul 2015 20:03:57 -0700 Subject: [PATCH 6/8] Move Console usage of stat wrapper to new IO shim --- .../src/Interop/Unix/Interop.Libraries.cs | 1 + .../Unix/System.IO.Native/Interop.NativeIO.cs | 47 +++++++++++++++++++ src/System.Console/src/System.Console.csproj | 3 ++ .../src/System/ConsolePal.Unix.cs | 10 ++-- 4 files changed, 56 insertions(+), 5 deletions(-) create mode 100644 src/Common/src/Interop/Unix/System.IO.Native/Interop.NativeIO.cs diff --git a/src/Common/src/Interop/Unix/Interop.Libraries.cs b/src/Common/src/Interop/Unix/Interop.Libraries.cs index 8f6fb30dc6c9..b8464795ba4c 100644 --- a/src/Common/src/Interop/Unix/Interop.Libraries.cs +++ b/src/Common/src/Interop/Unix/Interop.Libraries.cs @@ -9,6 +9,7 @@ private static partial class Libraries internal const string LibCoreClr= "libcoreclr"; // CoreCLR runtime internal const string LibCrypto = "libcrypto"; // OpenSSL crypto library internal const string Zlib = "libz"; // zlib compression library + internal const string IOInterop = "System.IO.Native"; internal const string CryptoInterop = "System.Security.Cryptography.Native"; } } diff --git a/src/Common/src/Interop/Unix/System.IO.Native/Interop.NativeIO.cs b/src/Common/src/Interop/Unix/System.IO.Native/Interop.NativeIO.cs new file mode 100644 index 000000000000..25818b17d2c4 --- /dev/null +++ b/src/Common/src/Interop/Unix/System.IO.Native/Interop.NativeIO.cs @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Runtime.InteropServices; + +internal static partial class Interop +{ + internal static partial class NativeIO + { + internal struct FileStats + { + private FileStatsFlags Flags; + internal int Mode; + internal int Uid; + internal int Gid; + internal int Size; + internal int AccessTime; + internal int ModificationTime; + internal int StatusChangeTime; + internal int CreationTime; + } + + internal static class FileTypes + { + internal const int S_IFMT = 0xF000; + internal const int S_IFIFO = 0x1000; + internal const int S_IFCHR = 0x2000; + internal const int S_IFDIR = 0x4000; + internal const int S_IFREG = 0x8000; + internal const int S_IFLNK = 0xA000; + } + + [Flags] + internal enum FileStatsFlags + { + None = 0, + HasCreationTime = 1, + } + + [DllImport(Libraries.IOInterop, SetLastError = true)] + internal static extern int GetFileStatsFromDescriptor(int fileDescriptor, out FileStats output); + + [DllImport(Libraries.IOInterop, SetLastError = true)] + internal static extern int GetFileStatsFromPath(string path, out FileStats output); + } +} diff --git a/src/System.Console/src/System.Console.csproj b/src/System.Console/src/System.Console.csproj index 813b8d393add..dc9c29008c6d 100644 --- a/src/System.Console/src/System.Console.csproj +++ b/src/System.Console/src/System.Console.csproj @@ -138,6 +138,9 @@ Common\Interop\Unix\Interop.GetFileInformation.cs" + + Common\Interop\Unix\Interop.NativeIO.cs" + Common\Interop\Unix\Interop.SetConsoleCtrlHandler.cs" diff --git a/src/System.Console/src/System/ConsolePal.Unix.cs b/src/System.Console/src/System/ConsolePal.Unix.cs index afad581dec29..847b055aaf7c 100644 --- a/src/System.Console/src/System/ConsolePal.Unix.cs +++ b/src/System.Console/src/System/ConsolePal.Unix.cs @@ -91,7 +91,7 @@ private static bool ConsoleOutIsTerminal UnixConsoleStream ucs = sw.BaseStream as UnixConsoleStream; if (ucs != null) { - return ucs._handleType == Interop.libcoreclr.FileTypes.S_IFCHR; + return ucs._handleType == Interop.NativeIO.FileTypes.S_IFCHR; } } } @@ -378,11 +378,11 @@ internal UnixConsoleStream(string devPath, FileAccess access) try { _handle.DangerousAddRef(ref gotFd); - Interop.libcoreclr.fileinfo buf; + Interop.NativeIO.FileStats buf; _handleType = - Interop.libcoreclr.GetFileInformationFromFd((int)_handle.DangerousGetHandle(), out buf) == 0 ? - (buf.mode & Interop.libcoreclr.FileTypes.S_IFMT) : - Interop.libcoreclr.FileTypes.S_IFREG; // if something goes wrong, don't fail, just say it's a regular file + Interop.NativeIO.GetFileStatsFromDescriptor((int)_handle.DangerousGetHandle(), out buf) == 0 ? + (buf.Mode & Interop.NativeIO.FileTypes.S_IFMT) : + Interop.NativeIO.FileTypes.S_IFREG; // if something goes wrong, don't fail, just say it's a regular file } finally { From e35129f7dc1cd7c74bad286729384ae91de2ea27 Mon Sep 17 00:00:00 2001 From: Nick Guerrera Date: Tue, 21 Jul 2015 20:38:02 -0700 Subject: [PATCH 7/8] Build and deploy native corefx components for test run --- run-test.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/run-test.sh b/run-test.sh index edc7f919ab00..3c3524985a29 100755 --- a/run-test.sh +++ b/run-test.sh @@ -128,6 +128,12 @@ create_test_overlay() exit 1 fi find $CoreFxBins -name '*.dll' -exec cp '{}' "$OverlayDir" ";" + + # Then the native linux CoreFX binaries + # + # TODO: Currently, CI does not build the native CoreFX components so build them here + # in the test phase for now. + ( $ProjectRoot/src/Native/build.sh && cp $ProjectRoot/bin/$OS.x64.$Configuration/Native/*.so $OverlayDir ) || exit 1 } copy_test_overlay() From 52f57e8e6c6aed656d40fb9b638081cff10c6e10 Mon Sep 17 00:00:00 2001 From: Nick Guerrera Date: Wed, 22 Jul 2015 10:51:32 -0700 Subject: [PATCH 8/8] Respond to PR feedback * Rename shims to match their underlying platform API names * Update guidelines * Add errno conversion TODO * Account for OS X in run-test.sh --- .../coding-guidelines/interop-guidelines.md | 28 +++++++++---------- run-test.sh | 4 +-- .../Unix/System.IO.Native/Interop.NativeIO.cs | 4 +-- src/Native/System.IO.Native/nativeio.cpp | 8 +++--- src/Native/System.IO.Native/nativeio.h | 4 +-- .../src/System/ConsolePal.Unix.cs | 2 +- 6 files changed, 25 insertions(+), 25 deletions(-) diff --git a/Documentation/coding-guidelines/interop-guidelines.md b/Documentation/coding-guidelines/interop-guidelines.md index 8f387b8bbfa7..80ffe1ead09b 100644 --- a/Documentation/coding-guidelines/interop-guidelines.md +++ b/Documentation/coding-guidelines/interop-guidelines.md @@ -254,8 +254,10 @@ rather than the underlying integral type. Often, various UNIX flavors offer the same API from the point-of-view of compatibility with C/C++ source code, but they do not have the same ABI. e.g. Fields can be laid out -differently, constants can have different numeric values, etc. Even the exports can -be named differently. +differently, constants cn have different numeric values, exports can +be named differently, etc. There are not only differences between operating systems +(Mac OS X vs. Ubuntu vs. FreeBSD), but also differences related to the underlying +processor architecture (x64 vs. x86 vs. ARM). This leaves us with a situation where we can't write portable P/Invoke declarations that will work on all flavors, and writing separate declarations per flavor is quite @@ -267,10 +269,6 @@ a P/Invoke to a C++ lib written specifically for corefx. These libs -- System.*. Generally, they are not there to add any significant abstraction, but to create a stable ABI such that the same IL assembly can work across UNIX flavors. -At this time, these shims are compiled in the dotnet/coreclr repository under the corefx -folder. This is temporary (issue #2301) until we add necessary infrastructure to build them -in this repository. - Guidelines for shim C++ API: - Keep them as "thin"/1:1 as possible. @@ -279,12 +277,14 @@ Guidelines for shim C++ API: easy to assume something is safe/guaranteed when it isn't. - Don't cheat and take advantage of coincidental agreement between one flavor's ABI and the shim's ABI. -- Use PascalCase and spell things out in a style closer to Win32 than libc. - - At first, it seemed that we'd want to use 1:1 names for the shims, but it - turns out there are many cases where being strictly 1:1 isn't practical. As such, - the libraries will end up looking more self-consistent if we give them their - own style with which to express themselves. -- Stick to data types which are guaranteed not to vary in size across flavors. (Pointers -and size_t variance across bitness is OK.) - - e.g. use int32_t, int64_t from stdint.h and not int, long. +- Use PascalCase in a style closer to Win32 than libc. + - If an export point has a 1:1 correspondence to the platform API, then name + it after the platform API in PascalCase (e.g. stat -> Stat, fstat -> FStat). + - If an export is not 1:1, then spell things out as we typically would in + CoreFX code (i.e. don't use abbreviations unless they come from the underlying + API. + - At first, it seemed that we'd want to use 1:1 names throughout, but it + turns out there are many cases where being strictly 1:1 isn't practical. +- Stick to data types which are guaranteed not to vary in size across flavors. + - e.g. use int32_t, int64_t from stdint.h and not int, long. diff --git a/run-test.sh b/run-test.sh index 3c3524985a29..0410f48db807 100755 --- a/run-test.sh +++ b/run-test.sh @@ -129,11 +129,11 @@ create_test_overlay() fi find $CoreFxBins -name '*.dll' -exec cp '{}' "$OverlayDir" ";" - # Then the native linux CoreFX binaries + # Then the native CoreFX binaries # # TODO: Currently, CI does not build the native CoreFX components so build them here # in the test phase for now. - ( $ProjectRoot/src/Native/build.sh && cp $ProjectRoot/bin/$OS.x64.$Configuration/Native/*.so $OverlayDir ) || exit 1 + ( $ProjectRoot/src/Native/build.sh && cp $ProjectRoot/bin/$OS.x64.$Configuration/Native/* $OverlayDir ) || exit 1 } copy_test_overlay() diff --git a/src/Common/src/Interop/Unix/System.IO.Native/Interop.NativeIO.cs b/src/Common/src/Interop/Unix/System.IO.Native/Interop.NativeIO.cs index 25818b17d2c4..903e18809f50 100644 --- a/src/Common/src/Interop/Unix/System.IO.Native/Interop.NativeIO.cs +++ b/src/Common/src/Interop/Unix/System.IO.Native/Interop.NativeIO.cs @@ -39,9 +39,9 @@ internal enum FileStatsFlags } [DllImport(Libraries.IOInterop, SetLastError = true)] - internal static extern int GetFileStatsFromDescriptor(int fileDescriptor, out FileStats output); + internal static extern int FStat(int fileDescriptor, out FileStats output); [DllImport(Libraries.IOInterop, SetLastError = true)] - internal static extern int GetFileStatsFromPath(string path, out FileStats output); + internal static extern int Stat(string path, out FileStats output); } } diff --git a/src/Native/System.IO.Native/nativeio.cpp b/src/Native/System.IO.Native/nativeio.cpp index 34bc7ff472e0..b7a66a39332a 100755 --- a/src/Native/System.IO.Native/nativeio.cpp +++ b/src/Native/System.IO.Native/nativeio.cpp @@ -33,7 +33,7 @@ static void ConvertFileStats(const struct stat_& src, FileStats* dst) extern "C" { - int32_t GetFileStatsFromPath(const char* path, struct FileStats* output) + int32_t Stat(const char* path, struct FileStats* output) { struct stat_ result; int ret = stat_(path, &result); @@ -43,10 +43,10 @@ extern "C" ConvertFileStats(result, output); } - return ret; + return ret; // TODO: errno conversion } - int32_t GetFileStatsFromDescriptor(int32_t fileDescriptor, FileStats* output) + int32_t FStat(int32_t fileDescriptor, FileStats* output) { struct stat_ result; int ret = fstat_(fileDescriptor, &result); @@ -56,6 +56,6 @@ extern "C" ConvertFileStats(result, output); } - return ret; + return ret; // TODO: errno conversion } } diff --git a/src/Native/System.IO.Native/nativeio.h b/src/Native/System.IO.Native/nativeio.h index 524d772906b3..b33466ca0057 100755 --- a/src/Native/System.IO.Native/nativeio.h +++ b/src/Native/System.IO.Native/nativeio.h @@ -33,13 +33,13 @@ extern "C" * * Returns 0 for success, -1 for failure. Sets errno on failure. */ - int32_t GetFileStatsFromDescriptor(int32_t fileDescriptor, FileStats* output); + int32_t FStat(int32_t fileDescriptor, FileStats* output); /** * Get file stats from a full path. Implemented as shim to stat(2). * * Returns 0 for success, -1 for failure. Sets errno on failure. */ - int32_t GetFileStatsFromPath(const char* path, FileStats* output); + int32_t Stat(const char* path, FileStats* output); } diff --git a/src/System.Console/src/System/ConsolePal.Unix.cs b/src/System.Console/src/System/ConsolePal.Unix.cs index 847b055aaf7c..d3c1251c8e6b 100644 --- a/src/System.Console/src/System/ConsolePal.Unix.cs +++ b/src/System.Console/src/System/ConsolePal.Unix.cs @@ -380,7 +380,7 @@ internal UnixConsoleStream(string devPath, FileAccess access) _handle.DangerousAddRef(ref gotFd); Interop.NativeIO.FileStats buf; _handleType = - Interop.NativeIO.GetFileStatsFromDescriptor((int)_handle.DangerousGetHandle(), out buf) == 0 ? + Interop.NativeIO.FStat((int)_handle.DangerousGetHandle(), out buf) == 0 ? (buf.Mode & Interop.NativeIO.FileTypes.S_IFMT) : Interop.NativeIO.FileTypes.S_IFREG; // if something goes wrong, don't fail, just say it's a regular file }