diff --git a/Dockerbuild.md b/Dockerbuild.md index ba8234f..f72303c 100644 --- a/Dockerbuild.md +++ b/Dockerbuild.md @@ -5,13 +5,6 @@ Quite often we need to build quite a bunch of applications to get the very latest tools and environment. Instead of hand holding every developer to get the latest environment, let's use Docker to bootstrap a common build environment. -Proxy setup -=========== - -If you are working in an environment where an http proxy is necessary, update -the files in `proxy-configuration` as required. With out this, you cannot -install the various packages needed to build the Docker image. - Versions of packages ==================== diff --git a/Dockerfile b/Dockerfile index 22bfc30..50b532f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,145 +3,98 @@ FROM $BASE_DISTRO ARG INSTALL_GCC=0 -# In case of Proxy based environment, leave the following enabled. -# in Direct internet cases, comment out the following two lines. -#--- PROXY SETUP START -# COPY proxy-configuration/ / -# RUN export DEBIAN_FRONTEND=noninteractive;apt-get update;apt-get install -y apt-transport-https socket corkscrew apt-utils -#--- END START - ARG DEBIAN_FRONTEND noninteractive -RUN apt-get update && \ - apt-get install -y \ - build-essential \ - wget \ - gcc \ - ccache \ - ncurses-dev \ - xz-utils \ - libssl-dev \ - bc \ - flex \ - libelf-dev \ - bison \ - libyaml-dev \ - python3-pip \ - libcurl4-gnutls-dev \ - libexpat1-dev \ - gettext \ - libz-dev \ - libssl-dev \ - flex \ - bison \ - pkg-config \ - sqlite3 \ - libsqlite3-dev \ - llvm \ - autoconf \ - pkg-config \ - ocaml-nox \ - ocaml-findlib \ - menhir \ - libmenhir-ocaml-dev \ - ocaml-native-compilers \ - libpcre-ocaml-dev \ - libparmap-ocaml-dev \ - libpython3.11 \ - libpython3.11-dev \ - libgmp-dev \ - libmpc-dev \ - diffstat \ - yamllint \ - swig \ - python3 \ - python3-ruamel.yaml \ - aria2 - -RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 - -COPY other-configs/ / - -COPY build-env.sh /tmp -RUN INSTALL_GCC=$INSTALL_GCC /tmp/build-env.sh - -FROM $BASE_DISTRO - -ARG INSTALL_GCC=0 +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + aria2 \ + autoconf \ + automake \ + bc \ + bison \ + build-essential \ + ca-certificates \ + ccache \ + diffstat \ + dumb-init \ + flex \ + gcc \ + gettext \ + gnupg2 \ + gosu \ + libcurl4-gnutls-dev \ + libelf-dev \ + libexpat1-dev \ + libgmp-dev \ + libmenhir-ocaml-dev \ + libmpc-dev \ + libparmap-ocaml-dev \ + libpcre-ocaml-dev \ + libpython3.11 \ + libpython3.11-dev \ + libsqlite3-dev \ + libssl-dev \ + libyaml-dev \ + libz-dev \ + menhir \ + ncurses-dev \ + ocaml-findlib \ + ocaml-native-compilers \ + ocaml-nox \ + pkg-config \ + python-is-python3 \ + python3 \ + python3-dev \ + python3-pip \ + python3-ruamel.yaml \ + sqlite3 \ + swig \ + wget \ + xz-utils \ + yamllint \ + && echo "**** cleanup ****" \ + && apt-get autoremove \ + && apt-get clean \ + && rm -rf \ + /tmp/* \ + /var/lib/apt/lists/* \ + /var/tmp/* \ + /var/log/* + +RUN --mount=type=bind,source=build-env.sh,target=/tmp/build-env.sh \ + INSTALL_GCC=$INSTALL_GCC /tmp/build-env.sh # Publish the source repository LABEL org.opencontainers.image.source https://github.com/nmenon/kernel_patch_verify -#--- PROXY SETUP START -# COPY proxy-configuration/ / -# RUN export DEBIAN_FRONTEND=noninteractive;apt-get update;apt-get install -y apt-transport-https socket corkscrew apt-utils -#--- END START - - -ARG DEBIAN_FRONTEND noninteractive -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - ca-certificates \ - gnupg2 \ - build-essential \ - wget \ - gcc \ - ccache \ - ncurses-dev \ - xz-utils \ - libssl-dev \ - bc \ - flex \ - libelf-dev \ - bison \ - libyaml-dev \ - python3-pip \ - libcurl4-gnutls-dev \ - libexpat1-dev \ - gettext \ - libz-dev \ - libssl-dev \ - flex \ - bison \ - pkg-config \ - sqlite3 \ - libsqlite3-dev \ - autoconf \ - pkg-config \ - ocaml-nox \ - ocaml-findlib \ - menhir \ - libmenhir-ocaml-dev \ - ocaml-native-compilers \ - libpcre-ocaml-dev \ - libparmap-ocaml-dev \ - libpython3.11 \ - libpython3.11-dev \ - libgmp-dev \ - libmpc-dev \ - diffstat \ - yamllint \ - swig \ - python3 \ - python3-dev \ - python3-ruamel.yaml - # Add our llvm repo configs COPY llvm-config / -RUN apt-get update && apt-get install -y --no-install-recommends\ - llvm \ - clang \ - lld +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + llvm \ + clang \ + lld \ + && echo "**** cleanup ****" \ + && apt-get autoremove \ + && apt-get clean \ + && rm -rf \ + /tmp/* \ + /var/lib/apt/lists/* \ + /var/tmp/* \ + /var/log/* -RUN apt-get clean && \ - rm -rf /var/lib/apt/lists/* +COPY other-configs/ / -RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 +RUN echo "**** create developer user and make our folders ****" \ + && useradd -u 1000 -U -d /config -s /bin/false developer \ + && usermod -G users developer \ + && mkdir /workdir && chown developer:developer /workdir \ + && mkdir /config && chown developer:developer /config -COPY other-configs/ / +ENTRYPOINT ["/init"] -COPY --from=0 /usr/local/ /usr/local/ +CMD ["/usr/bin/bash"] -RUN ldconfig /usr/local/lib +VOLUME /workdir COPY kernel_patch_verify /usr/bin/kernel_patch_verify diff --git a/build-env.sh b/build-env.sh index 5644723..171b629 100755 --- a/build-env.sh +++ b/build-env.sh @@ -18,7 +18,7 @@ export COCCI_TAG=1.1.1 # https://github.com/devicetree-org/dt-schema/tags export DTSCHEMA_REV=v2024.02 -ARIA_OPTS=(--timeout=180 --retry-wait=10 -m 0 -x 10 -j 10) +ARIA_OPTS=( --summary-interval=5 --timeout=180 --retry-wait=10 -m 0 -x 10 -j 10 ) download_build_install_git() { @@ -47,12 +47,13 @@ download_build_install_python_deps() # scripts/spdxcheck.py dependencies python -m pip install --upgrade --break-system-packages ply gitpython yamllint rfc3987 pylibfdt python -m pip install --break-system-packages git+https://github.com/devicetree-org/dt-schema.git@$DTSCHEMA_REV + rm -rf "/tmp/.cache/" } clone_and_cd() { cd /tmp && - git clone --depth=1 --branch "$1" "$2" "$3" && + git clone --progress --depth=1 --branch "$1" "$2" "$3" && cd /tmp/"$3" return $? } @@ -118,7 +119,7 @@ download_and_install_armgcc_64() cd /tmp mkdir -p /opt/cross-gcc-linux-9/ aria2c "${ARIA_OPTS[@]}" -o "$FILE" "$URL" - tar -C /usr/local/cross-gcc-linux-9/ --strip-components=1 -xf "$FILE" + tar -C /opt/cross-gcc-linux-9/ --strip-components=1 -xf "$FILE" rm -f /tmp/"$FILE" } @@ -131,7 +132,7 @@ download_and_install_armgcc_32() cd /tmp mkdir -p /opt/cross-gcc-linux-9/ aria2c "${ARIA_OPTS[@]}" -o "$FILE" "$URL" - tar -C /usr/local/cross-gcc-linux-9/ --strip-components=1 -xf "$FILE" + tar -C /opt/cross-gcc-linux-9/ --strip-components=1 -xf "$FILE" rm -f /tmp/"$FILE" } diff --git a/kernel_patch_verify b/kernel_patch_verify index 906c223..d8bf923 100755 --- a/kernel_patch_verify +++ b/kernel_patch_verify @@ -26,6 +26,11 @@ source /etc/profile shopt -s expand_aliases +# extend the path with the supplied extra directories +if [ -n "$KP_PATH" ]; then + export PATH=${KP_PATH}:${PATH} +fi + ccache=$(which ccache) # We would rather that we hit cache more often, than rebuild.. diff --git a/kp_common b/kp_common index 00a10ff..701d464 100644 --- a/kp_common +++ b/kp_common @@ -1,6 +1,6 @@ #!/bin/bash -export USER_ID GROUP_ID PATH DOCKER_MOUNT_DIRS IMAGE_ID +export DOCKER_ARGS IMG_NAME KP_PATH # Check if Docker image exists @@ -38,30 +38,46 @@ else fi fi -USER_ID=$(id -u) -GROUP_ID=$(id -g) - -DOCKER_MOUNT_DIRS=() -DOCKER_MOUNT_DIRS+=(-v /tmp:/tmp) -DOCKER_MOUNT_DIRS+=(-v /opt:/opt) -DOCKER_MOUNT_DIRS+=(-v "$CCACHEDIR":/ccache) -DOCKER_MOUNT_DIRS+=(-v "$(pwd)":/workdir) +DOCKER_ARGS=() +DOCKER_ARGS+=(-v /tmp:/tmp) +DOCKER_ARGS+=(-v /opt:/opt) +DOCKER_ARGS+=(-v "$CCACHEDIR":/ccache) # Check if current directory is a git directory -if ! GIT_WORKTREE_COMMONDIR=$(git rev-parse --git-common-dir); then +if ! git rev-parse --is-inside-work-tree > /dev/null; then exit 1 fi -# Mount parent directory if its a worktree -if [ "$GIT_WORKTREE_COMMONDIR" != ".git" ]; then - DOCKER_MOUNT_DIRS+=(-v "$GIT_WORKTREE_COMMONDIR":"$GIT_WORKTREE_COMMONDIR") +ROOT_GIT=$(git rev-parse --show-toplevel) +# if a submodule or worktree then we need to fetch and preserve the gitdir path +if [ -f "${ROOT_GIT}/.git" ]; then + gitdir_str=$(grep -P -o '(?<=gitdir:\s).*' "${ROOT_GIT}/.git") + relative_str=$(echo "$gitdir_str" | grep -P -o '^(../)*') + if [ -n "$relative_str" ]; then + # relative paths need to be preserved + common_path=$(realpath "$ROOT_GIT/$relative_str") + relative_pwd=$(realpath --relative-to="$common_path" "$PWD") + DOCKER_ARGS+=(-v "$common_path":"/workdir") + DOCKER_ARGS+=(-w "/workdir/${relative_pwd}") + else + # absolute paths can be passed through + gitdir_path=$(git rev-parse --git-common-dir) + DOCKER_ARGS+=(-v "$ROOT_GIT":"/workdir") + DOCKER_ARGS+=(-v "$gitdir_path":"$gitdir_path") + fi +else + # normal git directory, just toss everything in + DOCKER_ARGS+=(-v "$ROOT_GIT":"/workdir") fi -# Run our image to add our swuser -docker run "$IMG_NAME" /bin/bash -c "groupadd -r swuser -g $GROUP_ID && useradd -u $USER_ID -r -g swuser -d /workdir -s /sbin/nologin -c \"Docker kernel patch user\" swuser" -# Get the container ID of the last run container (above) -CONTAINER_ID=$(docker ps -lq) -# Commit the container state (returns an image_id with sha256: prefix cut off) -IMAGE_ID=$(docker commit "$CONTAINER_ID" | cut -c8-) +# list of paths to append to the PATH variable in the container +extra_paths=( + /workdir/scripts/dtc + /opt/cross-gcc-linux-13/bin + /opt/cross-gcc-linux-12/bin + /opt/cross-gcc-linux-11/bin + /opt/cross-gcc-linux-10/bin + /opt/cross-gcc-linux-9/bin +) -PATH=/workdir/scripts/dtc:/usr/local/bin:/usr/bin:/bin:/usr/local/games:/opt/cross-gcc-linux-13/bin:/opt/cross-gcc-linux-12/bin:/opt/cross-gcc-linux-11/bin:/opt/cross-gcc-linux-10/bin:/opt/cross-gcc-linux-9/bin:/usr/local/cross-gcc-linux-9/bin:/usr/local/cross-gcc-linux-10/bin +KP_PATH=$(IFS=:; printf '%s' "${extra_paths[*]}") diff --git a/kps b/kps index 6a7a07b..b5adaab 100755 --- a/kps +++ b/kps @@ -4,11 +4,7 @@ source "$(dirname "$(readlink -f "$0")")/kp_common" # If we wanted to get to bash shell: docker run --rm -ti \ - --user "$USER_ID":"$GROUP_ID" \ - -e PATH \ - "${DOCKER_MOUNT_DIRS[@]}" \ - "$IMAGE_ID" \ + -e KP_PATH -e CROSS_COMPILE -e ARCH \ + "${DOCKER_ARGS[@]}" \ + "$IMG_NAME" \ bash --init-file /etc/profile - -# TODO: we can reuse this image for future runs, for now just clean up after ourselves -docker rmi "$IMAGE_ID" diff --git a/kpv b/kpv index 2de6290..7d936cc 100755 --- a/kpv +++ b/kpv @@ -3,11 +3,7 @@ source "$(dirname "$(readlink -f "$0")")/kp_common" docker run --rm -ti \ - --user "$USER_ID":"$GROUP_ID" \ - -e PATH \ - "${DOCKER_MOUNT_DIRS[@]}" \ - "$IMAGE_ID" \ + -e KP_PATH -e CROSS_COMPILE -e ARCH \ + "${DOCKER_ARGS[@]}" \ + "$IMG_NAME" \ kernel_patch_verify -S /usr/local/smatch/bin/k_sm_check_script "$@" - -# TODO: we can reuse this image for future runs, for now just clean up after ourselves -docker rmi "$IMAGE_ID" diff --git a/other-configs/init b/other-configs/init new file mode 100755 index 0000000..de2c99a --- /dev/null +++ b/other-configs/init @@ -0,0 +1,43 @@ +#!/bin/bash + +get_attribs() { + local file_stats file_to_test useful_attribs + if file_to_test=$(realpath "$1") && [[ $2 =~ ^[0-9]+$ ]] ; then + useful_attribs=$(stat "$file_to_test" -t) + read -r -a file_stats <<< "${useful_attribs#"$file_to_test"}" + echo "${file_stats["$2"]}" + else + return 1 + fi +} + +get_build_uid() { + get_attribs /workdir 3 +} + +get_build_gid() { + get_attribs /workdir 4 +} + +if NEW_GID=$(get_build_gid) && NEW_UID=$(get_build_uid); then + # bypass everything if podman is remapping the id to root + if [ "${NEW_UID}" == "0" ]; then + if [ "$(id -u)" == "0" ]; then + exec dumb-init -- "$@" + else + echo "Unable to resolve ns mapping!" + fi + fi + + # change the uid and gid of developer otherwise + [ "$NEW_GID" != "$(id -g developer)" ] && groupmod -g "${NEW_GID}" developer + [ "$NEW_UID" != "$(id -u developer)" ] && usermod -u "${NEW_UID}" developer +else + echo "Not able to detect UID/GID for remapping!" +fi + +if [ "$(id -u)" == "$(id -u developer)" ]; then + exec dumb-init -- "$@" +else + exec dumb-init -- gosu developer "$@" +fi diff --git a/proxy-configuration/etc/apt/apt.conf.d/99-corporate-proxy b/proxy-configuration/etc/apt/apt.conf.d/99-corporate-proxy deleted file mode 100644 index ed6a852..0000000 --- a/proxy-configuration/etc/apt/apt.conf.d/99-corporate-proxy +++ /dev/null @@ -1,9 +0,0 @@ -// Use when Providing Corporate proxy environment -Acquire -{ - http { - Proxy "http://webproxy.ext.ti.com:80"; - Proxy::artifactory.itg.ti.com "DIRECT"; - } -} - diff --git a/proxy-configuration/etc/profile.d/99-corporate-env.sh b/proxy-configuration/etc/profile.d/99-corporate-env.sh deleted file mode 100755 index 1203128..0000000 --- a/proxy-configuration/etc/profile.d/99-corporate-env.sh +++ /dev/null @@ -1,6 +0,0 @@ - -export http_proxy=http://webproxy.ext.ti.com:80 -export https_proxy=http://webproxy.ext.ti.com:80 -export ftp_proxy=http://webproxy.ext.ti.com:80 -export no_proxy=.ti.com -export GIT_PROXY_COMMAND=/usr/bin/git-tunnel.sh diff --git a/proxy-configuration/usr/bin/git-tunnel.sh b/proxy-configuration/usr/bin/git-tunnel.sh deleted file mode 100755 index 6ea0c3b..0000000 --- a/proxy-configuration/usr/bin/git-tunnel.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash -# Export GIT_PROXY_COMMAND TO THIS -# usable with settings: -# no_proxy="$no_proxy,ti.com" -# http_proxy="http://wwwgate.ti.com:80/" -# OR -# http_proxy="http://wwwgate.ti.com:80" -# OR -# http_proxy=http://proxyconfig.itg.ti.com/proxy.pac - -# Apps required -WGET=/usr/bin/wget -CORKSCREW=/usr/bin/corkscrew -SOCKET=/usr/bin/socket -# Special note on pacparse -# http://code.google.com/p/pacparser/ -# I hand statically built this for ubuntu 9.10 -# but you can clone and install this original install -# will be pactester - just rename the following to use -PACPARSE=pacparse - -use_proxy=1 - -# Set this up if you are going to use this seamlessly -# over vpn -use_at_home=0 - -if [ $use_at_home -eq 1 ]; then - k=`ifconfig|grep tun0` - if [ -n "$k" ]; then - use_proxy=1 - else - use_proxy=0 - fi -fi - -while [ -n "$no_proxy" ]; do - no_proxy_entry=${no_proxy%%,*} - if [ "$1" != "${1%$no_proxy_entry}" ]; then - #echo "no_proxy match: $1 is in $no_proxy_entry" - use_proxy=0 - break; - fi - if [ "$no_proxy_entry" = "$no_proxy" ]; then - break - fi - no_proxy=${no_proxy#*,} -done - -if [ $use_proxy = 0 ]; then - (cat | $SOCKET $1 $2) -else - if [ -z "$GIT_PROXY_HOST" -o -z "$GIT_PROXY_PORT" ]; then - if [ -z "$http_proxy" ]; then - echo "FAILED ($1 $2)!! no proxy options? yet need proxy?" 1>&2 - fi - pac=`echo "$http_proxy"|grep "pac$"` - if [ -n "$pac" ]; then - #parse pac file - LOC="http://$1" - $WGET -O /tmp/proxy.pac "$http_proxy" - http_proxy=`$PACPARSE -p /tmp/proxy.pac -u $LOC|cut -d ' ' -f2` - else - http_proxy=`echo $http_proxy|tr -d '/'|sed -e "s/http://g"` - fi - #parse normal http_proxy config - export GIT_PROXY_HOST=`echo "$http_proxy"|cut -d ':' -f1` - export GIT_PROXY_PORT=`echo "$http_proxy"|cut -d ':' -f2` - fi - exec $CORKSCREW $GIT_PROXY_HOST $GIT_PROXY_PORT "$@" -fi - -