From b9b6975bfa8d57dc5c332587f5e46aec0ea9fb74 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 11 Feb 2021 17:18:43 -0800 Subject: [PATCH] Brand New Build! Features Linux x86_64 docker builds for all archs --- .compile_binaries | 73 ------------------- .github/workflows/build.yml | 124 ++++++++++++++++++++++++++++++++ .github_deploy | 35 +-------- .multi_arch_docker | 9 +-- .prepare_deploy | 36 ++++++---- .travis.yml | 97 ------------------------- Dockerfile | 29 -------- build/README.md | 13 ++++ build/build_builder | 12 ++++ build/darwin.x86_64/Dockerfile | 31 ++++++++ build/darwin.x86_64/build | 14 ++++ build/darwin.x86_64/tag | 1 + build/linux.aarch64/Dockerfile | 30 ++++++++ build/linux.aarch64/build | 15 ++++ build/linux.aarch64/tag | 1 + build/linux.armv6hf/Dockerfile | 59 +++++++++++++++ build/linux.armv6hf/build | 16 +++++ build/linux.armv6hf/tag | 1 + build/linux.x86_64/Dockerfile | 26 +++++++ build/linux.x86_64/build | 15 ++++ build/linux.x86_64/tag | 1 + build/run_builder | 30 ++++++++ build/windows.x86_64/Dockerfile | 27 +++++++ build/windows.x86_64/build | 19 +++++ build/windows.x86_64/tag | 1 + 25 files changed, 461 insertions(+), 254 deletions(-) delete mode 100755 .compile_binaries create mode 100644 .github/workflows/build.yml delete mode 100644 .travis.yml delete mode 100644 Dockerfile create mode 100644 build/README.md create mode 100755 build/build_builder create mode 100644 build/darwin.x86_64/Dockerfile create mode 100755 build/darwin.x86_64/build create mode 100644 build/darwin.x86_64/tag create mode 100644 build/linux.aarch64/Dockerfile create mode 100755 build/linux.aarch64/build create mode 100644 build/linux.aarch64/tag create mode 100644 build/linux.armv6hf/Dockerfile create mode 100755 build/linux.armv6hf/build create mode 100644 build/linux.armv6hf/tag create mode 100644 build/linux.x86_64/Dockerfile create mode 100755 build/linux.x86_64/build create mode 100644 build/linux.x86_64/tag create mode 100755 build/run_builder create mode 100644 build/windows.x86_64/Dockerfile create mode 100755 build/windows.x86_64/build create mode 100644 build/windows.x86_64/tag diff --git a/.compile_binaries b/.compile_binaries deleted file mode 100755 index 95939ae..0000000 --- a/.compile_binaries +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash - -build_linux() { - # Linux Docker image - name="$DOCKER_BASE" - DOCKER_BUILDS="$DOCKER_BUILDS $name" - docker build -t "$name:current" . - docker run "$name:current" --version - printf '%s\n' "#!/bin/sh" "echo 'hello world'" > myscript - docker run -v "$PWD:/mnt" "$name:current" myscript - - # Copy static executable from docker image - id=$(docker create "$name:current") - docker cp "$id:/bin/shellcheck" "shellcheck" - docker rm "$id" - ls -l shellcheck - ./shellcheck myscript - for tag in $TAGS - do - cp "shellcheck" "deploy/shellcheck-$tag.linux-x86_64"; - done -} - -build_aarch64() { - # Linux aarch64 static executable - docker run -v "$PWD:/mnt" koalaman/aarch64-builder 'buildsc' - for tag in $TAGS - do - cp "shellcheck" "deploy/shellcheck-$tag.linux-aarch64" - done -} - - -build_armv6hf() { - # Linux armv6hf static executable - docker run -v "$PWD:/mnt" koalaman/armv6hf-builder -c 'compile-shellcheck' - for tag in $TAGS - do - cp "shellcheck" "deploy/shellcheck-$tag.linux-armv6hf"; - done -} - -build_windows() { - # Windows .exe - docker run -v "$PWD:/appdata" koalaman/winghc cuib - for tag in $TAGS - do - cp "dist/build/ShellCheck/shellcheck.exe" "deploy/shellcheck-$tag.exe"; - done -} - -build_osx() { - # Darwin x86_64 executable - brew update - brew install cabal-install pandoc gnu-tar - sudo ln -sf /usr/local/bin/gsha512sum /usr/local/bin/sha512sum - sudo ln -sf /usr/local/bin/gtar /usr/local/bin/tar - export PATH="/usr/local/bin:$PATH" - - cabal update - cabal install --dependencies-only - cabal build shellcheck - - # Cabal 3 no longer has a predictable output path - path="$(find . -name 'shellcheck' -type f -perm +111)" - [[ -e "$path" ]] - - for tag in $TAGS - do - cp "$path" "deploy/shellcheck-$tag.darwin-x86_64"; - done -} - diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..cd9fca0 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,124 @@ +name: Build Lol + +# Run this workflow every time a new commit pushed to your repository +on: push + +jobs: + package_source: + name: Package Source Code + runs-on: ubuntu-latest + steps: + - name: Install Dependencies + run: | + sudo apt-get update + sudo apt-mark manual ghc # Don't bother installing ghc just to tar up source + sudo apt-get install cabal-install + + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Package Source + run: | + mkdir source + cabal sdist + mv dist/*.tar.gz source/source.tar.gz + + - name: Deduce tags + run: | + exec > source/tags + echo "latest" + if tag=$(git describe --exact-match --tags) + then + echo "stable" + echo "$tag" + fi + + - name: Upload artifact + uses: actions/upload-artifact@v2 + with: + name: source + path: source/ + + build_source: + name: Build Source Code + needs: package_source + strategy: + matrix: + build: [linux.x86_64, linux.aarch64, linux.armv6hf, darwin.x86_64, windows.x86_64] + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Download artifacts + uses: actions/download-artifact@v2 + + - name: Build source + run: | + mkdir -p bin + mkdir -p bin/${{matrix.build}} + ( cd bin && ../build/run_builder ../source/source.tar.gz ../build/${{matrix.build}} ) + + - name: Upload artifact + uses: actions/upload-artifact@v2 + with: + name: bin + path: bin/ + + package_binary: + name: Package Binaries + needs: build_source + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Download artifacts + uses: actions/download-artifact@v2 + + - name: Work around GitHub permissions bug + run: chmod +x bin/*/shellcheck* + + - name: Package binaries + run: | + export TAGS="$(cat source/tags)" + mkdir -p deploy + cp -r bin/* deploy + cd deploy + ../.prepare_deploy + rm -rf */ README* LICENSE* + + - name: Upload artifact + uses: actions/upload-artifact@v2 + with: + name: deploy + path: deploy/ + + deploy: + name: Deploy binaries + needs: package_binary + runs-on: ubuntu-latest + environment: Deploy + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Download artifacts + uses: actions/download-artifact@v2 + + - name: Upload to GitHub + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + export TAGS="$(cat source/tags)" + ./.github_deploy + + - name: Upload to Docker Hub + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + DOCKER_EMAIL: ${{ secrets.DOCKER_EMAIL }} + DOCKER_BASE: ${{ secrets.DOCKER_USERNAME }}/shellcheck + run: | + export TAGS="$(cat source/tags)" + ( source ./.multi_arch_docker && set -eux && multi_arch_docker::main ) diff --git a/.github_deploy b/.github_deploy index e5d6c3e..3de0ac2 100755 --- a/.github_deploy +++ b/.github_deploy @@ -2,39 +2,10 @@ set -x shopt -s extglob -if [[ "$TRAVIS_SECURE_ENV_VARS" != "true" ]] -then - echo >&2 "Missing TRAVIS_SECURE_ENV_VARS. Skipping GitHub deployment." - exit 0 -fi - -install_deps() { - version="2.7.0" # 2.14.1 fails to overwrite duplicates - case "$(uname)" in - Linux) - sudo apt-get update - sudo apt-get install curl - curl -L "https://github.com/github/hub/releases/download/v$version/hub-linux-amd64-$version.tgz" | tar xvz --strip-components=1 "hub-linux-amd64-$version/bin/hub" - ;; - Darwin) - curl -L "https://github.com/github/hub/releases/download/v$version/hub-darwin-amd64-$version.tgz" | tar xvz --strip-components=1 "hub-darwin-amd64-$version/bin/hub" - ;; - *) - echo "Unknown: $(uname)" - exit 1 - ;; - esac - - hub_path="$PWD/bin/hub" - hub() { - "$hub_path" "$@" - } -} -install_deps - export EDITOR="touch" # Sanity check +gh --version || exit 1 hub release show latest || exit 1 for tag in $TAGS @@ -51,8 +22,8 @@ do do [[ $file == *.@(xz|gz|zip) ]] || continue [[ $file == *"$tag"* ]] || continue - files+=(-a "$file") + files+=("$file") done - hub release edit "${files[@]}" "$tag" || exit 1 + gh release upload "$tag" "${files[@]}" --clobber || exit 1 done diff --git a/.multi_arch_docker b/.multi_arch_docker index 294fed2..a9f7401 100755 --- a/.multi_arch_docker +++ b/.multi_arch_docker @@ -1,12 +1,6 @@ #!/bin/bash # This script builds and deploys multi-architecture docker images from the -# binaries previously built and deployed to GCS by the Travis pipeline. - -if [[ "$TRAVIS_SECURE_ENV_VARS" != "true" ]] -then - echo >&2 "Missing TRAVIS_SECURE_ENV_VARS. Skipping Docker builds." - exit 0 -fi +# binaries previously built and deployed to GitHub. function multi_arch_docker::install_docker_buildx() { # Install up-to-date version of docker, with buildx support. @@ -108,6 +102,5 @@ function multi_arch_docker::main() { multi_arch_docker::install_docker_buildx multi_arch_docker::login_to_docker_hub multi_arch_docker::build_and_push_all - set +x multi_arch_docker::test_all } diff --git a/.prepare_deploy b/.prepare_deploy index a2ce361..9f39912 100755 --- a/.prepare_deploy +++ b/.prepare_deploy @@ -1,8 +1,9 @@ #!/bin/bash -# This script packages up Travis compiled binaries +# This script packages up compiled binaries set -ex shopt -s nullglob extglob -cd deploy + +ls -l cp ../LICENSE LICENSE.txt sed -e $'s/$/\r/' > README.txt << END @@ -22,26 +23,31 @@ This binary was compiled on $(date -u). $(git log -n 3) END -for file in ./*.exe +for dir in */ do - zip "${file%.*}.zip" README.txt LICENSE.txt "$file" + cp LICENSE.txt README.txt "$dir" done -for file in *.{linux,darwin}-* -do - base="${file%.*}" - ext="${file##*.}" - os="${ext%-*}" - arch="${ext##*-}" - cp "$file" "shellcheck" - tar -cJf "$base.$os.$arch.tar.xz" --transform="s:^:$base/:" README.txt LICENSE.txt shellcheck - rm "shellcheck" -done +echo "Tags are $TAGS" -rm !(*.xz|*.zip) +for tag in $TAGS +do + + for dir in windows.*/ + do + ( cd "$dir" && zip "../shellcheck-$tag.zip" * ) + done + + for dir in {linux,darwin}.*/ + do + base="${dir%/}" + ( cd "$dir" && tar -cJf "../shellcheck-$tag.$base.tar.xz" --transform="s:^:shellcheck-$tag/:" * ) + done +done for file in ./* do + [[ -f "$file" ]] || continue sha512sum "$file" > "$file.sha512sum" done diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index d7d202b..0000000 --- a/.travis.yml +++ /dev/null @@ -1,97 +0,0 @@ -language: shell -os: linux - -services: - - docker - -jobs: - include: - - stage: Build - # This must weirdly not have a dash, otherwise an empty job is created - env: BUILD=linux - workspaces: - create: - name: ws-linux - paths: deploy - - env: BUILD=windows - workspaces: - create: - name: ws-windows - paths: deploy - - env: BUILD=armv6hf - workspaces: - create: - name: ws-armv6hf - paths: deploy - - env: BUILD=aarch64 - workspaces: - create: - name: ws-aarch64 - paths: deploy - - env: BUILD=osx - os: osx - workspaces: - create: - name: ws-osx - paths: deploy - - - stage: Upload Artifacts to GitHub - workspaces: - use: - - ws-osx - - ws-linux - - ws-armv6hf - - ws-aarch64 - - ws-windows - script: - - ls -la ${CASHER_DIR}/ || true - # Kludge broken TravisCI workspaces - - tar -xvf ${CASHER_DIR}/ws-osx-fetch.tgz --strip-components=5 - - ls -la deploy - - ./.github_deploy - - - stage: Deploy docker image - # Deploy only for pushes to master branch, not other branches, not PRs. - if: type = push - script: - - source ./.multi_arch_docker - - set -ex; multi_arch_docker::main; set +x - -# This is in global context and runs for every stage that doesn't override it. -before_install: | - DOCKER_BASE="$DOCKER_USERNAME/shellcheck" - DOCKER_BUILDS="" - export TAGS="" - test "$TRAVIS_BRANCH" = master && TAGS="$TAGS latest" || true - test -n "$TRAVIS_TAG" && TAGS="$TAGS stable $TRAVIS_TAG" || true - echo "Tags are $TAGS" - -# This is in global context and runs for every stage that doesn't override it. -script: - - mkdir -p deploy - - source ./.compile_binaries - - ./striptests - - set -ex; build_"$BUILD"; set +x; - - ./.prepare_deploy - -# This is in global context and runs for every stage that doesn't override it. -after_failure: | - id - pwd - df -h - find . -name '*.log' -type f -exec grep "" /dev/null {} + - find . -ls - -# This is in global context and runs for every stage that doesn't override it. -deploy: - provider: gcs - skip_cleanup: true - access_key_id: GOOG7MDN7WEH6IIGBDCA - secret_access_key: - secure: Bcx2cT0/E2ikj7sdamVq52xlLZF9dz9ojGPtoKfPyQhkkZa+McVI4xgUSuyyoSxyKj77sofx2y8m6PJYYumT4g5hREV1tfeUkl0J2DQFMbGDYEt7kxVkXCxojNvhHwTzLFv0ezstrxWWxQm81BfQQ4U9lggRXtndAP4czZnOeHPINPSiue1QNwRAEw05r5UoIUJXy/5xyUrjIxn381pAs+gJqP2COeN9kTKYH53nS/AAws29RprfZFnPlo7xxWmcjRcdS5KPdGXI/c6tQp5zl2iTh510VC1PN2w1Wvnn/oNWhiNdqPyVDsojIX5+sS3nejzJA+KFMxXSBlyXIY3wPpS/MdscU79X6Q5f9ivsFfsm7gNBmxHUPNn0HAvU4ROT/CCE9j6jSbs5PC7QBo3CK4++jxAwE/pd9HUc2rs3k0ofx3rgveJ7txpy5yPKfwIIBi98kVKlC4w7dLvNTOfjW1Imt2yH87XTfsE0UIG9st1WII6s4l/WgBx2GuwKdt6+3QUYiAlCFckkxWi+fAvpHZUEL43Qxub5fN+ZV7Zib1n7opchH4QKGBb6/y0WaDCmtCfu0lppoe/TH6saOTjDFj67NJSElK6ZDxGZ3uw4R+ret2gm6WRKT2Oeub8J33VzSa7VkmFpMPrAAfPa9N1Z4ewBLoTmvxSg2A0dDrCdJio= - bucket: shellcheck-private - local_dir: deploy - on: - repo: koalaman/shellcheck - condition: $TRAVIS_BUILD_STAGE_NAME = Build - all_branches: true diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 2f8f79e..0000000 --- a/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -# Build-only image -FROM ubuntu:18.04 AS build -USER root -WORKDIR /opt/shellCheck - -# Install OS deps -RUN apt-get update && apt-get install -y ghc cabal-install - -# Install Haskell deps -# (This is a separate copy/run so that source changes don't require rebuilding) -COPY ShellCheck.cabal ./ -RUN cabal update && cabal install --dependencies-only --ghc-options="-optlo-Os -split-sections" - -# Copy source and build it -COPY LICENSE shellcheck.hs ./ -COPY src src -RUN cabal build Paths_ShellCheck && \ - ghc -optl-static -optl-pthread -isrc -idist/build/autogen --make shellcheck -split-sections -optc-Wl,--gc-sections -optlo-Os && \ - strip --strip-all shellcheck - -RUN mkdir -p /out/bin && \ - cp shellcheck /out/bin/ - -# Resulting ShellCheck image -FROM scratch -LABEL maintainer="Vidar Holen " -WORKDIR /mnt -COPY --from=build /out / -ENTRYPOINT ["/bin/shellcheck"] diff --git a/build/README.md b/build/README.md new file mode 100644 index 0000000..eb745a0 --- /dev/null +++ b/build/README.md @@ -0,0 +1,13 @@ +This directory contains Dockerfiles for all builds. + +A build image will: + +* Run on Linux x86\_64 with vanilla Docker (no exceptions) +* Not contain any software that would restrict easy modification or copying +* Take a `cabal sdist` style tar.gz of the ShellCheck directory on stdin +* Output a tar.gz of artifacts on stdout, in a directory named for the arch + +This makes it simple to build any release without exotic hardware or software. + +An image can be built and tagged using `build_builder`, +and run on a source tarball using `run_builder`. diff --git a/build/build_builder b/build/build_builder new file mode 100755 index 0000000..b34b996 --- /dev/null +++ b/build/build_builder @@ -0,0 +1,12 @@ +#!/bin/sh +if [ $# -eq 0 ] +then + echo >&2 "No build image directories specified" + echo >&2 "Example: $0 build/*/" + exit 1 +fi + +for dir +do + ( cd "$dir" && docker build -t "$(cat tag)" . ) || exit 1 +done diff --git a/build/darwin.x86_64/Dockerfile b/build/darwin.x86_64/Dockerfile new file mode 100644 index 0000000..e7425fe --- /dev/null +++ b/build/darwin.x86_64/Dockerfile @@ -0,0 +1,31 @@ +# DIGEST:sha256:fa32af4677e2860a1c5950bc8c360f309e2a87e2ddfed27b642fddf7a6093b76 +FROM liushuyu/osxcross:latest + +ENV TARGET x86_64-apple-darwin18 +ENV TARGETNAME darwin.x86_64 + +# Build dependencies +USER root +ENV DEBIAN_FRONTEND noninteractive +RUN apt-get update && apt-get install -y ghc automake autoconf llvm curl + +# Build GHC +WORKDIR /ghc +RUN curl -L "https://downloads.haskell.org/~ghc/8.10.4/ghc-8.10.4-src.tar.xz" | tar xJ --strip-components=1 +RUN ./boot && ./configure --host x86_64-linux-gnu --build x86_64-linux-gnu --target "$TARGET" +RUN cp mk/flavours/quick-cross.mk mk/build.mk && make -j "$(nproc)" +RUN make install +RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.2.0.0/cabal-install-3.2.0.0-x86_64-unknown-linux.tar.xz" | tar xJv -C /usr/local/bin + +# Due to an apparent cabal bug, we specify our options directly to cabal +# It won't reuse caches if ghc-options are specified in ~/.cabal/config +ENV CABALOPTS "--with-ghc=$TARGET-ghc;--with-hc-pkg=$TARGET-ghc-pkg" + +# Prebuild the dependencies +RUN cabal update && IFS=';' && cabal install $CABALOPTS --lib Diff-0.4.0 base-compat-0.11.2 base-orphans-0.8.4 dlist-1.0 hashable-1.3.0.0 indexed-traversable-0.1.1 integer-logarithms-1.0.3.1 primitive-0.7.1.0 regex-base-0.94.0.0 splitmix-0.1.0.3 tagged-0.8.6.1 th-abstraction-0.4.2.0 transformers-compat-0.6.6 base-compat-batteries-0.11.2 time-compat-1.9.5 unordered-containers-0.2.13.0 data-fix-0.3.1 vector-0.12.2.0 scientific-0.3.6.2 regex-tdfa-1.3.1.0 random-1.2.0 distributive-0.6.2.1 attoparsec-0.13.2.5 uuid-types-1.0.3 comonad-5.0.8 bifunctors-5.5.10 assoc-1.0.2 these-1.1.1.1 strict-0.4.0.1 aeson-1.5.5.1 + +# Copy the build script +COPY build /usr/bin + +WORKDIR /scratch +ENTRYPOINT ["/usr/bin/build"] diff --git a/build/darwin.x86_64/build b/build/darwin.x86_64/build new file mode 100755 index 0000000..03dfde7 --- /dev/null +++ b/build/darwin.x86_64/build @@ -0,0 +1,14 @@ +#!/bin/sh +set -xe +{ + tar xzv --strip-components=1 + ./striptests + mkdir "$TARGETNAME" + cabal update + ( IFS=';'; cabal build $CABALOPTS ) + find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \; + ls -l "$TARGETNAME" + "$TARGET-strip" -Sx "$TARGETNAME/shellcheck" + ls -l "$TARGETNAME" +} >&2 +tar czv "$TARGETNAME" diff --git a/build/darwin.x86_64/tag b/build/darwin.x86_64/tag new file mode 100644 index 0000000..237a65c --- /dev/null +++ b/build/darwin.x86_64/tag @@ -0,0 +1 @@ +koalaman/scbuilder-darwin-x86_64 diff --git a/build/linux.aarch64/Dockerfile b/build/linux.aarch64/Dockerfile new file mode 100644 index 0000000..8bf8d6f --- /dev/null +++ b/build/linux.aarch64/Dockerfile @@ -0,0 +1,30 @@ +FROM ubuntu:20.04 + +ENV TARGET aarch64-linux-gnu +ENV TARGETNAME linux.aarch64 + +# Build dependencies +USER root +ENV DEBIAN_FRONTEND noninteractive +RUN apt-get update && apt-get install -y ghc automake autoconf build-essential llvm curl qemu-user-static gcc-$TARGET + +# Build GHC +WORKDIR /ghc +RUN curl -L "https://downloads.haskell.org/~ghc/8.10.4/ghc-8.10.4-src.tar.xz" | tar xJ --strip-components=1 +RUN ./boot && ./configure --host x86_64-linux-gnu --build x86_64-linux-gnu --target "$TARGET" +RUN cp mk/flavours/quick-cross.mk mk/build.mk && make -j "$(nproc)" +RUN make install +RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.2.0.0/cabal-install-3.2.0.0-x86_64-unknown-linux.tar.xz" | tar xJv -C /usr/local/bin + +# Due to an apparent cabal bug, we specify our options directly to cabal +# It won't reuse caches if ghc-options are specified in ~/.cabal/config +ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections;--with-ghc=$TARGET-ghc;--with-hc-pkg=$TARGET-ghc-pkg" + +# Prebuild the dependencies +RUN cabal update && IFS=';' && cabal install $CABALOPTS --lib Diff-0.4.0 base-compat-0.11.2 base-orphans-0.8.4 dlist-1.0 hashable-1.3.0.0 indexed-traversable-0.1.1 integer-logarithms-1.0.3.1 primitive-0.7.1.0 regex-base-0.94.0.0 splitmix-0.1.0.3 tagged-0.8.6.1 th-abstraction-0.4.2.0 transformers-compat-0.6.6 base-compat-batteries-0.11.2 time-compat-1.9.5 unordered-containers-0.2.13.0 data-fix-0.3.1 vector-0.12.2.0 scientific-0.3.6.2 regex-tdfa-1.3.1.0 random-1.2.0 distributive-0.6.2.1 attoparsec-0.13.2.5 uuid-types-1.0.3 comonad-5.0.8 bifunctors-5.5.10 assoc-1.0.2 these-1.1.1.1 strict-0.4.0.1 aeson-1.5.5.1 + +# Copy the build script +COPY build /usr/bin + +WORKDIR /scratch +ENTRYPOINT ["/usr/bin/build"] diff --git a/build/linux.aarch64/build b/build/linux.aarch64/build new file mode 100755 index 0000000..1164dc1 --- /dev/null +++ b/build/linux.aarch64/build @@ -0,0 +1,15 @@ +#!/bin/sh +set -xe +{ + tar xzv --strip-components=1 + ./striptests + mkdir "$TARGETNAME" + cabal update + ( IFS=';'; cabal build $CABALOPTS --enable-executable-static ) + find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \; + ls -l "$TARGETNAME" + "$TARGET-strip" -s "$TARGETNAME/shellcheck" + ls -l "$TARGETNAME" + qemu-aarch64-static "$TARGETNAME/shellcheck" --version +} >&2 +tar czv "$TARGETNAME" diff --git a/build/linux.aarch64/tag b/build/linux.aarch64/tag new file mode 100644 index 0000000..6788e14 --- /dev/null +++ b/build/linux.aarch64/tag @@ -0,0 +1 @@ +koalaman/scbuilder-linux-aarch64 diff --git a/build/linux.armv6hf/Dockerfile b/build/linux.armv6hf/Dockerfile new file mode 100644 index 0000000..fee6d4c --- /dev/null +++ b/build/linux.armv6hf/Dockerfile @@ -0,0 +1,59 @@ +# I've again spent days trying to get a working armv6hf compiler going. +# God only knows how many recompilations of GCC, GHC, libraries, and +# ShellCheck itself, has gone into it. +# +# I tried Debian's toolchain. I tried my custom one built according to +# RPi `gcc -v`. I tried GHC9, glibc, musl, registerised vs not, but +# nothing has yielded an armv6hf binary that does not immediately +# segfault on qemu-arm-static or the RPi itself. +# +# I then tried the same but with armv7hf. Same story. +# +# Emulating the entire userspace with balenalib again? Very strange build +# failures where programs would fail to execute with > ~100 arguments. +# +# Finally, creating our own appears to work when using a custom QEmu +# patched to follow execve calls. +# +# PS: $100 bounty for getting a RPi1 compatible static build going +# with cross-compilation, similar to what the aarch64 build does. +# + +FROM ubuntu:20.04 + +ENV TARGETNAME linux.armv6hf + +# Build QEmu with execve follow support +USER root +ENV DEBIAN_FRONTEND noninteractive +RUN apt-get update +RUN apt-get install -y build-essential git ninja-build python3 pkg-config libglib2.0-dev libpixman-1-dev +WORKDIR /build +RUN git clone --depth 1 https://github.com/koalaman/qemu +RUN cd qemu && ./configure --static && cd build && ninja qemu-arm +RUN cp qemu/build/qemu-arm /build/qemu-arm-static +ENV QEMU_EXECVE 1 + +# Set up an armv6 userspace +WORKDIR / +RUN apt-get install -y debootstrap qemu-user-static +# We expect this to fail if the host doesn't have binfmt qemu support +RUN qemu-debootstrap --arch armhf bullseye pi http://mirrordirector.raspbian.org/raspbian || [ -e /pi/etc/issue ] +RUN cp /build/qemu-arm-static /pi/usr/bin/qemu-arm-static +RUN printf > /bin/pirun '%s\n' '#!/bin/sh' 'chroot /pi /usr/bin/qemu-arm-static /usr/bin/env "$@"' && chmod +x /bin/pirun +# If the debootstrap process didn't finish, continue it +RUN [ ! -e /pi/debootstrap ] || pirun '/debootstrap/debootstrap' --second-stage + +# Install deps in the chroot +RUN pirun apt-get update +RUN pirun apt-get install -y ghc cabal-install + +# Finally we can build the current dependencies. This takes hours. +ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections;--gcc-options;-Os -Wl,--gc-sections -ffunction-sections -fdata-sections" +RUN pirun cabal update +RUN IFS=";" && pirun cabal install --lib $CABALOPTS Diff-0.4.0 base-compat-0.11.2 base-orphans-0.8.4 dlist-1.0 hashable-1.3.1.0 indexed-traversable-0.1.1 integer-logarithms-1.0.3.1 primitive-0.7.1.0 regex-base-0.94.0.1 splitmix-0.1.0.3 tagged-0.8.6.1 th-abstraction-0.4.2.0 transformers-compat-0.6.6 base-compat-batteries-0.11.2 time-compat-1.9.5 unordered-containers-0.2.13.0 data-fix-0.3.1 vector-0.12.2.0 scientific-0.3.6.2 regex-tdfa-1.3.1.0 random-1.2.0 distributive-0.6.2.1 attoparsec-0.13.2.5 uuid-types-1.0.4 comonad-5.0.8 bifunctors-5.5.10 assoc-1.0.2 these-1.1.1.1 strict-0.4.0.1 aeson-1.5.6.0 + +# Copy the build script +WORKDIR /pi/scratch +COPY build /pi/usr/bin +ENTRYPOINT ["/bin/pirun", "/usr/bin/build"] diff --git a/build/linux.armv6hf/build b/build/linux.armv6hf/build new file mode 100755 index 0000000..4f2c7bc --- /dev/null +++ b/build/linux.armv6hf/build @@ -0,0 +1,16 @@ +#!/bin/sh +set -xe +cd /scratch +{ + tar xzv --strip-components=1 + ./striptests + mkdir "$TARGETNAME" + # This script does not cabal update because compiling anything new is slow + ( IFS=';'; cabal build $CABALOPTS --enable-executable-static ) + find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \; + ls -l "$TARGETNAME" + strip -s "$TARGETNAME/shellcheck" + ls -l "$TARGETNAME" + "$TARGETNAME/shellcheck" --version +} >&2 +tar czv "$TARGETNAME" diff --git a/build/linux.armv6hf/tag b/build/linux.armv6hf/tag new file mode 100644 index 0000000..9172c5c --- /dev/null +++ b/build/linux.armv6hf/tag @@ -0,0 +1 @@ +koalaman/scbuilder-linux-armv6hf diff --git a/build/linux.x86_64/Dockerfile b/build/linux.x86_64/Dockerfile new file mode 100644 index 0000000..f0ad16a --- /dev/null +++ b/build/linux.x86_64/Dockerfile @@ -0,0 +1,26 @@ +FROM ubuntu:20.04 + +ENV TARGETNAME linux.x86_64 + +# Install GHC and cabal +USER root +ENV DEBIAN_FRONTEND noninteractive +RUN apt-get update && apt-get install -y ghc curl xz-utils + +# So we'd like a later version of Cabal that supports --enable-executable-static, +# but we can't use Ubuntu 20.10 because coreutils has switched to new syscalls that +# the TravisCI kernel doesn't support. Download it manually. +RUN curl "https://downloads.haskell.org/~cabal/cabal-install-3.2.0.0/cabal-install-3.2.0.0-x86_64-unknown-linux.tar.xz" | tar xJv -C /usr/bin + +# Use ld.bfd instead of ld.gold due to +# x86_64-linux-gnu/libpthread.a(pthread_cond_init.o)(.note.stapsdt+0x14): error: +# relocation refers to local symbol "" [2], which is defined in a discarded section +ENV CABALOPTS "--ghc-options;-optl-Wl,-fuse-ld=bfd -split-sections -optc-Os -optc-Wl,--gc-sections" + +# Other archs pre-build dependencies here, but this one doesn't to detect ecosystem movement + +# Copy the build script +COPY build /usr/bin + +WORKDIR /scratch +ENTRYPOINT ["/usr/bin/build"] diff --git a/build/linux.x86_64/build b/build/linux.x86_64/build new file mode 100755 index 0000000..ba598e6 --- /dev/null +++ b/build/linux.x86_64/build @@ -0,0 +1,15 @@ +#!/bin/sh +set -xe +{ + tar xzv --strip-components=1 + ./striptests + mkdir "$TARGETNAME" + cabal update + ( IFS=';'; cabal build $CABALOPTS --enable-executable-static ) + find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \; + ls -l "$TARGETNAME" + strip -s "$TARGETNAME/shellcheck" + ls -l "$TARGETNAME" + "$TARGETNAME/shellcheck" --version +} >&2 +tar czv "$TARGETNAME" diff --git a/build/linux.x86_64/tag b/build/linux.x86_64/tag new file mode 100644 index 0000000..f0224de --- /dev/null +++ b/build/linux.x86_64/tag @@ -0,0 +1 @@ +koalaman/scbuilder-linux-x86_64 diff --git a/build/run_builder b/build/run_builder new file mode 100755 index 0000000..d6de27b --- /dev/null +++ b/build/run_builder @@ -0,0 +1,30 @@ +#!/bin/bash +if [ $# -lt 2 ] +then + echo >&2 "This script builds a source archive (as produced by cabal sdist)" + echo >&2 "Usage: $0 sourcefile.tar.gz builddir..." + exit 1 +fi + +file=$(realpath "$1") +shift + +if [ ! -e "$file" ] +then + echo >&2 "$file does not exist" + exit 1 +fi + +set -ex -o pipefail + +for dir +do + tagfile="$dir/tag" + if [ ! -e "$tagfile" ] + then + echo >&2 "$tagfile does not exist" + exit 2 + fi + + docker run -i "$(< "$tagfile")" < "$file" | tar xz +done diff --git a/build/windows.x86_64/Dockerfile b/build/windows.x86_64/Dockerfile new file mode 100644 index 0000000..3eb20fa --- /dev/null +++ b/build/windows.x86_64/Dockerfile @@ -0,0 +1,27 @@ +FROM ubuntu:20.04 + +ENV TARGETNAME windows.x86_64 + +# We don't need wine32, even though it complains +USER root +ENV DEBIAN_FRONTEND noninteractive +RUN apt-get update && apt-get install -y curl busybox wine + +# Fetch Windows version, will be available under z:\haskell +WORKDIR /haskell +RUN curl -L "https://downloads.haskell.org/~ghc/8.10.4/ghc-8.10.4-x86_64-unknown-mingw32.tar.xz" | tar xJ --strip-components=1 +WORKDIR /haskell/bin +RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.2.0.0/cabal-install-3.2.0.0-x86_64-unknown-mingw32.zip" | busybox unzip - +RUN curl -L "https://curl.se/windows/dl-7.75.0/curl-7.75.0-win64-mingw.zip" | busybox unzip - && mv curl-7.75.0-win64-mingw/bin/* . +ENV WINEPATH /haskell/bin + +# It's unknown whether Cabal on Windows suffers from the same issue +# that necessitated this but I don't care enough to find out +ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections" + +# Precompile some deps to speed up later builds. This list is just copied from `cabal build` +RUN wine /haskell/bin/cabal.exe update && IFS=';' && wine /haskell/bin/cabal.exe install $CABALOPTS --lib Diff-0.4.0 base-compat-0.11.2 base-orphans-0.8.4 dlist-1.0 hashable-1.3.0.0 indexed-traversable-0.1.1 integer-logarithms-1.0.3.1 primitive-0.7.1.0 regex-base-0.94.0.0 splitmix-0.1.0.3 tagged-0.8.6.1 th-abstraction-0.4.2.0 transformers-compat-0.6.6 base-compat-batteries-0.11.2 time-compat-1.9.5 unordered-containers-0.2.13.0 data-fix-0.3.1 vector-0.12.2.0 scientific-0.3.6.2 regex-tdfa-1.3.1.0 random-1.2.0 distributive-0.6.2.1 attoparsec-0.13.2.5 uuid-types-1.0.3 comonad-5.0.8 bifunctors-5.5.10 assoc-1.0.2 these-1.1.1.1 strict-0.4.0.1 aeson-1.5.5.1 + +COPY build /usr/bin +WORKDIR /scratch +ENTRYPOINT ["/usr/bin/build"] diff --git a/build/windows.x86_64/build b/build/windows.x86_64/build new file mode 100755 index 0000000..bedb870 --- /dev/null +++ b/build/windows.x86_64/build @@ -0,0 +1,19 @@ +#!/bin/sh +cabal() { + wine /haskell/bin/cabal.exe "$@" +} + +set -xe +{ + tar xzv --strip-components=1 + ./striptests + mkdir "$TARGETNAME" + cabal update + ( IFS=';'; cabal build $CABALOPTS ) + find dist*/ -name shellcheck.exe -type f -ls -exec mv {} "$TARGETNAME/" \; + ls -l "$TARGETNAME" + wine "/haskell/mingw/bin/strip.exe" -s "$TARGETNAME/shellcheck.exe" + ls -l "$TARGETNAME" + wine "$TARGETNAME/shellcheck.exe" --version +} >&2 +tar czv "$TARGETNAME" diff --git a/build/windows.x86_64/tag b/build/windows.x86_64/tag new file mode 100644 index 0000000..a85921b --- /dev/null +++ b/build/windows.x86_64/tag @@ -0,0 +1 @@ +koalaman/scbuilder-windows-x86_64