Drop attoparsec/text dependencies

improve short description for SC2038

.github/dependabot.yml

@@ -0,0 +1,7 @@
+version: 2
+
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"

.github/workflows/build.yml

@@ -1,4 +1,4 @@
-name: Build Lol
+name: Build ShellCheck
 
 # Run this workflow every time a new commit pushed to your repository
 on: push
@@ -15,26 +15,29 @@ jobs:
           sudo apt-get install cabal-install
 
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
-
+        with:
-      - name: Package Source
+          fetch-depth: 0
-        run: |
-          mkdir source
-          cabal sdist
-          mv dist/*.tar.gz source/source.tar.gz
 
       - name: Deduce tags
         run: |
-          exec > source/tags
+          mkdir source
-          echo "latest"
+          echo "latest" > source/tags
           if tag=$(git describe --exact-match --tags)
           then
-            echo "stable"
+            echo "stable" >> source/tags
-            echo "$tag"
+            echo "$tag" >> source/tags
           fi
+          cat source/tags
+
+      - name: Package Source
+        run: |
+          grep "stable" source/tags || ./setgitversion
+          cabal sdist
+          mv dist-newstyle/sdist/*.tar.gz source/source.tar.gz
 
       - name: Upload artifact
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: source
           path: source/
@@ -48,10 +51,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Download artifacts
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
 
       - name: Build source
         run: |
@@ -60,7 +63,7 @@ jobs:
           ( cd bin && ../build/run_builder ../source/source.tar.gz ../build/${{matrix.build}} )
 
       - name: Upload artifact
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: bin
           path: bin/
@@ -71,10 +74,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Download artifacts
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
 
       - name: Work around GitHub permissions bug
         run: chmod +x bin/*/shellcheck*
@@ -89,7 +92,7 @@ jobs:
           rm -rf */ README* LICENSE*
 
       - name: Upload artifact
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: deploy
           path: deploy/
@@ -101,10 +104,10 @@ jobs:
     environment: Deploy
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Download artifacts
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
 
       - name: Upload to GitHub
         env:
@@ -113,6 +116,10 @@ jobs:
           export TAGS="$(cat source/tags)"
           ./.github_deploy
 
+      - name: Waiting for GitHub to replicate uploaded releases
+        run: |
+          sleep 300
+
       - name: Upload to Docker Hub
         env:
           DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}

.github_deploy

@@ -26,4 +26,3 @@ do
   done
   gh release upload "$tag" "${files[@]}" --clobber || exit 1
 done
-

.multi_arch_docker

@@ -3,28 +3,10 @@
 # binaries previously built and deployed to GitHub.
 
 function multi_arch_docker::install_docker_buildx() {
-  # Install up-to-date version of docker, with buildx support.
-  local -r docker_apt_repo='https://download.docker.com/linux/ubuntu'
-  curl -fsSL "${docker_apt_repo}/gpg" | sudo apt-key add -
-  local -r os="$(lsb_release -cs)"
-  sudo add-apt-repository "deb [arch=amd64] $docker_apt_repo $os stable"
-  sudo apt-get update
-  sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
-
-  # Enable docker daemon experimental support (for 'pull --platform').
-  local -r config='/etc/docker/daemon.json'
-  if [[ -e "$config" ]]; then
-    sudo sed -i -e 's/{/{ "experimental": true, /' "$config"
-  else
-    echo '{ "experimental": true }' | sudo tee "$config"
-  fi
-  sudo systemctl restart docker
-
   # Install QEMU multi-architecture support for docker buildx.
   docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
 
   # Instantiate docker buildx builder with multi-architecture support.
-  export DOCKER_CLI_EXPERIMENTAL=enabled
   docker buildx create --name mybuilder
   docker buildx use mybuilder
   # Start up buildx and verify that all is OK.

CHANGELOG.md

@@ -1,13 +1,87 @@
 ## Git
 ### Added
+- SC2324: Warn when x+=1 appends instead of increments
+- SC2325: Warn about multiple `!`s in dash/sh.
+- SC2326: Warn about `foo | ! bar` in bash/dash/sh.
+
+### Fixed
+- source statements with here docs now work correctly
+
+### Changed
+
+
+## v0.9.0 - 2022-12-12
+### Added
+- SC2316: Warn about 'local readonly foo' and similar (thanks, patrickxia!)
+- SC2317: Warn about unreachable commands
+- SC2318: Warn about backreferences in 'declare x=1 y=$x'
+- SC2319/SC2320: Warn when $? refers to echo/printf/[ ]/[[ ]]/test
+- SC2321: Suggest removing $((..)) in array[$((idx))]=val
+- SC2322: Suggest collapsing double parentheses in arithmetic contexts
+- SC2323: Suggest removing wrapping parentheses in a[(x+1)]=val
+
+### Fixed
+- SC2086: Now uses DFA to make more accurate predictions about values
+- SC2086: No longer warns about values declared as integer with declare -i
+
+### Changed
+- ShellCheck now has a Data Flow Analysis engine to make smarter decisions
+  based on control flow rather than just syntax. Existing checks will
+  gradually start using it, which may cause them to trigger differently
+  (but more accurately).
+- Values in directives/shellcheckrc can now be quoted with '' or ""
+
+
+## v0.8.0 - 2021-11-06
+### Added
+- `disable=all` now conveniently disables all warnings
+- `external-sources=true` directive can be added to .shellcheckrc to make
+  shellcheck behave as if `-x` was specified.
+- Optional `check-extra-masked-returns` for pointing out commands with
+  suppressed exit codes (SC2312).
+- Optional `require-double-brackets` for recommending \[\[ ]] (SC2292).
+- SC2286-SC2288: Warn when command name ends in a symbol like `/.)'"`
+- SC2289: Warn when command name contains tabs or linefeeds
+- SC2291: Warn about repeated unquoted spaces between words in echo
+- SC2292: Suggest [[ over [ in Bash/Ksh scripts (optional)
+- SC2293/SC2294: Warn when calling `eval` with arrays
+- SC2295: Warn about "${x#$y}" treating $y as a pattern when not quoted
+- SC2296-SC2301: Improved warnings for bad parameter expansions
+- SC2302/SC2303: Warn about loops over array values when using them as keys
+- SC2304-SC2306: Warn about unquoted globs in expr arguments
+- SC2307: Warn about insufficient number of arguments to expr
+- SC2308: Suggest other approaches for non-standard expr extensions
+- SC2313: Warn about `read` with unquoted, array indexed variable
+
+### Fixed
+- SC2102 about repetitions in ranges no longer triggers on [[ -v arr[xx] ]]
+- SC2155 now recognizes `typeset` and local read-only `declare` statements
+- SC2181 now tries to avoid triggering for error handling functions
+- SC2290: Warn about misused = in declare & co, which were not caught by SC2270+
+- The flag --color=auto no longer outputs color when TERM is "dumb" or unset
+
+### Changed
+- SC2048: Warning about $\* now also applies to ${array[\*]}
+- SC2181 now only triggers on single condition tests like `[ $? = 0 ]`.
+- Quote warnings are now emitted for declaration utilities in sh
+- Leading `_` can now be used to suppress warnings about unused variables
+- TTY output now includes warning level in text as well as color
+
+### Removed
+- SC1004: Literal backslash+linefeed in '' was found to be usually correct
+
+
+## v0.7.2 - 2021-04-19
+### Added
 - `disable` directives can now be a range, e.g. `disable=SC3000-SC4000`
+- SC1143: Warn about line continuations in comments
 - SC2259/SC2260: Warn when redirections override pipes
 - SC2261: Warn about multiple competing redirections
 - SC2262/SC2263: Warn about aliases declared and used in the same parsing unit
 - SC2264: Warn about wrapper functions that blatantly recurse
 - SC2265/SC2266: Warn when using & or | with test statements
 - SC2267: Warn when using xargs -i instead of -I
-- Optional avoid-x-comparisons: Style warning SC2268 for `[ x$var = xval ]`
+- SC2268: Warn about unnecessary x-comparisons like `[ x$var = xval ]`
 
 ### Fixed
 - SC1072/SC1073 now respond to disable annotations, though ignoring parse errors
@@ -21,7 +95,7 @@
 - POSIX/dash unsupported feature warnings now have individual SC3xxx codes
 - SC1090: A leading `$x/` or `$(x)/` is now treated as `./` when locating files
 - SC2154: Variables appearing in -z/-n tests are no longer considered unassigned
-- SC2270-SC2285: Improved warnings about misused =, e.g. `${var}=42`
+- SC2270-SC2285: Improved warnings about misused `=`, e.g. `${var}=42`
 
 
 ## v0.7.1 - 2020-04-04
@@ -164,7 +238,7 @@
 - SC2204/SC2205: Warn about `( -z foo )` and `( foo -eq bar )`
 - SC2200/SC2201: Warn about brace expansion in [/[[
 - SC2198/SC2199: Warn about arrays in [/[[
-- SC2196/SC2197: Warn about deprected egrep/fgrep
+- SC2196/SC2197: Warn about deprecated egrep/fgrep
 - SC2195: Warn about unmatchable case branches
 - SC2194: Warn about constant 'case' statements
 - SC2193: Warn about `[[ file.png == *.mp3 ]]` and other unmatchables
@@ -181,7 +255,7 @@
 ### Fixed
 - `-c` no longer suggested when using `grep -o | wc`
 - Comments and whitespace are now allowed before filewide directives
-- Here doc delimters with esoteric quoting like `foo""` are now handled
+- Here doc delimiters with esoteric quoting like `foo""` are now handled
 - SC2095 about `ssh` in while read loops is now suppressed when using `-n`
 - `%(%Y%M%D)T` now recognized as a single formatter in `printf` checks
 - `grep -F` now suppresses regex related suggestions

LICENSE

@@ -1,13 +1,3 @@
-Employer mandated disclaimer:
-
-  I am providing code in the repository to you under an open source license.
-  Because this is my personal repository, the license you receive to my code is
-  from me and other individual contributors, and not my employer (Facebook).
-
-  - Vidar "koala_man" Holen
-
-----
-
                     GNU GENERAL PUBLIC LICENSE
                        Version 3, 29 June 2007
 
@@ -681,4 +671,4 @@ into proprietary programs.  If your program is a subroutine library, you
 may consider it more useful to permit linking proprietary applications with
 the library.  If this is what you want to do, use the GNU Lesser General
 Public License instead of this License.  But first, please read
-<https://www.gnu.org/philosophy/why-not-lgpl.html>.
+<https://www.gnu.org/licenses/why-not-lgpl.html>.

README.md

@@ -1,4 +1,5 @@
-[![Build Status](https://travis-ci.org/koalaman/shellcheck.svg?branch=master)](https://travis-ci.org/koalaman/shellcheck)
+[![Build Status](https://github.com/koalaman/shellcheck/actions/workflows/build.yml/badge.svg)](https://github.com/koalaman/shellcheck/actions/workflows/build.yml)
+
 
 # ShellCheck - A shell script static analysis tool
 
@@ -111,10 +112,7 @@ Services and platforms that have ShellCheck pre-installed and ready to use:
 * [Code Factor](https://www.codefactor.io/)
 * [CircleCI](https://circleci.com) via the [ShellCheck Orb](https://circleci.com/orbs/registry/orb/circleci/shellcheck)
 * [Github](https://github.com/features/actions) (only Linux)
-
+* [Trunk Check](https://trunk.io/products/check) (universal linter; [allows you to explicitly version your shellcheck install](https://github.com/trunk-io/plugins/blob/bcbb361dcdbe4619af51ea7db474d7fb87540d20/.trunk/trunk.yaml#L32)) via the [shellcheck plugin](https://github.com/trunk-io/plugins/blob/main/linters/shellcheck/plugin.yaml)
-Services and platforms with third party plugins:
-
-* [SonarQube](https://www.sonarqube.org/) through [sonar-shellcheck-plugin](https://github.com/emerald-squad/sonar-shellcheck-plugin)
 
 Most other services, including [GitLab](https://about.gitlab.com/), let you install
 ShellCheck yourself, either through the system's package manager (see [Installing](#installing)),
@@ -143,7 +141,7 @@ On systems with Stack (installs to `~/.local/bin`):
 
 On Debian based distros:
 
-    apt-get install shellcheck
+    sudo apt install shellcheck
 
 On Arch Linux based distros:
 
@@ -157,8 +155,8 @@ On Gentoo based distros:
 
 On EPEL based distros:
 
-    yum -y install epel-release
+    sudo yum -y install epel-release
-    yum install ShellCheck
+    sudo yum install ShellCheck
 
 On Fedora based distros:
 
@@ -235,6 +233,9 @@ Alternatively, you can download pre-compiled binaries for the latest release her
 or see the [GitHub Releases](https://github.com/koalaman/shellcheck/releases) for other releases
 (including the [latest](https://github.com/koalaman/shellcheck/releases/tag/latest) meta-release for daily git builds).
 
+There are currently no official binaries for Apple Silicon, but third party builds are available via
+[ShellCheck for Visual Studio Code](https://github.com/vscode-shellcheck/shellcheck-binaries/releases).
+
 Distro packages already come with a `man` page. If you are building from source, it can be installed with:
 
 ```console
@@ -242,6 +243,19 @@ pandoc -s -f markdown-smart -t man shellcheck.1.md -o shellcheck.1
 sudo mv shellcheck.1 /usr/share/man/man1
 ```
 
+### pre-commit
+
+To run ShellCheck via [pre-commit](https://pre-commit.com/), add the hook to your `.pre-commit-config.yaml`:
+
+```
+repos:
+-   repo: https://github.com/koalaman/shellcheck-precommit
+    rev: v0.7.2
+    hooks:
+    -   id: shellcheck
+#       args: ["--severity=warning"]  # Optionally only show errors and warnings
+```
+
 ### Travis CI
 
 Travis CI has now integrated ShellCheck by default, so you don't need to manually install it.
@@ -348,6 +362,7 @@ echo 'Don't forget to restart!'   # Singlequote closed by apostrophe
 echo 'Don\'t try this at home'    # Attempting to escape ' in ''
 echo 'Path is $PATH'              # Variables in single quotes
 trap "echo Took ${SECONDS}s" 0    # Prematurely expanded trap
+unset var[i]                      # Array index treated as glob
 ```
 
 ### Conditionals
@@ -366,6 +381,7 @@ ShellCheck can recognize many types of incorrect test statements.
 [ grep -q foo file ]              # Command without $(..)
 [[ "$$file" == *.jpg ]]           # Comparisons that can't succeed
 (( 1 -lt 2 ))                     # Using test operators in ((..))
+[ x ] & [ y ] | [ z ]             # Accidental backgrounding and piping
 ```
 
 ### Frequently misused commands
@@ -437,6 +453,8 @@ echo "Hello $name"                # Unassigned lowercase variables
 cmd | read bar; echo $bar         # Assignments in subshells
 cat foo | cp bar                  # Piping to commands that don't read
 printf '%s: %s\n' foo             # Mismatches in printf argument count
+eval "${array[@]}"                # Lost word boundaries in array eval
+for i in "${x[@]}"; do ${x[$i]}   # Using array value as key
 ```
 
 ### Robustness
@@ -461,6 +479,7 @@ ShellCheck will warn when using features not supported by the shebang. For examp
 echo {1..$n}                     # Works in ksh, but not bash/dash/sh
 echo {1..10}                     # Works in ksh and bash, but not dash/sh
 echo -n 42                       # Works in ksh, bash and dash, undefined in sh
+expr match str regex             # Unportable alias for `expr str : regex`
 trap 'exit 42' sigint            # Unportable signal spec
 cmd &> file                      # Unportable redirection operator
 read foo < /dev/tcp/host/22      # Unportable intercepted files
@@ -481,10 +500,15 @@ rm “file”                         # Unicode quotes
 echo "Hello world"                # Carriage return / DOS line endings
 echo hello \                      # Trailing spaces after \
 var=42 echo $var                  # Expansion of inlined environment
-#!/bin/bash -x -e                 # Common shebang errors
+!# bin/bash -x -e                 # Common shebang errors
 echo $((n/180*100))               # Unnecessary loss of precision
 ls *[:digit:].txt                 # Bad character class globs
 sed 's/foo/bar/' file > file      # Redirecting to input
+var2=$var2                        # Variable assigned to itself
+[ x$var = xval ]                  # Antiquated x-comparisons
+ls() { ls -l "$@"; }              # Infinitely recursive wrapper
+alias ls='ls -l'; ls foo          # Alias used before it takes effect
+for x; do for x; do               # Nested loop uses same variable
 while getopts "a" f; do case $f in "b") # Unhandled getopts flags
 ```
 

ShellCheck.cabal

@@ -1,5 +1,5 @@
 Name:             ShellCheck
-Version:          0.7.1
+Version:          0.9.0
 Synopsis:         Shell script analysis tool
 License:          GPL-3
 License-file:     LICENSE
@@ -8,7 +8,7 @@ Author:           Vidar Holen
 Maintainer:       vidar@vidarholen.net
 Homepage:         https://www.shellcheck.net/
 Build-Type:       Simple
-Cabal-Version:    >= 1.8
+Cabal-Version:    1.18
 Bug-reports:      https://github.com/koalaman/shellcheck/issues
 Description:
   The goals of ShellCheck are:
@@ -22,9 +22,11 @@ Description:
   * To point out subtle caveats, corner cases and pitfalls, that may cause an
     advanced user's otherwise working script to fail under future circumstances.
 
+Extra-Doc-Files:
+    README.md
+    CHANGELOG.md
 Extra-Source-Files:
     -- documentation
-    README.md
     shellcheck.1.md
     -- A script to build the man page using pandoc
     manpage
@@ -43,19 +45,26 @@ library
       build-depends:
         semigroups
     build-depends:
-      aeson,
+      -- The lower bounds are based on GHC 7.10.3
-      array,
+      -- The upper bounds are based on GHC 9.6.1
+      aeson                >= 1.4.0 && < 2.2,
+      array                >= 0.5.1 && < 0.6,
       base                 >= 4.8.0.0 && < 5,
-      bytestring,
+      bytestring           >= 0.10.6 && < 0.12,
-      containers >= 0.5,
+      containers           >= 0.5.6 && < 0.7,
-      deepseq >= 1.4.0.0,
+      deepseq              >= 1.4.1 && < 1.5,
-      Diff >= 0.2.0,
+      Diff                 >= 0.4.0 && < 0.5,
-      directory >= 1.2.3.0,
+      fgl                  (>= 5.7.0 && < 5.8.1.0) || (>= 5.8.1.1 && < 5.9),
-      mtl >= 2.2.1,
+      filepath             >= 1.4.0 && < 1.5,
-      filepath,
+      mtl                  >= 2.2.2 && < 2.4,
-      parsec,
+      parsec               >= 3.1.14 && < 3.2,
-      regex-tdfa,
+      QuickCheck           >= 2.14.2 && < 2.15,
-      QuickCheck >= 2.7.4,
+      regex-tdfa           >= 1.2.0 && < 1.4,
+      transformers         >= 0.4.2 && < 0.7,
+
+      -- getXdgDirectory from 1.2.3.0
+      directory            >= 1.2.3 && < 1.4,
+
       -- When cabal supports it, move this to setup-depends:
       process
     exposed-modules:
@@ -64,11 +73,15 @@ library
       ShellCheck.Analytics
       ShellCheck.Analyzer
       ShellCheck.AnalyzerLib
+      ShellCheck.CFG
+      ShellCheck.CFGAnalysis
       ShellCheck.Checker
       ShellCheck.Checks.Commands
+      ShellCheck.Checks.ControlFlow
       ShellCheck.Checks.Custom
       ShellCheck.Checks.ShellSupport
       ShellCheck.Data
+      ShellCheck.Debug
       ShellCheck.Fixer
       ShellCheck.Formatter.Format
       ShellCheck.Formatter.CheckStyle
@@ -80,9 +93,12 @@ library
       ShellCheck.Formatter.Quiet
       ShellCheck.Interface
       ShellCheck.Parser
+      ShellCheck.PortageVariables
+      ShellCheck.Prelude
       ShellCheck.Regex
     other-modules:
       Paths_ShellCheck
+    default-language: Haskell98
 
 executable shellcheck
     if impl(ghc < 8.0)
@@ -91,18 +107,21 @@ executable shellcheck
     build-depends:
       aeson,
       array,
-      base >= 4 && < 5,
+      base,
       bytestring,
       containers,
-      deepseq >= 1.4.0.0,
+      deepseq,
-      Diff >= 0.2.0,
+      Diff,
-      directory >= 1.2.3.0,
+      directory,
-      mtl >= 2.2.1,
+      fgl,
+      mtl,
       filepath,
-      parsec >= 3.0,
+      parsec,
-      QuickCheck >= 2.7.4,
+      QuickCheck,
       regex-tdfa,
+      transformers,
       ShellCheck
+    default-language: Haskell98
     main-is: shellcheck.hs
 
 test-suite test-shellcheck
@@ -110,17 +129,19 @@ test-suite test-shellcheck
     build-depends:
       aeson,
       array,
-      base >= 4 && < 5,
+      base,
       bytestring,
       containers,
-      deepseq >= 1.4.0.0,
+      deepseq,
-      Diff >= 0.2.0,
+      Diff,
-      directory >= 1.2.3.0,
+      directory,
-      mtl >= 2.2.1,
+      fgl,
       filepath,
+      mtl,
       parsec,
-      QuickCheck >= 2.7.4,
+      QuickCheck,
       regex-tdfa,
+      transformers,
       ShellCheck
+    default-language: Haskell98
     main-is: test/shellcheck.hs
-

build/darwin.x86_64/Dockerfile

@@ -1,5 +1,4 @@
-# DIGEST:sha256:fa32af4677e2860a1c5950bc8c360f309e2a87e2ddfed27b642fddf7a6093b76
+FROM liushuyu/osxcross@sha256:fa32af4677e2860a1c5950bc8c360f309e2a87e2ddfed27b642fddf7a6093b76
-FROM liushuyu/osxcross:latest
 
 ENV TARGET x86_64-apple-darwin18
 ENV TARGETNAME darwin.x86_64
@@ -7,22 +6,25 @@ ENV TARGETNAME darwin.x86_64
 # Build dependencies
 USER root
 ENV DEBIAN_FRONTEND noninteractive
-RUN apt-get update && apt-get install -y ghc automake autoconf llvm curl
+RUN sed -e 's/focal/kinetic/g' -i /etc/apt/sources.list
+RUN apt-get update
+RUN apt-get dist-upgrade -y
+RUN apt-get install -y ghc automake autoconf llvm curl alex happy
 
 # Build GHC
 WORKDIR /ghc
-RUN curl -L "https://downloads.haskell.org/~ghc/8.10.4/ghc-8.10.4-src.tar.xz" | tar xJ --strip-components=1
+RUN curl -L "https://downloads.haskell.org/~ghc/9.2.5/ghc-9.2.5-src.tar.xz" | tar xJ --strip-components=1
-RUN ./boot && ./configure --host x86_64-linux-gnu --build x86_64-linux-gnu --target "$TARGET"
+RUN ./configure --host x86_64-linux-gnu --build x86_64-linux-gnu --target "$TARGET"
 RUN cp mk/flavours/quick-cross.mk mk/build.mk && make -j "$(nproc)"
 RUN make install
-RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.2.0.0/cabal-install-3.2.0.0-x86_64-unknown-linux.tar.xz" | tar xJv -C /usr/local/bin
+RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.9.0.0/cabal-install-3.9-x86_64-linux-alpine.tar.xz" | tar xJv -C /usr/local/bin
 
 # Due to an apparent cabal bug, we specify our options directly to cabal
 # It won't reuse caches if ghc-options are specified in ~/.cabal/config
 ENV CABALOPTS "--with-ghc=$TARGET-ghc;--with-hc-pkg=$TARGET-ghc-pkg"
 
 # Prebuild the dependencies
-RUN cabal update && IFS=';' && cabal install $CABALOPTS --lib Diff-0.4.0 base-compat-0.11.2 base-orphans-0.8.4 dlist-1.0 hashable-1.3.0.0 indexed-traversable-0.1.1 integer-logarithms-1.0.3.1 primitive-0.7.1.0 regex-base-0.94.0.0 splitmix-0.1.0.3 tagged-0.8.6.1 th-abstraction-0.4.2.0 transformers-compat-0.6.6 base-compat-batteries-0.11.2 time-compat-1.9.5 unordered-containers-0.2.13.0 data-fix-0.3.1 vector-0.12.2.0 scientific-0.3.6.2 regex-tdfa-1.3.1.0 random-1.2.0 distributive-0.6.2.1 attoparsec-0.13.2.5 uuid-types-1.0.3 comonad-5.0.8 bifunctors-5.5.10 assoc-1.0.2 these-1.1.1.1 strict-0.4.0.1 aeson-1.5.5.1
+RUN cabal update && IFS=';' && cabal install --dependencies-only $CABALOPTS ShellCheck
 
 # Copy the build script
 COPY build /usr/bin

build/darwin.x86_64/build

@@ -2,7 +2,7 @@
 set -xe
 {
   tar xzv --strip-components=1
-  ./striptests
+  chmod +x striptests && ./striptests
   mkdir "$TARGETNAME"
   cabal update
   ( IFS=';'; cabal build $CABALOPTS )

build/linux.aarch64/Dockerfile

@@ -6,22 +6,28 @@ ENV TARGETNAME linux.aarch64
 # Build dependencies
 USER root
 ENV DEBIAN_FRONTEND noninteractive
-RUN apt-get update && apt-get install -y ghc automake autoconf build-essential llvm curl qemu-user-static gcc-$TARGET
+
+# These deps are from 20.04, because GHC's compiler/llvm support moves slowly
+RUN apt-get update && apt-get install -y llvm gcc-$TARGET
+
+# The rest are from 22.10
+RUN sed -e 's/focal/kinetic/g' -i /etc/apt/sources.list
+RUN apt-get update && apt-get install -y ghc alex happy automake autoconf build-essential curl qemu-user-static
 
 # Build GHC
 WORKDIR /ghc
-RUN curl -L "https://downloads.haskell.org/~ghc/8.10.4/ghc-8.10.4-src.tar.xz" | tar xJ --strip-components=1
+RUN curl -L "https://downloads.haskell.org/~ghc/9.2.5/ghc-9.2.5-src.tar.xz" | tar xJ --strip-components=1
 RUN ./boot && ./configure --host x86_64-linux-gnu --build x86_64-linux-gnu --target "$TARGET"
 RUN cp mk/flavours/quick-cross.mk mk/build.mk && make -j "$(nproc)"
 RUN make install
-RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.2.0.0/cabal-install-3.2.0.0-x86_64-unknown-linux.tar.xz" | tar xJv -C /usr/local/bin
+RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.9.0.0/cabal-install-3.9-x86_64-linux-alpine.tar.xz" | tar xJv -C /usr/local/bin
 
 # Due to an apparent cabal bug, we specify our options directly to cabal
 # It won't reuse caches if ghc-options are specified in ~/.cabal/config
-ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections;--with-ghc=$TARGET-ghc;--with-hc-pkg=$TARGET-ghc-pkg"
+ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections -optc-fPIC;--with-ghc=$TARGET-ghc;--with-hc-pkg=$TARGET-ghc-pkg"
 
 # Prebuild the dependencies
-RUN cabal update && IFS=';' && cabal install $CABALOPTS --lib Diff-0.4.0 base-compat-0.11.2 base-orphans-0.8.4 dlist-1.0 hashable-1.3.0.0 indexed-traversable-0.1.1 integer-logarithms-1.0.3.1 primitive-0.7.1.0 regex-base-0.94.0.0 splitmix-0.1.0.3 tagged-0.8.6.1 th-abstraction-0.4.2.0 transformers-compat-0.6.6 base-compat-batteries-0.11.2 time-compat-1.9.5 unordered-containers-0.2.13.0 data-fix-0.3.1 vector-0.12.2.0 scientific-0.3.6.2 regex-tdfa-1.3.1.0 random-1.2.0 distributive-0.6.2.1 attoparsec-0.13.2.5 uuid-types-1.0.3 comonad-5.0.8 bifunctors-5.5.10 assoc-1.0.2 these-1.1.1.1 strict-0.4.0.1 aeson-1.5.5.1
+RUN cabal update && IFS=';' && cabal install --dependencies-only $CABALOPTS ShellCheck
 
 # Copy the build script
 COPY build /usr/bin

build/linux.aarch64/build

@@ -2,7 +2,7 @@
 set -xe
 {
   tar xzv --strip-components=1
-  ./striptests
+  chmod +x striptests && ./striptests
   mkdir "$TARGETNAME"
   cabal update
   ( IFS=';'; cabal build $CABALOPTS --enable-executable-static )

build/linux.armv6hf/Dockerfile

@@ -51,7 +51,8 @@ RUN pirun apt-get install -y ghc cabal-install
 # Finally we can build the current dependencies. This takes hours.
 ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections;--gcc-options;-Os -Wl,--gc-sections -ffunction-sections -fdata-sections"
 RUN pirun cabal update
-RUN IFS=";" && pirun cabal install --lib $CABALOPTS Diff-0.4.0 base-compat-0.11.2 base-orphans-0.8.4 dlist-1.0 hashable-1.3.1.0 indexed-traversable-0.1.1 integer-logarithms-1.0.3.1 primitive-0.7.1.0 regex-base-0.94.0.1 splitmix-0.1.0.3 tagged-0.8.6.1 th-abstraction-0.4.2.0 transformers-compat-0.6.6 base-compat-batteries-0.11.2 time-compat-1.9.5 unordered-containers-0.2.13.0 data-fix-0.3.1 vector-0.12.2.0 scientific-0.3.6.2 regex-tdfa-1.3.1.0 random-1.2.0 distributive-0.6.2.1 attoparsec-0.13.2.5 uuid-types-1.0.4 comonad-5.0.8 bifunctors-5.5.10 assoc-1.0.2 these-1.1.1.1 strict-0.4.0.1 aeson-1.5.6.0
+RUN IFS=";" && pirun cabal install --dependencies-only $CABALOPTS ShellCheck
+RUN IFS=';' && pirun cabal install $CABALOPTS --lib fgl
 
 # Copy the build script
 WORKDIR /pi/scratch

build/linux.armv6hf/build

@@ -3,7 +3,7 @@ set -xe
 cd /scratch
 {
   tar xzv --strip-components=1
-  ./striptests
+  chmod +x striptests && ./striptests
   mkdir "$TARGETNAME"
   # This script does not cabal update because compiling anything new is slow
   ( IFS=';'; cabal build $CABALOPTS --enable-executable-static )

build/linux.ppc64le/Dockerfile

@@ -1,30 +0,0 @@
-FROM ubuntu:20.04
-
-ENV TARGET powerpc64le-linux-gnu
-ENV TARGETNAME linux.ppc64le
-
-# Build dependencies
-USER root
-ENV DEBIAN_FRONTEND noninteractive
-RUN apt-get update && apt-get install -y ghc automake autoconf build-essential llvm curl qemu-user-static gcc-$TARGET
-
-# Build GHC
-WORKDIR /ghc
-RUN curl -L "https://downloads.haskell.org/~ghc/8.10.4/ghc-8.10.4-src.tar.xz" | tar xJ --strip-components=1
-RUN ./boot && ./configure --host x86_64-linux-gnu --build x86_64-linux-gnu --target "$TARGET" --enable-unregisterised
-RUN cp mk/flavours/quick-cross.mk mk/build.mk && make -j "$(nproc)"
-RUN make install
-RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.2.0.0/cabal-install-3.2.0.0-x86_64-unknown-linux.tar.xz" | tar xJv -C /usr/local/bin
-
-# Due to an apparent cabal bug, we specify our options directly to cabal
-# It won't reuse caches if ghc-options are specified in ~/.cabal/config
-ENV CABALOPTS "--ghc-options;-optl-Wl,-fuse-ld=bfd -split-sections -optc-Os -optc-Wl,--gc-sections;--with-ghc=$TARGET-ghc;--with-hc-pkg=$TARGET-ghc-pkg"
-
-# Prebuild the dependencies
-#RUN cabal update && IFS=';' && cabal install $CABALOPTS --lib Diff-0.4.0 base-compat-0.11.2 base-orphans-0.8.4 dlist-1.0 hashable-1.3.0.0 indexed-traversable-0.1.1 integer-logarithms-1.0.3.1 primitive-0.7.1.0 regex-base-0.94.0.0 splitmix-0.1.0.3 tagged-0.8.6.1 th-abstraction-0.4.2.0 transformers-compat-0.6.6 base-compat-batteries-0.11.2 time-compat-1.9.5 unordered-containers-0.2.13.0 data-fix-0.3.1 vector-0.12.2.0 scientific-0.3.6.2 regex-tdfa-1.3.1.0 random-1.2.0 distributive-0.6.2.1 attoparsec-0.13.2.5 uuid-types-1.0.3 comonad-5.0.8 bifunctors-5.5.10 assoc-1.0.2 these-1.1.1.1 strict-0.4.0.1 aeson-1.5.5.1
-
-# Copy the build script
-COPY build /usr/bin
-
-WORKDIR /scratch
-ENTRYPOINT ["/usr/bin/build"]

build/linux.ppc64le/build

@@ -1,15 +0,0 @@
-#!/bin/sh
-set -xe
-{
-  tar xzv --strip-components=1
-  ./striptests
-  mkdir "$TARGETNAME"
-  cabal update
-  ( IFS=';'; cabal build $CABALOPTS --enable-executable-static )
-  find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \;
-  ls -l "$TARGETNAME"
-  "$TARGET-strip" -s "$TARGETNAME/shellcheck"
-  ls -l "$TARGETNAME"
-  qemu-ppc64le-static "$TARGETNAME/shellcheck" --version
-} >&2
-tar czv "$TARGETNAME"

build/linux.ppc64le/tag

@@ -1 +0,0 @@
-koalaman/scbuilder-linux-ppc64le

build/linux.x86_64/Dockerfile

@@ -1,16 +1,10 @@
-FROM ubuntu:20.04
+FROM alpine:latest
 
 ENV TARGETNAME linux.x86_64
 
 # Install GHC and cabal
 USER root
-ENV DEBIAN_FRONTEND noninteractive
+RUN apk add ghc cabal g++ libffi-dev curl bash
-RUN apt-get update && apt-get install -y ghc curl xz-utils
-
-# So we'd like a later version of Cabal that supports --enable-executable-static,
-# but we can't use Ubuntu 20.10 because coreutils has switched to new syscalls that
-# the TravisCI kernel doesn't support. Download it manually.
-RUN curl "https://downloads.haskell.org/~cabal/cabal-install-3.2.0.0/cabal-install-3.2.0.0-x86_64-unknown-linux.tar.xz" | tar xJv -C /usr/bin
 
 # Use ld.bfd instead of ld.gold due to
 # x86_64-linux-gnu/libpthread.a(pthread_cond_init.o)(.note.stapsdt+0x14): error:

build/linux.x86_64/build

@@ -2,7 +2,7 @@
 set -xe
 {
   tar xzv --strip-components=1
-  ./striptests
+  chmod +x striptests && ./striptests
   mkdir "$TARGETNAME"
   cabal update
   ( IFS=';'; cabal build $CABALOPTS --enable-executable-static )

build/windows.x86_64/Dockerfile

@@ -5,22 +5,22 @@ ENV TARGETNAME windows.x86_64
 # We don't need wine32, even though it complains
 USER root
 ENV DEBIAN_FRONTEND noninteractive
-RUN apt-get update && apt-get install -y curl busybox wine
+RUN apt-get update && apt-get install -y curl busybox wine winbind
 
 # Fetch Windows version, will be available under z:\haskell
 WORKDIR /haskell
 RUN curl -L "https://downloads.haskell.org/~ghc/8.10.4/ghc-8.10.4-x86_64-unknown-mingw32.tar.xz" | tar xJ --strip-components=1
 WORKDIR /haskell/bin
 RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.2.0.0/cabal-install-3.2.0.0-x86_64-unknown-mingw32.zip" | busybox unzip -
-RUN curl -L "https://curl.se/windows/dl-7.75.0/curl-7.75.0-win64-mingw.zip" | busybox unzip - && mv curl-7.75.0-win64-mingw/bin/* .
+RUN curl -L "https://curl.se/windows/dl-7.84.0/curl-7.84.0-win64-mingw.zip" | busybox unzip - && mv curl-7.84.0-win64-mingw/bin/* .
 ENV WINEPATH /haskell/bin
 
 # It's unknown whether Cabal on Windows suffers from the same issue
 # that necessitated this but I don't care enough to find out
 ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections"
 
-# Precompile some deps to speed up later builds. This list is just copied from `cabal build`
+# Precompile some deps to speed up later builds
-RUN wine /haskell/bin/cabal.exe update && IFS=';' && wine /haskell/bin/cabal.exe install $CABALOPTS --lib Diff-0.4.0 base-compat-0.11.2 base-orphans-0.8.4 dlist-1.0 hashable-1.3.0.0 indexed-traversable-0.1.1 integer-logarithms-1.0.3.1 primitive-0.7.1.0 regex-base-0.94.0.0 splitmix-0.1.0.3 tagged-0.8.6.1 th-abstraction-0.4.2.0 transformers-compat-0.6.6 base-compat-batteries-0.11.2 time-compat-1.9.5 unordered-containers-0.2.13.0 data-fix-0.3.1 vector-0.12.2.0 scientific-0.3.6.2 regex-tdfa-1.3.1.0 random-1.2.0 distributive-0.6.2.1 attoparsec-0.13.2.5 uuid-types-1.0.3 comonad-5.0.8 bifunctors-5.5.10 assoc-1.0.2 these-1.1.1.1 strict-0.4.0.1 aeson-1.5.5.1
+RUN wine /haskell/bin/cabal.exe update && IFS=';' && wine /haskell/bin/cabal.exe install --lib --dependencies-only $CABALOPTS ShellCheck
 
 COPY build /usr/bin
 WORKDIR /scratch

build/windows.x86_64/build

@@ -6,7 +6,7 @@ cabal() {
 set -xe
 {
   tar xzv --strip-components=1
-  ./striptests
+  chmod +x striptests && ./striptests
   mkdir "$TARGETNAME"
   cabal update
   ( IFS=';'; cabal build $CABALOPTS )

quickrun

@@ -2,4 +2,12 @@
 # quickrun runs ShellCheck in an interpreted mode.
 # This allows testing changes without recompiling.
 
-runghc -isrc -idist/build/autogen shellcheck.hs "$@"
+path=$(find . -type f -path './dist*/Paths_ShellCheck.hs' | sort | head -n 1)
+if [ -z "$path" ]
+then
+  echo >&2 "Unable to find Paths_ShellCheck.hs. Please 'cabal build' once."
+  exit 1
+fi
+path="${path%/*}"
+
+exec runghc -isrc -i"$path" shellcheck.hs "$@"

quicktest

@@ -3,8 +3,17 @@
 # This allows running tests without compiling, which can be faster.
 # 'cabal test' remains the source of truth.
 
+path=$(find . -type f -path './dist*/Paths_ShellCheck.hs' | sort | head -n 1)
+if [ -z "$path" ]
+then
+    echo >&2 "Unable to find Paths_ShellCheck.hs. Please 'cabal build' once."
+      exit 1
+fi
+path="${path%/*}"
+
+
 (
-var=$(echo 'main' | ghci test/shellcheck.hs 2>&1 | tee /dev/stderr)
+var=$(echo 'main' | ghci -isrc -i"$path" test/shellcheck.hs 2>&1 | tee /dev/stderr)
 if [[ $var == *ExitSuccess* ]]
 then
   exit 0

setgitversion

@@ -0,0 +1,11 @@
+#!/bin/sh -xe
+# This script hardcodes the `git describe` version as ShellCheck's version number.
+# This is done to allow shellcheck --version to differ from the cabal version when
+# building git snapshots.
+
+file="src/ShellCheck/Data.hs"
+test -e "$file"
+tmp=$(mktemp)
+version=$(git describe)
+sed -e "s/=.*VERSIONSTRING.*/= \"$version\" -- VERSIONSTRING, DO NOT SUBMIT/" "$file" > "$tmp"
+mv "$tmp" "$file"

shellcheck.1.md

@@ -87,8 +87,9 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts.
 
 :   Specify Bourne shell dialect. Valid values are *sh*, *bash*, *dash* and *ksh*.
     The default is to deduce the shell from the file's `shell` directive,
-    shebang, or `.bash/.bats/.dash/.ksh` extension, in that order. *sh* refers to
+    shebang, or `.bash/.bats/.dash/.ksh/.ebuild/.eclass` extension, in that
-    POSIX `sh` (not the system's), and will warn of portability issues.
+    order. *sh* refers to POSIX `sh` (not the system's), and will warn of
+    portability issues.
 
 **-S**\ *SEVERITY*,\ **--severity=***severity*
 
@@ -112,6 +113,9 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts.
     line (plus `/dev/null`). This option allows following any file the script
     may `source`.
 
+    This option may also be enabled using `external-sources=true` in
+    `.shellcheckrc`. This flag takes precedence.
+
 **FILES...**
 
 :   One or more script files to check, or "-" for standard input.
@@ -234,11 +238,20 @@ Valid keys are:
     The command can be a simple command like `echo foo`, or a compound command
     like a function definition, subshell block or loop. A range can be
     be specified with a dash, e.g. `disable=SC3000-SC4000` to exclude 3xxx.
+    All warnings can be disabled with `disable=all`.
 
 **enable**
 :   Enable an optional check by name, as listed with **--list-optional**.
     Only file-wide `enable` directives are considered.
 
+**external-sources**
+:   Set to `true` in `.shellcheckrc` to always allow ShellCheck to open
+    arbitrary files from 'source' statements (the way most tools do).
+
+    This option defaults to `false` only due to ShellCheck's origin as a
+    remote service for checking untrusted scripts. It can safely be enabled
+    for normal development.
+
 **source**
 :   Overrides the filename included by a `source`/`.` statement. This can be
     used to tell shellcheck where to look for a file whose name is determined
@@ -270,6 +283,12 @@ Here is an example `.shellcheckrc`:
     source-path=SCRIPTDIR
     source-path=/mnt/chroot
 
+    # Since 0.9.0, values can be quoted with '' or "" to allow spaces
+    source-path="My Documents/scripts"
+
+    # Allow opening any 'source'd file, even if not specified as input
+    external-sources=true
+
     # Turn on warnings for unquoted variables with safe values
     enable=quote-safe-variables
 
@@ -320,10 +339,32 @@ locales where encoding is unspecified (such as the `C` locale).
 Windows users seeing `commitBuffer: invalid argument (invalid character)`
 should set their terminal to use UTF-8 with `chcp 65001`.
 
-# AUTHORS
+# KNOWN INCOMPATIBILITIES
 
-ShellCheck is developed and maintained by Vidar Holen, with assistance from a
+(If nothing in this section makes sense, you are unlikely to be affected by it)
-long list of wonderful contributors.
+
+To avoid confusing and misguided suggestions, ShellCheck requires function
+bodies to be either `{ brace groups; }` or `( subshells )`, and function names
+containing `[]*=!` are only recognized after a `function` keyword.
+
+The following unconventional function definitions are identical in Bash,
+but ShellCheck only recognizes the latter.
+
+    [x!=y] () [[ $1 ]]
+    function [x!=y] () { [[ $1 ]]; }
+
+Shells without the `function` keyword do not allow these characters in function
+names to begin with.  Function names containing `{}` are not supported at all.
+
+Further, if ShellCheck sees `[x!=y]` it will assume this is an invalid
+comparison. To invoke the above function, quote the command as in `'[x!=y]'`,
+or to retain the same globbing behavior, use `command [x!=y]`.
+
+ShellCheck imposes additional restrictions on the `[` command to help diagnose
+common invalid uses. While `[ $x= 1 ]` is defined in POSIX, ShellCheck will
+assume it was intended as the much more likely comparison `[ "$x" = 1 ]` and
+fail accordingly. For unconventional or dynamic uses of the `[` command, use
+`test` or `\[` instead.
 
 # REPORTING BUGS
 
@@ -331,9 +372,14 @@ Bugs and issues can be reported on GitHub:
 
 https://github.com/koalaman/shellcheck/issues
 
+# AUTHORS
+
+ShellCheck is developed and maintained by Vidar Holen, with assistance from a
+long list of wonderful contributors.
+
 # COPYRIGHT
 
-Copyright 2012-2019, Vidar Holen and contributors.
+Copyright 2012-2022, Vidar Holen and contributors.
 Licensed under the GNU General Public License version 3 or later,
 see https://gnu.org/licenses/gpl.html
 

shellcheck.hs

@@ -21,6 +21,7 @@ import qualified ShellCheck.Analyzer
 import           ShellCheck.Checker
 import           ShellCheck.Data
 import           ShellCheck.Interface
+import           ShellCheck.PortageVariables
 import           ShellCheck.Regex
 
 import qualified ShellCheck.Formatter.CheckStyle
@@ -34,6 +35,8 @@ import qualified ShellCheck.Formatter.Quiet
 
 import           Control.Exception
 import           Control.Monad
+import           Control.Monad.IO.Class
+import           Control.Monad.Trans.Class
 import           Control.Monad.Except
 import           Data.Bits
 import           Data.Char
@@ -225,7 +228,7 @@ runFormatter sys format options files = do
     f :: Status -> FilePath -> IO Status
     f status file = do
         newStatus <- process file `catch` handler file
-        return $ status `mappend` newStatus
+        return $! status `mappend` newStatus
     handler :: FilePath -> IOException -> IO Status
     handler file e = reportFailure file (show e)
     reportFailure file str = do
@@ -234,7 +237,7 @@ runFormatter sys format options files = do
 
     process :: FilePath -> IO Status
     process filename = do
-        input <- siReadFile sys filename
+        input <- siReadFile sys Nothing filename
         either (reportFailure filename) check input
       where
         check contents = do
@@ -389,27 +392,30 @@ parseOption flag options =
             throwError SyntaxFailure
         return (Prelude.read num :: Integer)
 
+ioInterface :: Options -> [FilePath] -> IO (SystemInterface IO)
 ioInterface options files = do
     inputs <- mapM normalize files
     cache <- newIORef emptyCache
     configCache <- newIORef ("", Nothing)
-    return SystemInterface {
+    portageVars <- newIORef Nothing
+    return (newSystemInterface :: SystemInterface IO) {
         siReadFile = get cache inputs,
         siFindSource = findSourceFile inputs (sourcePaths options),
-        siGetConfig = getConfig configCache
+        siGetConfig = getConfig configCache,
+        siGetPortageVariables = getOrLoadPortage portageVars
     }
   where
     emptyCache :: Map.Map FilePath String
     emptyCache = Map.empty
 
-    get cache inputs file = do
+    get cache inputs rcSuggestsExternal file = do
         map <- readIORef cache
         case Map.lookup file map of
             Just x  -> return $ Right x
-            Nothing -> fetch cache inputs file
+            Nothing -> fetch cache inputs rcSuggestsExternal file
 
-    fetch cache inputs file = do
+    fetch cache inputs rcSuggestsExternal file = do
-        ok <- allowable inputs file
+        ok <- allowable rcSuggestsExternal inputs file
         if ok
           then (do
             (contents, shouldCache) <- inputFile file
@@ -417,13 +423,16 @@ ioInterface options files = do
                 modifyIORef cache $ Map.insert file contents
             return $ Right contents
             ) `catch` handler
+          else
+            if rcSuggestsExternal == Just False
+            then return $ Left (file ++ " was not specified as input, and external files were disabled via directive.")
             else return $ Left (file ++ " was not specified as input (see shellcheck -x).")
       where
         handler :: IOException -> IO (Either ErrorMessage String)
         handler ex = return . Left $ show ex
 
-    allowable inputs x =
+    allowable rcSuggestsExternal inputs x =
-        if externalSources options
+        if fromMaybe (externalSources options) rcSuggestsExternal
         then return True
         else do
             path <- normalize x
@@ -497,7 +506,7 @@ ioInterface options files = do
             b <- p x
             if b then pure (Just x) else acc
 
-    findSourceFile inputs sourcePathFlag currentScript sourcePathAnnotation original =
+    findSourceFile inputs sourcePathFlag currentScript rcSuggestsExternal sourcePathAnnotation original =
         if isAbsolute original
         then
             let (_, relative) = splitDrive original
@@ -506,7 +515,7 @@ ioInterface options files = do
             find original original
       where
         find filename deflt = do
-            sources <- findM ((allowable inputs) `andM` doesFileExist) $
+            sources <- findM ((allowable rcSuggestsExternal inputs) `andM` doesFileExist) $
                         (adjustPath filename):(map ((</> filename) . adjustPath) $ sourcePathFlag ++ sourcePathAnnotation)
             case sources of
                 Nothing -> return deflt
@@ -517,6 +526,21 @@ ioInterface options files = do
                 ("SCRIPTDIR":rest) -> joinPath (scriptdir:rest)
                 _ -> str
 
+    getOrLoadPortage cache = do
+        x <- readIORef cache
+        case x of
+            Just m -> do
+                return m
+            Nothing -> do
+                vars <- readPortageVariables `catch` handler
+                writeIORef cache $ Just vars
+                return vars
+      where
+        handler :: IOException -> IO (Map.Map String [String])
+        handler e = do
+            hPutStrLn stderr $ "Error finding portage repos, eclass definitions will be ignored: " ++ show e
+            return $ Map.empty
+
 inputFile file = do
     (handle, shouldCache) <-
             if file == "-"

src/ShellCheck/AST.hs

@@ -45,6 +45,7 @@ data InnerToken t =
     | Inner_TA_Variable String [t]
     | Inner_TA_Expansion [t]
     | Inner_TA_Sequence [t]
+    | Inner_TA_Parenthesis t
     | Inner_TA_Trinary t t t
     | Inner_TA_Unary String t
     | Inner_TC_And ConditionType String t t
@@ -141,7 +142,7 @@ data InnerToken t =
     | Inner_T_CoProcBody t
     | Inner_T_Include t
     | Inner_T_SourceCommand t t
-    | Inner_T_BatsTest t t
+    | Inner_T_BatsTest String t
     deriving (Show, Eq, Functor, Foldable, Traversable)
 
 data Annotation =
@@ -150,6 +151,7 @@ data Annotation =
     | SourceOverride String
     | ShellOverride String
     | SourcePath String
+    | ExternalSources Bool
     deriving (Show, Eq)
 data ConditionType = DoubleBracket | SingleBracket deriving (Show, Eq)
 
@@ -203,6 +205,7 @@ pattern T_Annotation id anns t = OuterToken id (Inner_T_Annotation anns t)
 pattern T_Arithmetic id c = OuterToken id (Inner_T_Arithmetic c)
 pattern T_Array id t = OuterToken id (Inner_T_Array t)
 pattern TA_Sequence id l = OuterToken id (Inner_TA_Sequence l)
+pattern TA_Parentesis id t = OuterToken id (Inner_TA_Parenthesis t)
 pattern T_Assignment id mode var indices value = OuterToken id (Inner_T_Assignment mode var indices value)
 pattern TA_Trinary id t1 t2 t3 = OuterToken id (Inner_TA_Trinary t1 t2 t3)
 pattern TA_Unary id op t1 = OuterToken id (Inner_TA_Unary op t1)
@@ -255,7 +258,7 @@ pattern T_Subshell id l = OuterToken id (Inner_T_Subshell l)
 pattern T_UntilExpression id c l = OuterToken id (Inner_T_UntilExpression c l)
 pattern T_WhileExpression id c l = OuterToken id (Inner_T_WhileExpression c l)
 
-{-# COMPLETE T_AND_IF, T_Bang, T_Case, TC_Empty, T_CLOBBER, T_DGREAT, T_DLESS, T_DLESSDASH, T_Do, T_DollarSingleQuoted, T_Done, T_DSEMI, T_Elif, T_Else, T_EOF, T_Esac, T_Fi, T_For, T_Glob, T_GREATAND, T_Greater, T_If, T_In, T_Lbrace, T_Less, T_LESSAND, T_LESSGREAT, T_Literal, T_Lparen, T_NEWLINE, T_OR_IF, T_ParamSubSpecialChar, T_Pipe, T_Rbrace, T_Rparen, T_Select, T_Semi, T_SingleQuoted, T_Then, T_UnparsedIndex, T_Until, T_While, TA_Assignment, TA_Binary, TA_Expansion, T_AndIf, T_Annotation, T_Arithmetic, T_Array, TA_Sequence, T_Assignment, TA_Trinary, TA_Unary, TA_Variable, T_Backgrounded, T_Backticked, T_Banged, T_BatsTest, T_BraceExpansion, T_BraceGroup, TC_And, T_CaseExpression, TC_Binary, TC_Group, TC_Nullary, T_Condition, T_CoProcBody, T_CoProc, TC_Or, TC_Unary, T_DollarArithmetic, T_DollarBraceCommandExpansion, T_DollarBraced, T_DollarBracket, T_DollarDoubleQuoted, T_DollarExpansion, T_DoubleQuoted, T_Extglob, T_FdRedirect, T_ForArithmetic, T_ForIn, T_Function, T_HereDoc, T_HereString, T_IfExpression, T_Include, T_IndexedElement, T_IoDuplicate, T_IoFile, T_NormalWord, T_OrIf, T_Pipeline, T_ProcSub, T_Redirecting, T_Script, T_SelectIn, T_SimpleCommand, T_SourceCommand, T_Subshell, T_UntilExpression, T_WhileExpression #-}
+{-# COMPLETE T_AND_IF, T_Bang, T_Case, TC_Empty, T_CLOBBER, T_DGREAT, T_DLESS, T_DLESSDASH, T_Do, T_DollarSingleQuoted, T_Done, T_DSEMI, T_Elif, T_Else, T_EOF, T_Esac, T_Fi, T_For, T_Glob, T_GREATAND, T_Greater, T_If, T_In, T_Lbrace, T_Less, T_LESSAND, T_LESSGREAT, T_Literal, T_Lparen, T_NEWLINE, T_OR_IF, T_ParamSubSpecialChar, T_Pipe, T_Rbrace, T_Rparen, T_Select, T_Semi, T_SingleQuoted, T_Then, T_UnparsedIndex, T_Until, T_While, TA_Assignment, TA_Binary, TA_Expansion, T_AndIf, T_Annotation, T_Arithmetic, T_Array, TA_Sequence, TA_Parentesis, T_Assignment, TA_Trinary, TA_Unary, TA_Variable, T_Backgrounded, T_Backticked, T_Banged, T_BatsTest, T_BraceExpansion, T_BraceGroup, TC_And, T_CaseExpression, TC_Binary, TC_Group, TC_Nullary, T_Condition, T_CoProcBody, T_CoProc, TC_Or, TC_Unary, T_DollarArithmetic, T_DollarBraceCommandExpansion, T_DollarBraced, T_DollarBracket, T_DollarDoubleQuoted, T_DollarExpansion, T_DoubleQuoted, T_Extglob, T_FdRedirect, T_ForArithmetic, T_ForIn, T_Function, T_HereDoc, T_HereString, T_IfExpression, T_Include, T_IndexedElement, T_IoDuplicate, T_IoFile, T_NormalWord, T_OrIf, T_Pipeline, T_ProcSub, T_Redirecting, T_Script, T_SelectIn, T_SimpleCommand, T_SourceCommand, T_Subshell, T_UntilExpression, T_WhileExpression #-}
 
 instance Eq Token where
     OuterToken _ a == OuterToken _ b = a == b

src/ShellCheck/ASTLib.hs

@@ -1,5 +1,5 @@
 {-
-    Copyright 2012-2019 Vidar Holen
+    Copyright 2012-2021 Vidar Holen
 
     This file is part of ShellCheck.
     https://www.shellcheck.net
@@ -17,9 +17,12 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 -}
+{-# LANGUAGE TemplateHaskell #-}
 module ShellCheck.ASTLib where
 
 import ShellCheck.AST
+import ShellCheck.Prelude
+import ShellCheck.Regex
 
 import Control.Monad.Writer
 import Control.Monad
@@ -31,7 +34,7 @@ import Data.Maybe
 import qualified Data.Map as Map
 import Numeric (showHex)
 
-arguments (T_SimpleCommand _ _ (cmd:args)) = args
+import Test.QuickCheck
 
 -- Is this a type of loop?
 isLoop t = case t of
@@ -55,10 +58,28 @@ willSplit x =
     T_NormalWord _ l -> any willSplit l
     _ -> False
 
-isGlob T_Extglob {} = True
+isGlob t = case t of
-isGlob T_Glob {} = True
+    T_Extglob {} -> True
-isGlob (T_NormalWord _ l) = any isGlob l
+    T_Glob {} -> True
-isGlob _ = False
+    T_NormalWord _ l -> any isGlob l || hasSplitRange l
+    _ -> False
+  where
+    -- foo[x${var}y] gets parsed as foo,[,x,$var,y],
+    -- so check if there's such an interval
+    hasSplitRange l =
+        let afterBracket = dropWhile (not . isHalfOpenRange) l
+        in any isClosingRange afterBracket
+
+    isHalfOpenRange t =
+        case t of
+            T_Literal _ "[" -> True
+            _ -> False
+
+    isClosingRange t =
+        case t of
+            T_Literal _ str -> ']' `elem` str
+            _ -> False
+
 
 -- Is this shell word a constant?
 isConstant token =
@@ -116,7 +137,7 @@ getFlagsUntil stopCondition (T_SimpleCommand _ _ (_:args)) =
     flag (x, '-':'-':arg) = [ (x, takeWhile (/= '=') arg) ]
     flag (x, '-':args) = map (\v -> (x, [v])) args
     flag (x, _) = [ (x, "") ]
-getFlagsUntil _ _ = error "Internal shellcheck error, please report! (getFlags on non-command)"
+getFlagsUntil _ _ = error $ pleaseReport "getFlags on non-command"
 
 -- Get all flags in a GNU way, up until --
 getAllFlags :: Token -> [(Token, String)]
@@ -224,6 +245,39 @@ getOpts (gnu, arbitraryLongOpts) string longopts args = process args
 
     listToArgs = map (\x -> ("", (x, x)))
 
+
+-- Generic getOpts that doesn't rely on a format string, but may also be inaccurate.
+-- This provides a best guess interpretation instead of failing when new options are added.
+--
+--    "--" is treated as end of arguments
+--    "--anything[=foo]" is treated as a long option without argument
+--    "-any" is treated as -a -n -y, with the next arg as an option to -y unless it starts with -
+--    anything else is an argument
+getGenericOpts :: [Token] -> [(String, (Token, Token))]
+getGenericOpts = process
+  where
+    process (token:rest) =
+        case getLiteralStringDef "\0" token of
+            "--" -> map (\c -> ("", (c,c))) rest
+            '-':'-':word -> (takeWhile (`notElem` "\0=") word, (token, token)) : process rest
+            '-':optString ->
+                let opts = takeWhile (/= '\0') optString
+                in
+                    case rest of
+                        next:_ | "-" `isPrefixOf` getLiteralStringDef "\0" next  ->
+                            map (\c -> ([c], (token, token))) opts ++ process rest
+                        next:remainder ->
+                            case reverse opts of
+                                last:initial ->
+                                    map (\c -> ([c], (token, token))) (reverse initial)
+                                        ++ [([last], (token, next))]
+                                        ++ process remainder
+                                [] -> process remainder
+                        [] -> map (\c -> ([c], (token, token))) opts
+            _ -> ("", (token, token)) : process rest
+    process [] = []
+
+
 -- Is this an expansion of multiple items of an array?
 isArrayExpansion (T_DollarBraced _ _ l) =
     let string = concat $ oversimplify l in
@@ -232,14 +286,14 @@ isArrayExpansion (T_DollarBraced _ _ l) =
 isArrayExpansion _ = False
 
 -- Is it possible that this arg becomes multiple args?
-mayBecomeMultipleArgs t = willBecomeMultipleArgs t || f t
+mayBecomeMultipleArgs t = willBecomeMultipleArgs t || f False t
   where
-    f (T_DollarBraced _ _ l) =
+    f quoted (T_DollarBraced _ _ l) =
         let string = concat $ oversimplify l in
-            "!" `isPrefixOf` string
+            not quoted || "!" `isPrefixOf` string
-    f (T_DoubleQuoted _ parts) = any f parts
+    f quoted (T_DoubleQuoted _ parts) = any (f True) parts
-    f (T_NormalWord _ parts) = any f parts
+    f quoted (T_NormalWord _ parts) = any (f quoted) parts
-    f _ = False
+    f _ _ = False
 
 -- Is it certain that this word will becomes multiple words?
 willBecomeMultipleArgs t = willConcatInAssignment t || f t
@@ -247,7 +301,6 @@ willBecomeMultipleArgs t = willConcatInAssignment t || f t
     f T_Extglob {} = True
     f T_Glob {} = True
     f T_BraceExpansion {} = True
-    f (T_DoubleQuoted _ parts) = any f parts
     f (T_NormalWord _ parts) = any f parts
     f _ = False
 
@@ -315,6 +368,21 @@ getGlobOrLiteralString = getLiteralStringExt f
     f (T_Glob _ str) = return str
     f _ = Nothing
 
+
+prop_getLiteralString1 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\x01") == Just "\1"
+prop_getLiteralString2 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\xyz") == Just "\\xyz"
+prop_getLiteralString3 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\x1") == Just "\x1"
+prop_getLiteralString4 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\x1y") == Just "\x1y"
+prop_getLiteralString5 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\xy") == Just "\\xy"
+prop_getLiteralString6 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\x") == Just "\\x"
+prop_getLiteralString7 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\1x") == Just "\1x"
+prop_getLiteralString8 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\12x") == Just "\o12x"
+prop_getLiteralString9 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\123x") == Just "\o123x"
+prop_getLiteralString10 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\1234") == Just "\o123\&4"
+prop_getLiteralString11 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\1") == Just "\1"
+prop_getLiteralString12 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\12") == Just "\o12"
+prop_getLiteralString13 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\123") == Just "\o123"
+
 -- Maybe get the literal value of a token, using a custom function
 -- to map unrecognized Tokens into strings.
 getLiteralStringExt :: Monad m => (Token -> m String) -> Token -> m String
@@ -347,14 +415,15 @@ getLiteralStringExt more = g
             '\\' -> '\\' : rest
             'x' ->
                 case cs of
-                    (x:y:more) ->
+                    (x:y:more) | isHexDigit x && isHexDigit y ->
-                        if isHexDigit x && isHexDigit y
+                        chr (16*(digitToInt x) + (digitToInt y)) : decodeEscapes more
-                        then chr (16*(digitToInt x) + (digitToInt y)) : rest
+                    (x:more) | isHexDigit x ->
-                        else '\\':c:rest
+                        chr (digitToInt x) : decodeEscapes more
+                    more -> '\\' : 'x' : decodeEscapes more
             _ | isOctDigit c ->
-                let digits = take 3 $ takeWhile isOctDigit (c:cs)
+                let (digits, more) = spanMax isOctDigit 3 (c:cs)
-                    num = parseOct digits
+                    num = (parseOct digits) `mod` 256
-                in (if num < 256 then chr num else '?') : rest
+                in (chr num) : decodeEscapes more
             _ -> '\\' : c : rest
       where
         rest = decodeEscapes cs
@@ -362,6 +431,11 @@ getLiteralStringExt more = g
           where
             f n "" = n
             f n (c:rest) = f (n * 8 + digitToInt c) rest
+        spanMax f n list =
+            let (first, second) = span f list
+                (prefix, suffix) = splitAt n first
+            in
+                (prefix, suffix ++ second)
     decodeEscapes (c:cs) = c : decodeEscapes cs
     decodeEscapes [] = []
 
@@ -470,17 +544,16 @@ getCommandNameAndToken direct t = fromMaybe (Nothing, t) $ do
                     return t
                 _ -> fail ""
 
--- If a command substitution is a single command, get its name.
+-- If a command substitution is a single SimpleCommand, return it.
---  $(date +%s) = Just "date"
+getSimpleCommandFromExpansion :: Token -> Maybe Token
-getCommandNameFromExpansion :: Token -> Maybe String
+getSimpleCommandFromExpansion t =
-getCommandNameFromExpansion t =
     case t of
         T_DollarExpansion _ [c] -> extract c
         T_Backticked _ [c] -> extract c
         T_DollarBraceCommandExpansion _ [c] -> extract c
         _ -> Nothing
   where
-    extract (T_Pipeline _ _ [cmd]) = getCommandName cmd
+    extract (T_Pipeline _ _ [c]) = getCommand c
     extract _ = Nothing
 
 -- Get the basename of a token representing a command
@@ -488,6 +561,10 @@ getCommandBasename = fmap basename . getCommandName
 
 basename = reverse . takeWhile (/= '/') . reverse
 
+-- Get the arguments to a command
+arguments (T_SimpleCommand _ _ (cmd:args)) = args
+arguments t = maybe [] arguments (getCommand t)
+
 isAssignment t =
     case t of
         T_Redirecting _ _ w -> isAssignment w
@@ -672,3 +749,184 @@ isAnnotationIgnoringCode code t =
   where
     hasNum (DisableComment from to) = code >= from && code < to
     hasNum _                   = False
+
+prop_executableFromShebang1 = executableFromShebang "/bin/sh" == "sh"
+prop_executableFromShebang2 = executableFromShebang "/bin/bash" == "bash"
+prop_executableFromShebang3 = executableFromShebang "/usr/bin/env ksh" == "ksh"
+prop_executableFromShebang4 = executableFromShebang "/usr/bin/env -S foo=bar bash -x" == "bash"
+prop_executableFromShebang5 = executableFromShebang "/usr/bin/env --split-string=bash -x" == "bash"
+prop_executableFromShebang6 = executableFromShebang "/usr/bin/env --split-string=foo=bar bash -x" == "bash"
+prop_executableFromShebang7 = executableFromShebang "/usr/bin/env --split-string bash -x" == "bash"
+prop_executableFromShebang8 = executableFromShebang "/usr/bin/env --split-string foo=bar bash -x" == "bash"
+prop_executableFromShebang9 = executableFromShebang "/usr/bin/env foo=bar dash" == "dash"
+prop_executableFromShebang10 = executableFromShebang "/bin/busybox sh" == "ash"
+prop_executableFromShebang11 = executableFromShebang "/bin/busybox ash" == "ash"
+
+-- Get the shell executable from a string like '/usr/bin/env bash'
+executableFromShebang :: String -> String
+executableFromShebang = shellFor
+  where
+    re = mkRegex "/env +(-S|--split-string=?)? *(.*)"
+    shellFor s | s `matches` re =
+        case matchRegex re s of
+            Just [flag, shell] -> fromEnvArgs (words shell)
+            _ -> ""
+    shellFor sb =
+        case words sb of
+            [] -> ""
+            [x] -> basename x
+            (first:second:args) | basename first == "busybox" ->
+                case basename second of
+                   "sh" -> "ash" -- busybox sh is ash
+                   x -> x
+            (first:args) | basename first == "env" ->
+                fromEnvArgs args
+            (first:_) -> basename first
+
+    fromEnvArgs args = fromMaybe "" $ find (notElem '=') $ skipFlags args
+    basename s = reverse . takeWhile (/= '/') . reverse $ s
+    skipFlags = dropWhile ("-" `isPrefixOf`)
+
+
+-- Determining if a name is a variable
+isVariableStartChar x = x == '_' || isAsciiLower x || isAsciiUpper x
+isVariableChar x = isVariableStartChar x || isDigit x
+isSpecialVariableChar = (`elem` "*@#?-$!")
+variableNameRegex = mkRegex "[_a-zA-Z][_a-zA-Z0-9]*"
+
+prop_isVariableName1 = isVariableName "_fo123"
+prop_isVariableName2 = not $ isVariableName "4"
+prop_isVariableName3 = not $ isVariableName "test: "
+isVariableName (x:r) = isVariableStartChar x && all isVariableChar r
+isVariableName _     = False
+
+
+-- Get the variable name from an expansion like ${var:-foo}
+prop_getBracedReference1 = getBracedReference "foo" == "foo"
+prop_getBracedReference2 = getBracedReference "#foo" == "foo"
+prop_getBracedReference3 = getBracedReference "#" == "#"
+prop_getBracedReference4 = getBracedReference "##" == "#"
+prop_getBracedReference5 = getBracedReference "#!" == "!"
+prop_getBracedReference6 = getBracedReference "!#" == "#"
+prop_getBracedReference7 = getBracedReference "!foo#?" == "foo"
+prop_getBracedReference8 = getBracedReference "foo-bar" == "foo"
+prop_getBracedReference9 = getBracedReference "foo:-bar" == "foo"
+prop_getBracedReference10 = getBracedReference "foo: -1" == "foo"
+prop_getBracedReference11 = getBracedReference "!os*" == ""
+prop_getBracedReference11b = getBracedReference "!os@" == ""
+prop_getBracedReference12 = getBracedReference "!os?bar**" == ""
+prop_getBracedReference13 = getBracedReference "foo[bar]" == "foo"
+getBracedReference s = fromMaybe s $
+    nameExpansion s `mplus` takeName noPrefix `mplus` getSpecial noPrefix `mplus` getSpecial s
+  where
+    noPrefix = dropPrefix s
+    dropPrefix (c:rest) | c `elem` "!#" = rest
+    dropPrefix cs = cs
+    takeName s = do
+        let name = takeWhile isVariableChar s
+        guard . not $ null name
+        return name
+    getSpecial (c:_) | isSpecialVariableChar c = return [c]
+    getSpecial _ = fail "empty or not special"
+
+    nameExpansion ('!':next:rest) = do -- e.g. ${!foo*bar*}
+        guard $ isVariableChar next -- e.g. ${!@}
+        first <- find (not . isVariableChar) rest
+        guard $ first `elem` "*?@"
+        return ""
+    nameExpansion _ = Nothing
+
+-- Get the variable modifier like /a/b in ${var/a/b}
+prop_getBracedModifier1 = getBracedModifier "foo:bar:baz" == ":bar:baz"
+prop_getBracedModifier2 = getBracedModifier "!var:-foo" == ":-foo"
+prop_getBracedModifier3 = getBracedModifier "foo[bar]" == "[bar]"
+prop_getBracedModifier4 = getBracedModifier "foo[@]@Q" == "[@]@Q"
+prop_getBracedModifier5 = getBracedModifier "@@Q" == "@Q"
+getBracedModifier s = headOrDefault "" $ do
+    let var = getBracedReference s
+    a <- dropModifier s
+    dropPrefix var a
+  where
+    dropPrefix [] t        = return t
+    dropPrefix (a:b) (c:d) | a == c = dropPrefix b d
+    dropPrefix _ _         = []
+
+    dropModifier (c:rest) | c `elem` "#!" = [rest, c:rest]
+    dropModifier x        = [x]
+
+-- Get the variables from indices like ["x", "y"] in ${var[x+y+1]}
+prop_getIndexReferences1 = getIndexReferences "var[x+y+1]" == ["x", "y"]
+getIndexReferences s = fromMaybe [] $ do
+    match <- matchRegex re s
+    index <- match !!! 0
+    return $ matchAllStrings variableNameRegex index
+  where
+    re = mkRegex "(\\[.*\\])"
+
+prop_getOffsetReferences1 = getOffsetReferences ":bar" == ["bar"]
+prop_getOffsetReferences2 = getOffsetReferences ":bar:baz" == ["bar", "baz"]
+prop_getOffsetReferences3 = getOffsetReferences "[foo]:bar" == ["bar"]
+prop_getOffsetReferences4 = getOffsetReferences "[foo]:bar:baz" == ["bar", "baz"]
+getOffsetReferences mods = fromMaybe [] $ do
+-- if mods start with [, then drop until ]
+    match <- matchRegex re mods
+    offsets <- match !!! 1
+    return $ matchAllStrings variableNameRegex offsets
+  where
+    re = mkRegex "^(\\[.+\\])? *:([^-=?+].*)"
+
+
+-- Returns whether a token is a parameter expansion without any modifiers.
+-- True for $var ${var} $1 $#
+-- False for ${#var} ${var[x]} ${var:-0}
+isUnmodifiedParameterExpansion t =
+    case t of
+        T_DollarBraced _ False _ -> True
+        T_DollarBraced _ _ list ->
+            let str = concat $ oversimplify list
+            in getBracedReference str == str
+        _ -> False
+
+-- Return the referenced variable if (and only if) it's an unmodified parameter expansion.
+getUnmodifiedParameterExpansion t =
+    case t of
+        T_DollarBraced _ _ list -> do
+            let str = concat $ oversimplify list
+            guard $ getBracedReference str == str
+            return str
+        _ -> Nothing
+
+--- A list of the element and all its parents up to the root node.
+getPath tree t = t :
+    case Map.lookup (getId t) tree of
+        Nothing     -> []
+        Just parent -> getPath tree parent
+
+isClosingFileOp op =
+    case op of
+        T_IoDuplicate _ (T_GREATAND _) "-" -> True
+        T_IoDuplicate _ (T_LESSAND  _) "-" -> True
+        _                                  -> False
+
+getEnableDirectives root =
+    case root of
+        T_Annotation _ list _ -> [s | EnableComment s <- list]
+        _ -> []
+
+
+commandExpansionShouldBeSplit t = do
+    cmd <- getSimpleCommandFromExpansion t
+    name <- getCommandName cmd
+    case () of
+        -- Should probably be split
+        _ | name `elem` ["seq", "pgrep"] -> return True
+        -- Portage macros that return a single word or nothing
+        _ | name `elem` ["usev", "use_with", "use_enable"] -> return True
+        -- Portage macros that are fine as long as the arguments have no spaces
+        _ | name `elem` ["usex", "meson_use", "meson_feature"] -> do
+            return . not $ any (' ' `elem`) $ map (getLiteralStringDef " ") $ arguments cmd
+        _ -> Nothing
+
+
+return []
+runTests = $quickCheckAll

src/ShellCheck/Analyzer.hs

@@ -1,5 +1,5 @@
 {-
-    Copyright 2012-2019 Vidar Holen
+    Copyright 2012-2022 Vidar Holen
 
     This file is part of ShellCheck.
     https://www.shellcheck.net
@@ -25,28 +25,31 @@ import ShellCheck.Interface
 import Data.List
 import Data.Monoid
 import qualified ShellCheck.Checks.Commands
+import qualified ShellCheck.Checks.ControlFlow
 import qualified ShellCheck.Checks.Custom
 import qualified ShellCheck.Checks.ShellSupport
 
 
 -- TODO: Clean up the cruft this is layered on
-analyzeScript :: AnalysisSpec -> AnalysisResult
+analyzeScript :: Monad m => SystemInterface m -> AnalysisSpec -> m AnalysisResult
-analyzeScript spec = newAnalysisResult {
+analyzeScript sys spec = do
+    params <- makeParameters sys spec
+    return $ newAnalysisResult {
             arComments =
                 filterByAnnotation spec params . nub $
-            runAnalytics spec
+                    runChecker params (checkers spec params)
-            ++ runChecker params (checkers spec params)
         }
-  where
-    params = makeParameters spec
 
 checkers spec params = mconcat $ map ($ params) [
+    ShellCheck.Analytics.checker spec,
     ShellCheck.Checks.Commands.checker spec,
+    ShellCheck.Checks.ControlFlow.checker spec,
     ShellCheck.Checks.Custom.checker,
     ShellCheck.Checks.ShellSupport.checker
     ]
 
 optionalChecks = mconcat $ [
     ShellCheck.Analytics.optionalChecks,
-    ShellCheck.Checks.Commands.optionalChecks
+    ShellCheck.Checks.Commands.optionalChecks,
+    ShellCheck.Checks.ControlFlow.optionalChecks
     ]

src/ShellCheck/AnalyzerLib.hs

@@ -1,5 +1,5 @@
 {-
-    Copyright 2012-2019 Vidar Holen
+    Copyright 2012-2022 Vidar Holen
 
     This file is part of ShellCheck.
     https://www.shellcheck.net
@@ -23,13 +23,16 @@ module ShellCheck.AnalyzerLib where
 
 import ShellCheck.AST
 import ShellCheck.ASTLib
+import qualified ShellCheck.CFGAnalysis as CF
 import ShellCheck.Data
 import ShellCheck.Interface
 import ShellCheck.Parser
+import ShellCheck.Prelude
 import ShellCheck.Regex
 
 import Control.Arrow (first)
 import Control.DeepSeq
+import Control.Monad
 import Control.Monad.Identity
 import Control.Monad.RWS
 import Control.Monad.State
@@ -79,10 +82,18 @@ composeAnalyzers f g x = f x >> g x
 data Parameters = Parameters {
     -- Whether this script has the 'lastpipe' option set/default.
     hasLastpipe        :: Bool,
+    -- Whether this script has the 'inherit_errexit' option set/default.
+    hasInheritErrexit  :: Bool,
     -- Whether this script has 'set -e' anywhere.
     hasSetE            :: Bool,
+    -- Whether this script has 'set -o pipefail' anywhere.
+    hasPipefail        :: Bool,
+    -- Whether this script is an Ebuild file.
+    isPortage          :: Bool,
     -- A linear (bad) analysis of data flow
     variableFlow       :: [StackData],
+    -- A map from Id to Token
+    idMap              :: Map.Map Id Token,
     -- A map from Id to parent Token
     parentMap          :: Map.Map Id Token,
     -- The shell type, such as Bash or Ksh
@@ -92,9 +103,14 @@ data Parameters = Parameters {
     -- The root node of the AST
     rootNode           :: Token,
     -- map from token id to start and end position
-    tokenPositions     :: Map.Map Id (Position, Position)
+    tokenPositions     :: Map.Map Id (Position, Position),
+    -- Result from Control Flow Graph analysis (including data flow analysis)
+    cfgAnalysis :: CF.CFGAnalysis,
+    -- A set of additional variables known to be set (TODO: make this more general?)
+    additionalKnownVariables :: [String]
     } deriving (Show)
 
+
 -- TODO: Cache results of common AST ops here
 data Cache = Cache {}
 
@@ -141,8 +157,8 @@ producesComments c s = do
         let pr = pScript s
         prRoot pr
         let spec = defaultSpec pr
-        let params = makeParameters spec
+        let params = runIdentity $ makeParameters (mockedSystemInterface []) spec
-        return . not . null $ runChecker params c
+        return . not . null $ filterByAnnotation spec params $ runChecker params c
 
 makeComment :: Severity -> Id -> Code -> String -> TokenComment
 makeComment severity id code note =
@@ -167,6 +183,8 @@ errWithFix :: MonadWriter [TokenComment] m => Id -> Code -> String -> Fix -> m (
 errWithFix  = addCommentWithFix ErrorC
 warnWithFix :: MonadWriter [TokenComment] m => Id -> Code -> String -> Fix -> m ()
 warnWithFix  = addCommentWithFix WarningC
+infoWithFix :: MonadWriter [TokenComment] m => Id -> Code -> String -> Fix -> m ()
+infoWithFix = addCommentWithFix InfoC
 styleWithFix :: MonadWriter [TokenComment] m => Id -> Code -> String -> Fix -> m ()
 styleWithFix = addCommentWithFix StyleC
 
@@ -178,28 +196,64 @@ makeCommentWithFix :: Severity -> Id -> Code -> String -> Fix -> TokenComment
 makeCommentWithFix severity id code str fix =
     let comment = makeComment severity id code str
         withFix = comment {
-            tcFix = Just fix
+            -- If fix is empty, pretend it wasn't there.
+            tcFix = if null (fixReplacements fix) then Nothing else Just fix
         }
     in force withFix
 
-makeParameters spec =
+makeParameters :: Monad m => SystemInterface m -> AnalysisSpec -> m Parameters
-    let params = Parameters {
+makeParameters sys spec = do
+    extraVars <-
+        if asIsPortage spec
+        then do
+            vars <- siGetPortageVariables sys
+            let classes = getInheritedEclasses root
+            return $ concatMap (\c -> Map.findWithDefault [] c vars) classes
+        else
+            return []
+
+    return $ makeParams extraVars
+  where
+    shell = fromMaybe (determineShell (asFallbackShell spec) root) $ asShellType spec
+    makeParams extraVars = params
+      where
+        params = Parameters {
             rootNode = root,
-        shellType = fromMaybe (determineShell (asFallbackShell spec) root) $ asShellType spec,
+            shellType = shell,
             hasSetE = containsSetE root,
             hasLastpipe =
                 case shellType params of
-                Bash -> containsLastpipe root
+                    Bash -> isOptionSet "lastpipe" root
                     Dash -> False
                     Sh   -> False
                     Ksh  -> True,
-
+            hasInheritErrexit =
+                case shellType params of
+                    Bash -> isOptionSet "inherit_errexit" root
+                    Dash -> True
+                    Sh   -> True
+                    Ksh  -> False,
+            hasPipefail =
+                case shellType params of
+                    Bash -> isOptionSet "pipefail" root
+                    Dash -> True
+                    Sh   -> True
+                    Ksh  -> isOptionSet "pipefail" root,
             shellTypeSpecified = isJust (asShellType spec) || isJust (asFallbackShell spec),
+            isPortage = asIsPortage spec,
+            idMap = getTokenMap root,
             parentMap = getParentTree root,
             variableFlow = getVariableFlow params root,
-        tokenPositions = asTokenPositions spec
+            tokenPositions = asTokenPositions spec,
-    } in params
+            cfgAnalysis = CF.analyzeControlFlow cfParams root,
-  where root = asScript spec
+            additionalKnownVariables = extraVars
+        }
+        cfParams = CF.CFGParameters {
+            CF.cfLastpipe = hasLastpipe params,
+            CF.cfPipefail = hasPipefail params,
+            CF.cfAdditionalInitialVariables = additionalKnownVariables params
+        }
+    root = asScript spec
 
 
 -- Does this script mention 'set -e' anywhere?
@@ -216,18 +270,30 @@ containsSetE root = isNothing $ doAnalysis (guard . not . isSetE) root
             _ -> False
     re = mkRegex "[[:space:]]-[^-]*e"
 
--- Does this script mention 'shopt -s lastpipe' anywhere?
+
--- Also used as a hack.
+containsSetOption opt root = isNothing $ doAnalysis (guard . not . isPipefail) root
-containsLastpipe root =
+  where
+    isPipefail t =
+        case t of
+            T_SimpleCommand {}  ->
+                t `isUnqualifiedCommand` "set" &&
+                    (opt `elem` oversimplify t ||
+                        "o" `elem` map snd (getAllFlags t))
+            _ -> False
+
+containsShopt shopt root =
         isNothing $ doAnalysis (guard . not . isShoptLastPipe) root
     where
         isShoptLastPipe t =
             case t of
                 T_SimpleCommand {}  ->
                     t `isUnqualifiedCommand` "shopt" &&
-                        ("lastpipe" `elem` oversimplify t)
+                        (shopt `elem` oversimplify t)
                 _ -> False
 
+-- Does this script mention 'shopt -s $opt' or 'set -o $opt' anywhere?
+isOptionSet opt root = containsShopt opt root || containsSetOption opt root
+
 
 prop_determineShell0 = determineShellTest "#!/bin/sh" == Sh
 prop_determineShell1 = determineShellTest "#!/usr/bin/env ksh" == Ksh
@@ -240,6 +306,8 @@ prop_determineShell7 = determineShellTest "#! /bin/ash" == Dash
 prop_determineShell8 = determineShellTest' (Just Ksh) "#!/bin/sh" == Sh
 prop_determineShell9 = determineShellTest "#!/bin/env -S dash -x" == Dash
 prop_determineShell10 = determineShellTest "#!/bin/env --split-string= dash -x" == Dash
+prop_determineShell11 = determineShellTest "#!/bin/busybox sh" == Dash -- busybox sh is a specific shell, not posix sh
+prop_determineShell12 = determineShellTest "#!/bin/busybox ash" == Dash
 
 determineShellTest = determineShellTest' Nothing
 determineShellTest' fallbackShell = determineShell fallbackShell . fromJust . prRoot . pScript
@@ -253,19 +321,6 @@ determineShell fallbackShell t = fromMaybe Bash $
         headOrDefault (fromShebang s) [s | ShellOverride s <- annotations]
     fromShebang (T_Script _ (T_Literal _ s) _) = executableFromShebang s
 
--- Given a string like "/bin/bash" or "/usr/bin/env dash",
--- return the shell basename like "bash" or "dash"
-executableFromShebang :: String -> String
-executableFromShebang = shellFor
-  where
-    shellFor s | "/env " `isInfixOf` s = case matchRegex re s of
-        Just [flag, shell] -> shell
-        _ -> ""
-    shellFor s | ' ' `elem` s = shellFor $ takeWhile (/= ' ') s
-    shellFor s = reverse . takeWhile (/= '/') . reverse $ s
-    re = mkRegex "/env +(-S|--split-string=?)? *([^ ]*)"
-
-
 -- Given a root node, make a map from Id to parent Token.
 -- This is used to populate parentMap in Parameters
 getParentTree :: Token -> Map.Map Id Token
@@ -298,14 +353,14 @@ isStrictlyQuoteFree = isQuoteFreeNode True
 isQuoteFree = isQuoteFreeNode False
 
 
-isQuoteFreeNode strict tree t =
+isQuoteFreeNode strict shell tree t =
     isQuoteFreeElement t ||
-        headOrDefault False (mapMaybe isQuoteFreeContext (drop 1 $ getPath tree t))
+        (fromMaybe False $ msum $ map isQuoteFreeContext $ drop 1 $ getPath tree t)
   where
     -- Is this node self-quoting in itself?
     isQuoteFreeElement t =
         case t of
-            T_Assignment {} -> True
+            T_Assignment {} -> assignmentIsQuoting t
             T_FdRedirect {} -> True
             _               -> False
 
@@ -317,7 +372,7 @@ isQuoteFreeNode strict tree t =
             TC_Binary _ DoubleBracket _ _ _ -> return True
             TA_Sequence {}                  -> return True
             T_Arithmetic {}                 -> return True
-            T_Assignment {}                 -> return True
+            T_Assignment {}                 -> return $ assignmentIsQuoting t
             T_Redirecting {}                -> return False
             T_DoubleQuoted _ _              -> return True
             T_DollarDoubleQuoted _ _        -> return True
@@ -329,6 +384,18 @@ isQuoteFreeNode strict tree t =
             T_SelectIn {}                   -> return (not strict)
             _                               -> Nothing
 
+    -- Check whether this assignment is self-quoting due to being a recognized
+    -- assignment passed to a Declaration Utility. This will soon be required
+    -- by POSIX: https://austingroupbugs.net/view.php?id=351
+    assignmentIsQuoting t = shellParsesParamsAsAssignments || not (isAssignmentParamToCommand t)
+    shellParsesParamsAsAssignments = shell /= Sh
+
+    -- Is this assignment a parameter to a command like export/typeset/etc?
+    isAssignmentParamToCommand (T_Assignment id _ _ _ _) =
+        case Map.lookup id tree of
+            Just (T_SimpleCommand _ _ (_:args)) -> id `elem` (map getId args)
+            _ -> False
+
 -- Check if a token is a parameter to a certain command by name:
 -- Example: isParamTo (parentMap params) "sed" t
 isParamTo :: Map.Map Id Token -> String -> Token -> Bool
@@ -374,12 +441,6 @@ usedAsCommandName tree token = go (getId token) (tail $ getPath tree token)
         getId word == currentId || getId (getCommandTokenOrThis t) == currentId
     go _ _ = False
 
--- A list of the element and all its parents up to the root node.
-getPath tree t = t :
-    case Map.lookup (getId t) tree of
-        Nothing     -> []
-        Just parent -> getPath tree parent
-
 -- Version of the above taking the map from the current context
 -- Todo: give this the name "getPath"
 getPathM t = do
@@ -477,13 +538,11 @@ getModifiedVariables t =
         T_SimpleCommand {} ->
             getModifiedVariableCommand t
 
-        TA_Unary _ "++|" v@(TA_Variable _ name _)  ->
+        TA_Unary _ op v@(TA_Variable _ name _) | "--" `isInfixOf` op || "++" `isInfixOf` op ->
-            [(t, v, name, DataString $ SourceFrom [v])]
+            [(t, v, name, DataString SourceInteger)]
-        TA_Unary _ "|++" v@(TA_Variable _ name _)  ->
-            [(t, v, name, DataString $ SourceFrom [v])]
         TA_Assignment _ op (TA_Variable _ name _) rhs -> do
             guard $ op `elem` ["=", "*=", "/=", "%=", "+=", "-=", "<<=", ">>=", "&=", "^=", "|="]
-            return (t, t, name, DataString $ SourceFrom [rhs])
+            return (t, t, name, DataString SourceInteger)
 
         T_BatsTest {} -> [
             (t, t, "lines", DataArray SourceExternal),
@@ -493,14 +552,8 @@ getModifiedVariables t =
 
         -- Count [[ -v foo ]] as an "assignment".
         -- This is to prevent [ -v foo ] being unassigned or unused.
-        TC_Unary id _ "-v" token -> do
+        TC_Unary id _ "-v" token -> maybeToList $ do
-            str <- fmap (takeWhile (/= '[')) $ -- Quoted index
+            str <- getVariableForTestDashV token
-                    flip getLiteralStringExt token $ \x ->
-                case x of
-                    T_Glob _ s -> return s -- Unquoted index
-                    _          -> []
-
-            guard . not . null $ str
             return (t, token, str, DataString SourceChecked)
 
         TC_Unary _ _ "-n" token -> markAsChecked t token
@@ -533,24 +586,16 @@ getModifiedVariables t =
                     return (place, t, str, DataString SourceChecked)
             _ -> Nothing
 
-isClosingFileOp op =
-    case op of
-        T_IoDuplicate _ (T_GREATAND _) "-" -> True
-        T_IoDuplicate _ (T_LESSAND  _) "-" -> True
-        _                                  -> False
-
 
 -- Consider 'export/declare -x' a reference, since it makes the var available
 getReferencedVariableCommand base@(T_SimpleCommand _ _ (T_NormalWord _ (T_Literal _ x:_):rest)) =
     case x of
+        "declare" -> forDeclare
+        "typeset" -> forDeclare
+
         "export" -> if "f" `elem` flags
             then []
             else concatMap getReference rest
-        "declare" -> if
-                any (`elem` flags) ["x", "p"] &&
-                    (not $ any (`elem` flags) ["f", "F"])
-            then concatMap getReference rest
-            else []
         "local" -> if "x" `elem` flags
             then concatMap getReference rest
             else []
@@ -559,8 +604,26 @@ getReferencedVariableCommand base@(T_SimpleCommand _ _ (T_NormalWord _ (T_Litera
                 head:_ -> map (\x -> (base, head, x)) $ getVariablesFromLiteralToken head
                 _ -> []
         "alias" -> [(base, token, name) | token <- rest, name <- getVariablesFromLiteralToken token]
+
+        -- tc-export makes a list of toolchain variables available, similar to export.
+        -- Usage tc-export CC CXX
+        "tc-export" -> concatMap getReference rest
+
+        -- tc-export_build_env exports the listed variables plus a bunch of BUILD_XX variables.
+        -- Usage tc-export_build_env BUILD_CC
+        "tc-export_build_env" ->
+            concatMap getReference rest
+                ++ [ (base, base, v) | v <- portageBuildEnvVariables ]
+
         _ -> []
   where
+    forDeclare =
+            if
+                any (`elem` flags) ["x", "p"] &&
+                    (not $ any (`elem` flags) ["f", "F"])
+            then concatMap getReference rest
+            else []
+
     getReference t@(T_Assignment _ _ name _ value) = [(t, t, name)]
     getReference t@(T_NormalWord _ [T_Literal _ name]) | not ("-" `isPrefixOf` name) = [(t, t, name)]
     getReference _ = []
@@ -600,8 +663,8 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T
         "export" ->
             if "f" `elem` flags then [] else concatMap getModifierParamString rest
 
-        "declare" -> if any (`elem` flags) ["F", "f", "p"] then [] else declaredVars
+        "declare" -> forDeclare
-        "typeset" -> declaredVars
+        "typeset" -> forDeclare
 
         "local" -> concatMap getModifierParamString rest
         "readonly" ->
@@ -613,6 +676,7 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T
             return (base, base, "@", DataString $ SourceFrom params)
 
         "printf" -> maybeToList $ getPrintfVariable rest
+        "wait" ->   maybeToList $ getWaitVariable rest
 
         "mapfile" -> maybeToList $ getMapfileArray base rest
         "readarray" -> maybeToList $ getMapfileArray base rest
@@ -622,6 +686,16 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T
         "DEFINE_integer" -> maybeToList $ getFlagVariable rest
         "DEFINE_string" -> maybeToList $ getFlagVariable rest
 
+        "tc-export" -> concatMap getModifierParamString rest
+
+        -- tc-export_build_env exports the listed variables plus a bunch of BUILD_XX variables.
+        -- Usage tc-export_build_env BUILD_CC
+        "tc-export_build_env" ->
+            concatMap getModifierParamString rest
+                ++ [ (base, base, var, DataString $ SourceExternal) |
+                        var <- ["BUILD_" ++ x, x ++ "_FOR_BUILD" ],
+                        x <- portageBuildEnvVariables ]
+
         _ -> []
   where
     flags = map snd $ getAllFlags base
@@ -632,6 +706,8 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T
         T_NormalWord id1 [T_DoubleQuoted id2 [T_Literal id3 (stripEquals s)]]
     stripEqualsFrom t = t
 
+    forDeclare = if any (`elem` flags) ["F", "f", "p"] then [] else declaredVars
+
     declaredVars = concatMap (getModifierParam defaultType) rest
       where
         defaultType = if any (`elem` flags) ["a", "A"] then DataArray else DataString
@@ -670,15 +746,15 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T
                 _            -> return (t:fromMaybe [] (getSetParams rest))
     getSetParams [] = Nothing
 
-    getPrintfVariable list = f $ map (\x -> (x, getLiteralString x)) list
+    getPrintfVariable list = getFlagAssignedVariable "v" (SourceFrom list) $ getBsdOpts "v:" list
-      where
+    getWaitVariable   list = getFlagAssignedVariable "p" SourceInteger     $ return $ getGenericOpts list
-        f ((_, Just "-v") : (t, Just var) : _) = return (base, t, varName, varType $ SourceFrom list)
+
-            where
+    getFlagAssignedVariable str dataSource maybeFlags = do
-                (varName, varType) = case elemIndex '[' var of
+        flags <- maybeFlags
-                    Just i -> (take i var, DataArray)
+        (_, (flag, value)) <- find ((== str) . fst) flags
-                    Nothing -> (var, DataString)
+        variableName <- getLiteralStringExt (const $ return "!") value
-        f (_:rest) = f rest
+        let (baseName, index) = span (/= '[') variableName
-        f [] = fail "not found"
+        return (base, value, baseName, (if null index then DataString else DataArray) dataSource)
 
     -- mapfile has some curious syntax allowing flags plus 0..n variable names
     -- where only the first non-option one is used if any.
@@ -712,24 +788,19 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T
 
 getModifiedVariableCommand _ = []
 
-getIndexReferences s = fromMaybe [] $ do
+-- Given a NormalWord like foo or foo[$bar], get foo.
-    match <- matchRegex re s
+-- Primarily used to get references for [[ -v foo[bar] ]]
-    index <- match !!! 0
+getVariableForTestDashV :: Token -> Maybe String
-    return $ matchAllStrings variableNameRegex index
+getVariableForTestDashV t = do
+    str <- takeWhile ('[' /=) <$> getLiteralStringExt toStr t
+    guard $ isVariableName str
+    return str
   where
-    re = mkRegex "(\\[.*\\])"
+    -- foo[bar] gets parsed with [bar] as a glob, so undo that
-
+    toStr (T_Glob _ s) = return s
-prop_getOffsetReferences1 = getOffsetReferences ":bar" == ["bar"]
+    -- Turn foo[$x] into foo[\0] so that we can get the constant array name
-prop_getOffsetReferences2 = getOffsetReferences ":bar:baz" == ["bar", "baz"]
+    -- in a non-constant expression (while filtering out foo$x[$y])
-prop_getOffsetReferences3 = getOffsetReferences "[foo]:bar" == ["bar"]
+    toStr _ = return "\0"
-prop_getOffsetReferences4 = getOffsetReferences "[foo]:bar:baz" == ["bar", "baz"]
-getOffsetReferences mods = fromMaybe [] $ do
--- if mods start with [, then drop until ]
-    match <- matchRegex re mods
-    offsets <- match !!! 1
-    return $ matchAllStrings variableNameRegex offsets
-  where
-    re = mkRegex "^(\\[.+\\])? *:([^-=?+].*)"
 
 getReferencedVariables parents t =
     case t of
@@ -748,7 +819,7 @@ getReferencedVariables parents t =
         TC_Unary id _ "-v" token -> getIfReference t token
         TC_Unary id _ "-R" token -> getIfReference t token
         TC_Binary id DoubleBracket op lhs rhs ->
-            if isDereferencing op
+            if isDereferencingBinaryOp op
             then concatMap (getIfReference t) [lhs, rhs]
             else []
 
@@ -777,17 +848,16 @@ getReferencedVariables parents t =
         T_Glob _ s -> return s    -- Also when parsed as globs
         _          -> []
 
-    getIfReference context token = do
+    getIfReference context token = maybeToList $ do
-            str@(h:_) <- getLiteralStringExt literalizer token
+            str <- getVariableForTestDashV token
-            when (isDigit h) $ fail "is a number"
             return (context, token, getBracedReference str)
 
-    isDereferencing = (`elem` ["-eq", "-ne", "-lt", "-le", "-gt", "-ge"])
-
     isArithmeticAssignment t = case getPath parents t of
         this: TA_Assignment _ "=" lhs _ :_ -> lhs == t
         _                                  -> False
 
+isDereferencingBinaryOp = (`elem` ["-eq", "-ne", "-lt", "-le", "-gt", "-ge"])
+
 dataTypeFrom defaultType v = (case v of T_Array {} -> DataArray; _ -> defaultType) $ SourceFrom [v]
 
 
@@ -810,16 +880,6 @@ isConfusedGlobRegex ('*':_) = True
 isConfusedGlobRegex [x,'*'] | x `notElem` "\\." = True
 isConfusedGlobRegex _       = False
 
-isVariableStartChar x = x == '_' || isAsciiLower x || isAsciiUpper x
-isVariableChar x = isVariableStartChar x || isDigit x
-variableNameRegex = mkRegex "[_a-zA-Z][_a-zA-Z0-9]*"
-
-prop_isVariableName1 = isVariableName "_fo123"
-prop_isVariableName2 = not $ isVariableName "4"
-prop_isVariableName3 = not $ isVariableName "test: "
-isVariableName (x:r) = isVariableStartChar x && all isVariableChar r
-isVariableName _     = False
-
 getVariablesFromLiteralToken token =
     getVariablesFromLiteral (getLiteralStringDef " " token)
 
@@ -832,67 +892,6 @@ getVariablesFromLiteral string =
   where
     variableRegex = mkRegex "\\$\\{?([A-Za-z0-9_]+)"
 
--- Get the variable name from an expansion like ${var:-foo}
-prop_getBracedReference1 = getBracedReference "foo" == "foo"
-prop_getBracedReference2 = getBracedReference "#foo" == "foo"
-prop_getBracedReference3 = getBracedReference "#" == "#"
-prop_getBracedReference4 = getBracedReference "##" == "#"
-prop_getBracedReference5 = getBracedReference "#!" == "!"
-prop_getBracedReference6 = getBracedReference "!#" == "#"
-prop_getBracedReference7 = getBracedReference "!foo#?" == "foo"
-prop_getBracedReference8 = getBracedReference "foo-bar" == "foo"
-prop_getBracedReference9 = getBracedReference "foo:-bar" == "foo"
-prop_getBracedReference10= getBracedReference "foo: -1" == "foo"
-prop_getBracedReference11= getBracedReference "!os*" == ""
-prop_getBracedReference11b= getBracedReference "!os@" == ""
-prop_getBracedReference12= getBracedReference "!os?bar**" == ""
-prop_getBracedReference13= getBracedReference "foo[bar]" == "foo"
-getBracedReference s = fromMaybe s $
-    nameExpansion s `mplus` takeName noPrefix `mplus` getSpecial noPrefix `mplus` getSpecial s
-  where
-    noPrefix = dropPrefix s
-    dropPrefix (c:rest) | c `elem` "!#" = rest
-    dropPrefix cs = cs
-    takeName s = do
-        let name = takeWhile isVariableChar s
-        guard . not $ null name
-        return name
-    getSpecial (c:_) | c `elem` "*@#?-$!" = return [c]
-    getSpecial _ = fail "empty or not special"
-
-    nameExpansion ('!':next:rest) = do -- e.g. ${!foo*bar*}
-        guard $ isVariableChar next -- e.g. ${!@}
-        first <- find (not . isVariableChar) rest
-        guard $ first `elem` "*?@"
-        return ""
-    nameExpansion _ = Nothing
-
-prop_getBracedModifier1 = getBracedModifier "foo:bar:baz" == ":bar:baz"
-prop_getBracedModifier2 = getBracedModifier "!var:-foo" == ":-foo"
-prop_getBracedModifier3 = getBracedModifier "foo[bar]" == "[bar]"
-getBracedModifier s = headOrDefault "" $ do
-    let var = getBracedReference s
-    a <- dropModifier s
-    dropPrefix var a
-  where
-    dropPrefix [] t        = return t
-    dropPrefix (a:b) (c:d) | a == c = dropPrefix b d
-    dropPrefix _ _         = []
-
-    dropModifier (c:rest) | c `elem` "#!" = [rest, c:rest]
-    dropModifier x        = [x]
-
--- Useful generic functions.
-
--- Get element 0 or a default. Like `head` but safe.
-headOrDefault _ (a:_) = a
-headOrDefault def _   = def
-
---- Get element n of a list, or Nothing. Like `!!` but safe.
-(!!!) list i =
-    case drop i list of
-        []    -> Nothing
-        (r:_) -> Just r
 
 -- Run a command if the shell is in the given list
 whenShell l c = do
@@ -945,5 +944,30 @@ isBashLike params =
         Dash -> False
         Sh -> False
 
+isTrueAssignmentSource c =
+    case c of
+        DataString SourceChecked -> False
+        DataString SourceDeclaration -> False
+        DataArray SourceChecked -> False
+        DataArray SourceDeclaration -> False
+        _ -> True
+
+modifiesVariable params token name =
+    or $ map check flow
+  where
+    flow = getVariableFlow params token
+    check t =
+        case t of
+            Assignment (_, _, n, source) -> isTrueAssignmentSource source && n == name
+            _ -> False
+
+-- Ebuild files inherit eclasses using 'inherit myclass1 myclass2'
+getInheritedEclasses :: Token -> [String]
+getInheritedEclasses root = execWriter $ doAnalysis findInheritedEclasses root
+  where
+    findInheritedEclasses cmd
+        | cmd `isCommand` "inherit" = tell $ catMaybes $ getLiteralString <$> (arguments cmd)
+    findInheritedEclasses _ = return ()
+
 return []
 runTests =  $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |])

src/ShellCheck/Checker.hs

@@ -1,5 +1,5 @@
 {-
-    Copyright 2012-2019 Vidar Holen
+    Copyright 2012-2022 Vidar Holen
 
     This file is part of ShellCheck.
     https://www.shellcheck.net
@@ -20,10 +20,12 @@
 {-# LANGUAGE TemplateHaskell #-}
 module ShellCheck.Checker (checkScript, ShellCheck.Checker.runTests) where
 
+import ShellCheck.Analyzer
+import ShellCheck.ASTLib
 import ShellCheck.Interface
 import ShellCheck.Parser
-import ShellCheck.Analyzer
 
+import Data.Char
 import Data.Either
 import Data.Functor
 import Data.List
@@ -53,6 +55,8 @@ shellFromFilename filename = listToMaybe candidates
     shellExtensions = [(".ksh", Ksh)
                       ,(".bash", Bash)
                       ,(".bats", Bash)
+                      ,(".ebuild", Bash)
+                      ,(".eclass", Bash)
                       ,(".dash", Dash)]
                       -- The `.sh` is too generic to determine the shell:
                       -- We fallback to Bash in this case and emit SC2148 if there is no shebang
@@ -83,18 +87,24 @@ checkScript sys spec = do
                     asShellType = csShellTypeOverride spec,
                     asFallbackShell = shellFromFilename $ csFilename spec,
                     asCheckSourced = csCheckSourced spec,
+                    asIsPortage = isPortage $ csFilename spec,
                     asExecutionMode = Executed,
                     asTokenPositions = tokenPositions,
-                    asOptionalChecks = csOptionalChecks spec
+                    asOptionalChecks = getEnableDirectives root ++ csOptionalChecks spec
                 } where as = newAnalysisSpec root
-        let analysisMessages =
+        let getAnalysisMessages =
-                maybe []
+                case prRoot result of
-                    (arComments . analyzeScript . analysisSpec)
+                    Just root -> arComments <$> (analyzeScript sys $ analysisSpec root)
-                        $ prRoot result
+                    Nothing -> return []
         let translator = tokenToPosition tokenPositions
+        analysisMessages <- getAnalysisMessages
         return . nub . sortMessages . filter shouldInclude $
             (parseMessages ++ map translator analysisMessages)
 
+    isPortage filename =
+        let f = map toLower filename in
+            ".ebuild" `isSuffixOf` f || ".eclass" `isSuffixOf` f
+
     shouldInclude pc =
             severity <= csMinSeverity spec &&
             case csIncludedWarnings spec of
@@ -156,6 +166,11 @@ checkWithIncludesAndSourcePath includes mapper = getErrors
         siFindSource = mapper
     }
 
+checkWithRcIncludesAndSourcePath rc includes mapper = getErrors
+    (mockRcFile rc $ mockedSystemInterface includes) {
+        siFindSource = mapper
+    }
+
 prop_findsParseIssue = check "echo \"$12\"" == [1037]
 
 prop_commentDisablesParseIssue1 =
@@ -238,6 +253,9 @@ prop_canStripPrefixAndSource2 =
 prop_canSourceDynamicWhenRedirected =
     null $ checkWithIncludes [("lib", "")] "#shellcheck source=lib\n. \"$1\""
 
+prop_canRedirectWithSpaces =
+    null $ checkWithIncludes [("my file", "")] "#shellcheck source=\"my file\"\n. \"$1\""
+
 prop_recursiveAnalysis =
     [2086] == checkRecursive [("lib", "echo $1")] "source lib"
 
@@ -301,6 +319,13 @@ prop_canDisableShebangWarning = null $ result
         csScript = "#shellcheck disable=SC2148\nfoo"
     }
 
+prop_canDisableAllWarnings = result == [2086]
+  where
+    result = checkWithSpec [] emptyCheckSpec {
+        csFilename = "file.sh",
+        csScript = "#!/bin/sh\necho $1\n#shellcheck disable=all\necho `echo $1`"
+    }
+
 prop_canDisableParseErrors = null $ result
   where
     result = checkWithSpec [] emptyCheckSpec {
@@ -384,7 +409,7 @@ prop_canEnableOptionalsWithRc = result == [2244]
 
 prop_sourcePathRedirectsName = result == [2086]
   where
-    f "dir/myscript" _ "lib" = return "foo/lib"
+    f "dir/myscript" _ _ "lib" = return "foo/lib"
     result = checkWithIncludesAndSourcePath [("foo/lib", "echo $1")] f emptyCheckSpec {
         csScript = "#!/bin/bash\nsource lib",
         csFilename = "dir/myscript",
@@ -393,22 +418,113 @@ prop_sourcePathRedirectsName = result == [2086]
 
 prop_sourcePathAddsAnnotation = result == [2086]
   where
-    f "dir/myscript" ["mypath"] "lib" = return "foo/lib"
+    f "dir/myscript" _ ["mypath"] "lib" = return "foo/lib"
     result = checkWithIncludesAndSourcePath [("foo/lib", "echo $1")] f emptyCheckSpec {
         csScript = "#!/bin/bash\n# shellcheck source-path=mypath\nsource lib",
         csFilename = "dir/myscript",
         csCheckSourced = True
     }
 
+prop_sourcePathWorksWithSpaces = result == [2086]
+  where
+    f "dir/myscript" _ ["my path"] "lib" = return "foo/lib"
+    result = checkWithIncludesAndSourcePath [("foo/lib", "echo $1")] f emptyCheckSpec {
+        csScript = "#!/bin/bash\n# shellcheck source-path='my path'\nsource lib",
+        csFilename = "dir/myscript",
+        csCheckSourced = True
+    }
+
 prop_sourcePathRedirectsDirective = result == [2086]
   where
-    f "dir/myscript" _ "lib" = return "foo/lib"
+    f "dir/myscript" _ _ "lib" = return "foo/lib"
-    f _ _ _ = return "/dev/null"
+    f _ _ _ _ = return "/dev/null"
     result = checkWithIncludesAndSourcePath [("foo/lib", "echo $1")] f emptyCheckSpec {
         csScript = "#!/bin/bash\n# shellcheck source=lib\nsource kittens",
         csFilename = "dir/myscript",
         csCheckSourced = True
     }
 
+prop_rcCanAllowExternalSources = result == [2086]
+  where
+    f "dir/myscript" (Just True) _ "mylib" = return "resolved/mylib"
+    f a b c d = error $ show ("Unexpected", a, b, c, d)
+    result = checkWithRcIncludesAndSourcePath "external-sources=true" [("resolved/mylib", "echo $1")] f emptyCheckSpec {
+        csScript = "#!/bin/bash\nsource mylib",
+        csFilename = "dir/myscript",
+        csCheckSourced = True
+    }
+
+prop_rcCanDenyExternalSources = result == [2086]
+  where
+    f "dir/myscript" (Just False) _ "mylib" = return "resolved/mylib"
+    f a b c d = error $ show ("Unexpected", a, b, c, d)
+    result = checkWithRcIncludesAndSourcePath "external-sources=false" [("resolved/mylib", "echo $1")] f emptyCheckSpec {
+        csScript = "#!/bin/bash\nsource mylib",
+        csFilename = "dir/myscript",
+        csCheckSourced = True
+    }
+
+prop_rcCanLeaveExternalSourcesUnspecified = result == [2086]
+  where
+    f "dir/myscript" Nothing _ "mylib" = return "resolved/mylib"
+    f a b c d = error $ show ("Unexpected", a, b, c, d)
+    result = checkWithRcIncludesAndSourcePath "" [("resolved/mylib", "echo $1")] f emptyCheckSpec {
+        csScript = "#!/bin/bash\nsource mylib",
+        csFilename = "dir/myscript",
+        csCheckSourced = True
+    }
+
+prop_fileCanDisableExternalSources = result == [2006, 2086]
+  where
+    f "dir/myscript" (Just True) _ "withExternal" = return "withExternal"
+    f "dir/myscript" (Just False) _ "withoutExternal" = return "withoutExternal"
+    f a b c d = error $ show ("Unexpected", a, b, c, d)
+    result = checkWithRcIncludesAndSourcePath "external-sources=true" [("withExternal", "echo $1"), ("withoutExternal", "_=`foo`")] f emptyCheckSpec {
+        csScript = "#!/bin/bash\ntrue\nsource withExternal\n# shellcheck external-sources=false\nsource withoutExternal",
+        csFilename = "dir/myscript",
+        csCheckSourced = True
+    }
+
+prop_fileCannotEnableExternalSources = result == [1144]
+  where
+    f "dir/myscript" Nothing _ "foo" = return "foo"
+    f a b c d = error $ show ("Unexpected", a, b, c, d)
+    result = checkWithRcIncludesAndSourcePath "" [("foo", "true")] f emptyCheckSpec {
+        csScript = "#!/bin/bash\n# shellcheck external-sources=true\nsource foo",
+        csFilename = "dir/myscript",
+        csCheckSourced = True
+    }
+
+prop_fileCannotEnableExternalSources2 = result == [1144]
+  where
+    f "dir/myscript" (Just False) _ "foo" = return "foo"
+    f a b c d = error $ show ("Unexpected", a, b, c, d)
+    result = checkWithRcIncludesAndSourcePath "external-sources=false" [("foo", "true")] f emptyCheckSpec {
+        csScript = "#!/bin/bash\n# shellcheck external-sources=true\nsource foo",
+        csFilename = "dir/myscript",
+        csCheckSourced = True
+    }
+
+prop_rcCanSuppressEarlyProblems1 = null result
+  where
+    result = checkWithRc "disable=1071" emptyCheckSpec {
+        csScript = "#!/bin/zsh\necho $1"
+    }
+
+prop_rcCanSuppressEarlyProblems2 = null result
+  where
+    result = checkWithRc "disable=1104" emptyCheckSpec {
+        csScript = "!/bin/bash\necho 'hello world'"
+    }
+
+prop_sourceWithHereDocWorks = null result
+  where
+    result = checkWithIncludes [("bar", "true\n")] "source bar << eof\nlol\neof"
+
+prop_hereDocsAreParsedWithoutTrailingLinefeed = 1044 `elem` result
+  where
+    result = check "cat << eof"
+
+
 return []
 runTests = $quickCheckAll

src/ShellCheck/Checks/Commands.hs

@@ -1,5 +1,5 @@
 {-
-    Copyright 2012-2019 Vidar Holen
+    Copyright 2012-2022 Vidar Holen
 
     This file is part of ShellCheck.
     https://www.shellcheck.net
@@ -27,21 +27,28 @@ module ShellCheck.Checks.Commands (checker, optionalChecks, ShellCheck.Checks.Co
 import ShellCheck.AST
 import ShellCheck.ASTLib
 import ShellCheck.AnalyzerLib
+import ShellCheck.CFG
+import qualified ShellCheck.CFGAnalysis as CF
 import ShellCheck.Data
 import ShellCheck.Interface
 import ShellCheck.Parser
+import ShellCheck.Prelude
 import ShellCheck.Regex
 
 import Control.Monad
 import Control.Monad.RWS
 import Data.Char
 import Data.Functor.Identity
+import qualified Data.Graph.Inductive.Graph as G
 import Data.List
 import Data.Maybe
-import qualified Data.Map.Strict as Map
+import qualified Data.Map.Strict as M
+import qualified Data.Set as S
 import Test.QuickCheck.All (forAllProperties)
 import Test.QuickCheck.Test (quickCheckWithResult, stdArgs, maxSuccess)
 
+import Debug.Trace -- STRIP
+
 data CommandName = Exactly String | Basename String
     deriving (Eq, Ord)
 
@@ -57,7 +64,7 @@ commandChecks :: [CommandCheck]
 commandChecks = [
     checkTr
     ,checkFindNameGlob
-    ,checkNeedlessExpr
+    ,checkExpr
     ,checkGrepRe
     ,checkTrapQuotes
     ,checkReturn
@@ -95,7 +102,14 @@ commandChecks = [
     ,checkSourceArgs
     ,checkChmodDashr
     ,checkXargsDashi
+    ,checkUnquotedEchoSpaces
+    ,checkEvalArray
     ]
+    ++ map checkArgComparison ("alias" : declaringCommands)
+    ++ map checkMaskedReturns declaringCommands
+    ++ map checkMultipleDeclaring declaringCommands
+    ++ map checkBackreferencingDeclaration declaringCommands
+
 
 optionalChecks = map fst optionalCommandChecks
 optionalCommandChecks :: [(CheckDescription, CommandCheck)]
@@ -107,7 +121,7 @@ optionalCommandChecks = [
         cdNegative = "command -v javac"
     }, checkWhich)
     ]
-optionalCheckMap = Map.fromList $ map (\(desc, check) -> (cdName desc, check)) optionalCommandChecks
+optionalCheckMap = M.fromList $ map (\(desc, check) -> (cdName desc, check)) optionalCommandChecks
 
 prop_verifyOptionalExamples = all check optionalCommandChecks
   where
@@ -131,40 +145,52 @@ prop_checkGetOptsS3 = checkGetOpts "-f -x" ["f", "x"] [] $ getOpts (True, True)
 prop_checkGetOptsS4 = checkGetOpts "-f -x" ["f"] [] $ getOpts (True, True) "f:" []
 prop_checkGetOptsS5 = checkGetOpts "-fx" [] [] $ getOpts (True, True) "fx:" []
 
+prop_checkGenericOptsS1 = checkGetOpts "-f x" ["f"] [] $ return . getGenericOpts
+prop_checkGenericOptsS2 = checkGetOpts "-abc x" ["a", "b", "c"] [] $ return . getGenericOpts
+prop_checkGenericOptsS3 = checkGetOpts "-abc -x" ["a", "b", "c", "x"] [] $ return . getGenericOpts
+prop_checkGenericOptsS4 = checkGetOpts "-x" ["x"] [] $ return . getGenericOpts
+
 -- Long options
 prop_checkGetOptsL1 = checkGetOpts "--foo=bar baz" ["foo"] ["baz"] $ getOpts (True, False) "" [("foo", True)]
 prop_checkGetOptsL2 = checkGetOpts "--foo bar baz" ["foo"] ["baz"] $ getOpts (True, False) "" [("foo", True)]
 prop_checkGetOptsL3 = checkGetOpts "--foo baz" ["foo"] ["baz"] $ getOpts (True, True) "" []
 prop_checkGetOptsL4 = checkGetOpts "--foo baz" [] [] $ getOpts (True, False) "" []
 
+prop_checkGenericOptsL1 = checkGetOpts "--foo=bar" ["foo"] [] $ return . getGenericOpts
+prop_checkGenericOptsL2 = checkGetOpts "--foo bar" ["foo"] ["bar"] $ return . getGenericOpts
+prop_checkGenericOptsL3 = checkGetOpts "-x --foo" ["x", "foo"] [] $ return . getGenericOpts
+
 -- Know when to terminate
 prop_checkGetOptsT1 = checkGetOpts "-a x -b" ["a", "b"] ["x"] $ getOpts (True, True) "ab" []
 prop_checkGetOptsT2 = checkGetOpts "-a x -b" ["a"] ["x","-b"] $ getOpts (False, True) "ab" []
 prop_checkGetOptsT3 = checkGetOpts "-a -- -b" ["a"] ["-b"] $ getOpts (True, True) "ab" []
 prop_checkGetOptsT4 = checkGetOpts "-a -- -b" ["a", "b"] [] $ getOpts (True, True) "a:b" []
 
+prop_checkGenericOptsT1 = checkGetOpts "-x -- -y" ["x"] ["-y"] $ return . getGenericOpts
+prop_checkGenericOptsT2 = checkGetOpts "-xy --" ["x", "y"] [] $ return . getGenericOpts
 
-buildCommandMap :: [CommandCheck] -> Map.Map CommandName (Token -> Analysis)
+
-buildCommandMap = foldl' addCheck Map.empty
+buildCommandMap :: [CommandCheck] -> M.Map CommandName (Token -> Analysis)
+buildCommandMap = foldl' addCheck M.empty
   where
     addCheck map (CommandCheck name function) =
-        Map.insertWith composeAnalyzers name function map
+        M.insertWith composeAnalyzers name function map
 
 
-checkCommand :: Map.Map CommandName (Token -> Analysis) -> Token -> Analysis
+checkCommand :: M.Map CommandName (Token -> Analysis) -> Token -> Analysis
 checkCommand map t@(T_SimpleCommand id cmdPrefix (cmd:rest)) = sequence_ $ do
     name <- getLiteralString cmd
     return $
         if '/' `elem` name
         then
-            Map.findWithDefault nullCheck (Basename $ basename name) map t
+            M.findWithDefault nullCheck (Basename $ basename name) map t
         else if name == "builtin" && not (null rest) then
             let t' = T_SimpleCommand id cmdPrefix rest
                 selectedBuiltin = fromMaybe "" $ getLiteralString . head $ rest
-            in Map.findWithDefault nullCheck (Exactly selectedBuiltin) map t'
+            in M.findWithDefault nullCheck (Exactly selectedBuiltin) map t'
         else do
-            Map.findWithDefault nullCheck (Exactly name) map t
+            M.findWithDefault nullCheck (Exactly name) map t
-            Map.findWithDefault nullCheck (Basename name) map t
+            M.findWithDefault nullCheck (Basename name) map t
 
   where
     basename = reverse . takeWhile (/= '/') . reverse
@@ -186,7 +212,7 @@ checker spec params = getChecker $ commandChecks ++ optionals
     optionals =
         if "all" `elem` keys
         then map snd optionalCommandChecks
-        else mapMaybe (\x -> Map.lookup x optionalCheckMap) keys
+        else mapMaybe (\x -> M.lookup x optionalCheckMap) keys
 
 prop_checkTr1 = verify checkTr "tr [a-f] [A-F]"
 prop_checkTr2 = verify checkTr "tr 'a-z' 'A-Z'"
@@ -235,19 +261,74 @@ checkFindNameGlob = CommandCheck (Basename "find") (f . arguments)  where
         acc b
 
 
-prop_checkNeedlessExpr = verify checkNeedlessExpr "foo=$(expr 3 + 2)"
+prop_checkExpr = verify checkExpr "foo=$(expr 3 + 2)"
-prop_checkNeedlessExpr2 = verify checkNeedlessExpr "foo=`echo \\`expr 3 + 2\\``"
+prop_checkExpr2 = verify checkExpr "foo=`echo \\`expr 3 + 2\\``"
-prop_checkNeedlessExpr3 = verifyNot checkNeedlessExpr "foo=$(expr foo : regex)"
+prop_checkExpr3 = verifyNot checkExpr "foo=$(expr foo : regex)"
-prop_checkNeedlessExpr4 = verifyNot checkNeedlessExpr "foo=$(expr foo \\< regex)"
+prop_checkExpr4 = verifyNot checkExpr "foo=$(expr foo \\< regex)"
-checkNeedlessExpr = CommandCheck (Basename "expr") f where
+prop_checkExpr5 = verify checkExpr "# shellcheck disable=SC2003\nexpr match foo bar"
-    f t =
+prop_checkExpr6 = verify checkExpr "# shellcheck disable=SC2003\nexpr foo : fo*"
+prop_checkExpr7 = verify checkExpr "# shellcheck disable=SC2003\nexpr 5 -3"
+prop_checkExpr8 = verifyNot checkExpr "# shellcheck disable=SC2003\nexpr \"$@\""
+prop_checkExpr9 = verifyNot checkExpr "# shellcheck disable=SC2003\nexpr 5 $rest"
+prop_checkExpr10 = verify checkExpr "# shellcheck disable=SC2003\nexpr length \"$var\""
+prop_checkExpr11 = verify checkExpr "# shellcheck disable=SC2003\nexpr foo > bar"
+prop_checkExpr12 = verify checkExpr "# shellcheck disable=SC2003\nexpr 1 | 2"
+prop_checkExpr13 = verify checkExpr "# shellcheck disable=SC2003\nexpr 1 * 2"
+prop_checkExpr14 = verify checkExpr "# shellcheck disable=SC2003\nexpr \"$x\" >=  \"$y\""
+
+checkExpr = CommandCheck (Basename "expr") f where
+    f t = do
         when (all (`notElem` exceptions) (words $ arguments t)) $
             style (getId $ getCommandTokenOrThis t) 2003
                 "expr is antiquated. Consider rewriting this using $((..)), ${} or [[ ]]."
+
+        case arguments t of
+            [lhs, op, rhs] -> do
+                checkOp lhs
+                case getWordParts op of
+                    [T_Glob _ "*"] ->
+                        err (getId op) 2304
+                            "* must be escaped to multiply: \\*. Modern $((x * y)) avoids this issue."
+                    [T_Literal _ ":"] | isGlob rhs ->
+                        warn (getId rhs) 2305
+                            "Quote regex argument to expr to avoid it expanding as a glob."
+                    _ -> return ()
+
+            [single] | not (willSplit single) ->
+                warn (getId single) 2307
+                    "'expr' expects 3+ arguments but sees 1. Make sure each operator/operand is a separate argument, and escape <>&|."
+
+            [first, second] |
+                (fromMaybe "" $ getLiteralString first) /= "length"
+                  && not (willSplit first || willSplit second) -> do
+                    checkOp first
+                    warn (getId t) 2307
+                        "'expr' expects 3+ arguments, but sees 2. Make sure each operator/operand is a separate argument, and escape <>&|."
+
+            (first:rest) -> do
+                checkOp first
+                forM_ rest $ \t ->
+                    -- We already find 95%+ of multiplication and regex earlier, so don't bother classifying this further.
+                    when (isGlob t) $ warn (getId t) 2306 "Escape glob characters in arguments to expr to avoid pathname expansion."
+
+            _ -> return ()
+
     -- These operators are hard to replicate in POSIX
-    exceptions = [ ":", "<", ">", "<=", ">=" ]
+    exceptions = [ ":", "<", ">", "<=", ">=",
+        -- We can offer better suggestions for these
+        "match", "length", "substr", "index"]
     words = mapMaybe getLiteralString
 
+    checkOp side =
+        case getLiteralString side of
+            Just "match" -> msg "'expr match' has unspecified results. Prefer 'expr str : regex'."
+            Just "length" -> msg "'expr length' has unspecified results. Prefer ${#var}."
+            Just "substr" -> msg "'expr substr' has unspecified results. Prefer 'cut' or ${var#???}."
+            Just "index" -> msg "'expr index' has unspecified results. Prefer x=${var%%[chars]*}; $((${#x}+1))."
+            _ -> return ()
+      where
+        msg = info (getId side) 2308
+
 
 prop_checkGrepRe1 = verify checkGrepRe "cat foo | grep *.mp3"
 prop_checkGrepRe2 = verify checkGrepRe "grep -Ev cow*test *.mp3"
@@ -400,9 +481,16 @@ prop_checkUnusedEchoEscapes2 = verifyNot checkUnusedEchoEscapes "echo -e 'foi\\n
 prop_checkUnusedEchoEscapes3 = verify checkUnusedEchoEscapes "echo \"n:\\t42\""
 prop_checkUnusedEchoEscapes4 = verifyNot checkUnusedEchoEscapes "echo lol"
 prop_checkUnusedEchoEscapes5 = verifyNot checkUnusedEchoEscapes "echo -n -e '\n'"
+prop_checkUnusedEchoEscapes6 = verify checkUnusedEchoEscapes "echo '\\506'"
+prop_checkUnusedEchoEscapes7 = verify checkUnusedEchoEscapes "echo '\\5a'"
+prop_checkUnusedEchoEscapes8 = verifyNot checkUnusedEchoEscapes "echo '\\8a'"
+prop_checkUnusedEchoEscapes9 = verifyNot checkUnusedEchoEscapes "echo '\\d5a'"
+prop_checkUnusedEchoEscapes10 = verify checkUnusedEchoEscapes "echo '\\x4a'"
+prop_checkUnusedEchoEscapes11 = verify checkUnusedEchoEscapes "echo '\\xat'"
+prop_checkUnusedEchoEscapes12 = verifyNot checkUnusedEchoEscapes "echo '\\xth'"
 checkUnusedEchoEscapes = CommandCheck (Basename "echo") f
   where
-    hasEscapes = mkRegex "\\\\[rnt]"
+    hasEscapes = mkRegex "\\\\([rntabefv\\']|[0-7]{1,3}|x([0-9]|[A-F]|[a-f]){1,2})"
     f cmd =
         whenShell [Sh, Bash, Ksh] $
             unless (cmd `hasFlag` "e") $
@@ -602,6 +690,7 @@ checkPrintfVar = CommandCheck (Exactly "printf") (f . arguments) where
             let formats = getPrintfFormats string
             let formatCount = length formats
             let argCount = length more
+            let pluraliseIfMany word n = if n > 1 then word ++ "s" else word
 
             return $ if
                 | argCount == 0 && formatCount == 0 ->
@@ -617,7 +706,8 @@ checkPrintfVar = CommandCheck (Exactly "printf") (f . arguments) where
                     return () -- Great: a suitable number of arguments
                 | otherwise ->
                     warn (getId format) 2183 $
-                        "This format string has " ++ show formatCount ++ " variables, but is passed " ++ show argCount ++ " arguments."
+                        "This format string has " ++ show formatCount ++ " " ++ pluraliseIfMany "variable" formatCount ++
+                        ", but is passed " ++ show argCount ++ pluraliseIfMany " argument" argCount ++ "."
 
         unless ('%' `elem` concat (oversimplify format) || isLiteral format) $
           info (getId format) 2059
@@ -663,7 +753,7 @@ getPrintfFormats = getFormats
         --            \____ _____/\___ ____/   \____ ____/\_________ _________/ \______ /
         --                 V          V             V               V               V
         --               flags    field width  precision   format character        rest
-        -- field width and precision can be specified with a '*' instead of a digit,
+        -- field width and precision can be specified with an '*' instead of a digit,
         -- in which case printf will accept one more argument for each '*' used
 
 
@@ -718,6 +808,7 @@ prop_checkReadExpansions5 = verify checkReadExpansions "read \"$var\""
 prop_checkReadExpansions6 = verify checkReadExpansions "read -a $var"
 prop_checkReadExpansions7 = verifyNot checkReadExpansions "read $1"
 prop_checkReadExpansions8 = verifyNot checkReadExpansions "read ${var?}"
+prop_checkReadExpansions9 = verify checkReadExpansions "read arr[val]"
 checkReadExpansions = CommandCheck (Exactly "read") check
   where
     options = getGnuOpts flagsForRead
@@ -725,13 +816,26 @@ checkReadExpansions = CommandCheck (Exactly "read") check
         opts <- options $ arguments cmd
         return [y | (x,(_, y)) <- opts, null x || x == "a"]
 
-    check cmd = mapM_ warning $ getVars cmd
+    check cmd = do
-    warning t = sequence_ $ do
+        mapM_ dollarWarning $ getVars cmd
+        mapM_ arrayWarning $ arguments cmd
+
+    dollarWarning t = sequence_ $ do
         name <- getSingleUnmodifiedBracedString t
         guard $ isVariableName name   -- e.g. not $1
         return . warn (getId t) 2229 $
             "This does not read '" ++ name ++ "'. Remove $/${} for that, or use ${var?} to quiet."
 
+    arrayWarning word =
+        when (any isUnquotedBracket $ getWordParts word) $
+            warn (getId word) 2313 $
+                "Quote array indices to avoid them expanding as globs."
+
+    isUnquotedBracket t =
+        case t of
+            T_Glob _ ('[':_) -> True
+            _ -> False
+
 -- Return the single variable expansion that makes up this word, if any.
 -- e.g. $foo -> $foo, "$foo"'' -> $foo , "hello $name" -> Nothing
 getSingleUnmodifiedBracedString :: Token -> Maybe String
@@ -771,6 +875,9 @@ checkAliasesExpandEarly = CommandCheck (Exactly "alias") (f . arguments)
 
 prop_checkUnsetGlobs1 = verify checkUnsetGlobs "unset foo[1]"
 prop_checkUnsetGlobs2 = verifyNot checkUnsetGlobs "unset foo"
+prop_checkUnsetGlobs3 = verify checkUnsetGlobs "unset foo[$i]"
+prop_checkUnsetGlobs4 = verify checkUnsetGlobs "unset foo[x${i}y]"
+prop_checkUnsetGlobs5 = verifyNot checkUnsetGlobs "unset foo]["
 checkUnsetGlobs = CommandCheck (Exactly "unset") (mapM_ check . arguments)
   where
     check arg =
@@ -852,6 +959,22 @@ checkLocalScope = CommandCheck (Exactly "local") $ \t ->
         unless (any isFunctionLike path) $
             err (getId $ getCommandTokenOrThis t) 2168 "'local' is only valid in functions."
 
+prop_checkMultipleDeclaring1 = verify (checkMultipleDeclaring "local") "q() { local readonly var=1; }"
+prop_checkMultipleDeclaring2 = verifyNot (checkMultipleDeclaring "local") "q() { local var=1; }"
+prop_checkMultipleDeclaring3 = verify (checkMultipleDeclaring "readonly") "readonly local foo=5"
+prop_checkMultipleDeclaring4 = verify (checkMultipleDeclaring "export") "export readonly foo=5"
+prop_checkMultipleDeclaring5 = verifyNot (checkMultipleDeclaring "local") "f() { local -r foo=5; }"
+prop_checkMultipleDeclaring6 = verifyNot (checkMultipleDeclaring "declare") "declare -rx foo=5"
+prop_checkMultipleDeclaring7 = verifyNot (checkMultipleDeclaring "readonly") "readonly 'local' foo=5"
+checkMultipleDeclaring cmd = CommandCheck (Exactly cmd) (mapM_ check . arguments)
+  where
+    check t = sequence_ $ do
+        lit <- getUnquotedLiteral t
+        guard $ lit `elem` declaringCommands
+        return $ err (getId $ getCommandTokenOrThis t) 2316 $
+                 "This applies " ++ cmd ++ " to the variable named " ++ lit ++
+                 ", which is probably not what you want. Use a separate command or the appropriate `declare` options instead."
+
 prop_checkDeprecatedTempfile1 = verify checkDeprecatedTempfile "var=$(tempfile)"
 prop_checkDeprecatedTempfile2 = verifyNot checkDeprecatedTempfile "tempfile=$(mktemp)"
 checkDeprecatedTempfile = CommandCheck (Basename "tempfile") $
@@ -870,34 +993,48 @@ prop_checkWhileGetoptsCase2 = verify checkWhileGetoptsCase "while getopts 'a:' x
 prop_checkWhileGetoptsCase3 = verifyNot checkWhileGetoptsCase "while getopts 'a:b' x; do case $x in a) foo;; b) bar;; *) :;esac; done"
 prop_checkWhileGetoptsCase4 = verifyNot checkWhileGetoptsCase "while getopts 'a:123' x; do case $x in a) foo;; [0-9]) bar;; esac; done"
 prop_checkWhileGetoptsCase5 = verifyNot checkWhileGetoptsCase "while getopts 'a:' x; do case $x in a) foo;; \\?) bar;; *) baz;; esac; done"
+prop_checkWhileGetoptsCase6 = verifyNot checkWhileGetoptsCase "while getopts 'a:b' x; do case $y in a) foo;; esac; done"
+prop_checkWhileGetoptsCase7 = verifyNot checkWhileGetoptsCase "while getopts 'a:b' x; do case x$x in xa) foo;; xb) foo;; esac; done"
+prop_checkWhileGetoptsCase8 = verifyNot checkWhileGetoptsCase "while getopts 'a:b' x; do x=a; case $x in a) foo;; esac; done"
 checkWhileGetoptsCase = CommandCheck (Exactly "getopts") f
   where
     f :: Token -> Analysis
-    f t@(T_SimpleCommand _ _ (cmd:arg1:_))  = do
+    f t@(T_SimpleCommand _ _ (cmd:arg1:name:_))  = do
         path <- getPathM t
+        params <- ask
         sequence_ $ do
             options <- getLiteralString arg1
+            getoptsVar <- getLiteralString name
             (T_WhileExpression _ _ body) <- findFirst whileLoop path
-            caseCmd <- mapMaybe findCase body !!! 0
+            caseCmd@(T_CaseExpression _ var _) <- mapMaybe findCase body !!! 0
+
+            -- Make sure getopts name and case variable matches
+            [T_DollarBraced _ _ bracedWord] <- return $ getWordParts var
+            [T_Literal _ caseVar] <- return $ getWordParts bracedWord
+            guard $ caseVar == getoptsVar
+
+            -- Make sure the variable isn't modified
+            guard . not $ modifiesVariable params (T_BraceGroup (Id 0) body) getoptsVar
+
             return $ check (getId arg1) (map (:[]) $ filter (/= ':') options) caseCmd
     f _ = return ()
 
     check :: Id -> [String] -> Token -> Analysis
     check optId opts (T_CaseExpression id _ list) = do
-            unless (Nothing `Map.member` handledMap) $ do
+            unless (Nothing `M.member` handledMap) $ do
-                mapM_ (warnUnhandled optId id) $ catMaybes $ Map.keys notHandled
+                mapM_ (warnUnhandled optId id) $ catMaybes $ M.keys notHandled
 
-                unless (any (`Map.member` handledMap) [Just "*",Just "?"]) $
+                unless (any (`M.member` handledMap) [Just "*",Just "?"]) $
                     warn id 2220 "Invalid flags are not handled. Add a *) case."
 
-            mapM_ warnRedundant $ Map.toList notRequested
+            mapM_ warnRedundant $ M.toList notRequested
 
         where
-            handledMap = Map.fromList (concatMap getHandledStrings list)
+            handledMap = M.fromList (concatMap getHandledStrings list)
-            requestedMap = Map.fromList $ map (\x -> (Just x, ())) opts
+            requestedMap = M.fromList $ map (\x -> (Just x, ())) opts
 
-            notHandled = Map.difference requestedMap handledMap
+            notHandled = M.difference requestedMap handledMap
-            notRequested = Map.difference handledMap requestedMap
+            notRequested = M.difference handledMap requestedMap
 
     warnUnhandled optId caseId str =
         warn caseId 2213 $ "getopts specified -" ++ (e4m str) ++ ", but it's not handled by this 'case'."
@@ -1056,7 +1193,7 @@ checkFindRedirections = CommandCheck (Basename "find") f
 
 prop_checkWhich = verify checkWhich "which '.+'"
 checkWhich = CommandCheck (Basename "which") $
-    \t -> info (getId $ getCommandTokenOrThis t) 2230 "which is non-standard. Use builtin 'command -v' instead."
+    \t -> info (getId $ getCommandTokenOrThis t) 2230 "'which' is non-standard. Use builtin 'command -v' instead."
 
 prop_checkSudoRedirect1 = verify checkSudoRedirect "sudo echo 3 > /proc/file"
 prop_checkSudoRedirect2 = verify checkSudoRedirect "sudo cmd < input"
@@ -1143,5 +1280,192 @@ checkXargsDashi = CommandCheck (Basename "xargs") f
         return $ info (getId option) 2267 "GNU xargs -i is deprecated in favor of -I{}"
     parseOpts = getBsdOpts "0oprtxadR:S:J:L:l:n:P:s:e:E:i:I:"
 
+
+prop_checkArgComparison1 = verify (checkArgComparison "declare") "declare a = b"
+prop_checkArgComparison2 = verify (checkArgComparison "declare") "declare a =b"
+prop_checkArgComparison3 = verifyNot (checkArgComparison "declare") "declare a=b"
+prop_checkArgComparison4 = verify (checkArgComparison "export") "export a +=b"
+prop_checkArgComparison7 = verifyNot (checkArgComparison "declare") "declare -a +i foo"
+prop_checkArgComparison8 = verify (checkArgComparison "let") "let x = 0"
+prop_checkArgComparison9 = verify (checkArgComparison "alias") "alias x =0"
+-- This mirrors checkSecondArgIsComparison but for arguments to local/readonly/declare/export
+checkArgComparison cmd = CommandCheck (Exactly cmd) wordsWithEqual
+  where
+    wordsWithEqual t = mapM_ check $ arguments t
+    check arg = do
+      sequence_ $ do
+        str <- getLeadingUnquotedString arg
+        case str of
+            '=':_ ->
+                return $ err (headId arg) 2290 $
+                    "Remove spaces around = to assign."
+            '+':'=':_ ->
+                return $ err (headId arg) 2290 $
+                    "Remove spaces around += to append."
+            _ -> Nothing
+
+       -- 'let' is parsed as a sequence of arithmetic expansions,
+       -- so we want the additional warning for "x="
+      when (cmd == "let") $ sequence_ $ do
+        token <- getTrailingUnquotedLiteral arg
+        str <- getLiteralString token
+        guard $ "=" `isSuffixOf` str
+        return $ err (getId token) 2290 $
+            "Remove spaces around = to assign."
+
+    headId t =
+        case t of
+            T_NormalWord _ (x:_) -> getId x
+            _ -> getId t
+
+
+prop_checkMaskedReturns1 = verify (checkMaskedReturns "local") "f() { local a=$(false); }"
+prop_checkMaskedReturns2 = verify (checkMaskedReturns "declare") "declare a=$(false)"
+prop_checkMaskedReturns3 = verify (checkMaskedReturns "declare") "declare a=\"`false`\""
+prop_checkMaskedReturns4 = verify (checkMaskedReturns "readonly") "readonly a=$(false)"
+prop_checkMaskedReturns5 = verify (checkMaskedReturns "readonly") "readonly a=\"`false`\""
+prop_checkMaskedReturns6 = verifyNot (checkMaskedReturns "declare") "declare a; a=$(false)"
+prop_checkMaskedReturns7 = verifyNot (checkMaskedReturns "local") "f() { local -r a=$(false); }"
+prop_checkMaskedReturns8 = verifyNot (checkMaskedReturns "readonly") "a=$(false); readonly a"
+prop_checkMaskedReturns9 = verify (checkMaskedReturns "typeset") "#!/bin/ksh\n f() { typeset -r x=$(false); }"
+prop_checkMaskedReturns10 = verifyNot (checkMaskedReturns "typeset") "#!/bin/ksh\n function f { typeset -r x=$(false); }"
+prop_checkMaskedReturns11 = verifyNot (checkMaskedReturns "typeset") "#!/bin/bash\n f() { typeset -r x=$(false); }"
+prop_checkMaskedReturns12 = verify (checkMaskedReturns "typeset") "typeset -r x=$(false);"
+prop_checkMaskedReturns13 = verify (checkMaskedReturns "typeset") "f() { typeset -g x=$(false); }"
+prop_checkMaskedReturns14 = verify (checkMaskedReturns "declare") "declare x=${ false; }"
+prop_checkMaskedReturns15 = verify (checkMaskedReturns "declare") "f() { declare x=$(false); }"
+checkMaskedReturns str = CommandCheck (Exactly str) checkCmd
+  where
+    checkCmd t = do
+        path <- getPathM t
+        shell <- asks shellType
+        sequence_ $ do
+            name <- getCommandName t
+
+            let flags = map snd (getAllFlags t)
+            let hasDashR =  "r" `elem` flags
+            let hasDashG =  "g" `elem` flags
+            let isInScopedFunction = any (isScopedFunction shell) path
+
+            let isLocal = not hasDashG && isLocalInFunction name && isInScopedFunction
+            let isReadOnly = name == "readonly" || hasDashR
+
+            -- Don't warn about local variables that are declared readonly,
+            -- because the workaround `local x; x=$(false); local -r x;` is annoying
+            guard . not $ isLocal && isReadOnly
+
+            return $ mapM_ checkArgs $ arguments t
+
+    checkArgs (T_Assignment id _ _ _ word) | any hasReturn $ getWordParts word =
+        warn id 2155 "Declare and assign separately to avoid masking return values."
+    checkArgs _ = return ()
+
+    isLocalInFunction = (`elem` ["local", "declare", "typeset"])
+    isScopedFunction shell t =
+        case t of
+            T_BatsTest {} -> True
+            -- In ksh, only functions declared with 'function' have their own scope
+            T_Function _ (FunctionKeyword hasFunction) _ _ _ -> shell /= Ksh || hasFunction
+            _ -> False
+
+    hasReturn t = case t of
+        T_Backticked {} -> True
+        T_DollarExpansion {} -> True
+        T_DollarBraceCommandExpansion {} -> True
+        _ -> False
+
+
+prop_checkUnquotedEchoSpaces1 = verify checkUnquotedEchoSpaces "echo foo         bar"
+prop_checkUnquotedEchoSpaces2 = verifyNot checkUnquotedEchoSpaces "echo       foo"
+prop_checkUnquotedEchoSpaces3 = verifyNot checkUnquotedEchoSpaces "echo foo  bar"
+prop_checkUnquotedEchoSpaces4 = verifyNot checkUnquotedEchoSpaces "echo 'foo          bar'"
+prop_checkUnquotedEchoSpaces5 = verifyNot checkUnquotedEchoSpaces "echo a > myfile.txt b"
+prop_checkUnquotedEchoSpaces6 = verifyNot checkUnquotedEchoSpaces "        echo foo\\\n        bar"
+checkUnquotedEchoSpaces = CommandCheck (Basename "echo") check
+  where
+    check t = do
+        let args = arguments t
+        m <- asks tokenPositions
+        redir <- getClosestCommandM t
+        sequence_ $ do
+            let positions = mapMaybe (\c -> M.lookup (getId c) m) args
+            let pairs = zip positions (drop 1 positions)
+            (T_Redirecting _ redirTokens _) <- redir
+            let redirPositions = mapMaybe (\c -> fst <$> M.lookup (getId c) m) redirTokens
+            guard $ any (hasSpacesBetween redirPositions) pairs
+            return $ info (getId t) 2291 "Quote repeated spaces to avoid them collapsing into one."
+
+    hasSpacesBetween redirs ((a,b), (c,d)) =
+        posLine a == posLine d
+        && ((posColumn c) - (posColumn b)) >= 4
+        && not (any (\x -> b < x && x < c) redirs)
+
+
+prop_checkEvalArray1 = verify checkEvalArray  "eval $@"
+prop_checkEvalArray2 = verify checkEvalArray  "eval \"${args[@]}\""
+prop_checkEvalArray3 = verify checkEvalArray  "eval \"${args[@]@Q}\""
+prop_checkEvalArray4 = verifyNot checkEvalArray  "eval \"${args[*]@Q}\""
+prop_checkEvalArray5 = verifyNot checkEvalArray  "eval \"$*\""
+checkEvalArray = CommandCheck (Exactly "eval") (mapM_ check . concatMap getWordParts . arguments)
+  where
+    check t =
+        when (isArrayExpansion t) $
+            if isEscaped t
+            then style (getId t) 2293 "When eval'ing @Q-quoted words, use * rather than @ as the index."
+            else warn (getId t) 2294 "eval negates the benefit of arrays. Drop eval to preserve whitespace/symbols (or eval as string)."
+
+    isEscaped q =
+        case q of
+            -- Match ${arr[@]@Q} and ${@@Q} and such
+            T_DollarBraced _ _ l -> 'Q' `elem` getBracedModifier (concat $ oversimplify l)
+            _ -> False
+
+
+prop_checkBackreferencingDeclaration1 = verify (checkBackreferencingDeclaration "declare") "declare x=1 y=foo$x"
+prop_checkBackreferencingDeclaration2 = verify (checkBackreferencingDeclaration "readonly") "readonly x=1 y=$((1+x))"
+prop_checkBackreferencingDeclaration3 = verify (checkBackreferencingDeclaration "local") "local x=1 y=$(echo $x)"
+prop_checkBackreferencingDeclaration4 = verify (checkBackreferencingDeclaration "local") "local x=1 y[$x]=z"
+prop_checkBackreferencingDeclaration5 = verify (checkBackreferencingDeclaration "declare") "declare x=var $x=1"
+prop_checkBackreferencingDeclaration6 = verify (checkBackreferencingDeclaration "declare") "declare x=var $x=1"
+prop_checkBackreferencingDeclaration7 = verify (checkBackreferencingDeclaration "declare") "declare x=var $k=$x"
+checkBackreferencingDeclaration cmd = CommandCheck (Exactly cmd) check
+  where
+    check t = foldM_ perArg M.empty $ arguments t
+
+    perArg leftArgs t =
+        case t of
+            T_Assignment id _ name idx t -> do
+                warnIfBackreferencing leftArgs $ t:idx
+                return $ M.insert name id leftArgs
+            t -> do
+                warnIfBackreferencing leftArgs [t]
+                return leftArgs
+
+    warnIfBackreferencing backrefs l = do
+        references <- findReferences l
+        let reused = M.intersection backrefs references
+        mapM msg $ M.toList reused
+
+    msg (name, id) = warn id 2318 $ "This assignment is used again in this '" ++ cmd ++ "', but won't have taken effect. Use two '" ++ cmd ++ "'s."
+
+    findReferences list = do
+        cfga <- asks cfgAnalysis
+        let graph = CF.graph cfga
+        let nodesMap = CF.tokenToNodes cfga
+        let nodes = S.unions $ map (\id -> M.findWithDefault S.empty id nodesMap) $ map getId $ list
+        let labels = mapMaybe (G.lab graph) $ S.toList nodes
+        let references = M.fromList $ concatMap refFromLabel labels
+        return references
+
+    refFromLabel lab =
+        case lab of
+            CFApplyEffects effects -> mapMaybe refFromEffect effects
+            _ -> []
+    refFromEffect e =
+        case e of
+            IdTagged id (CFReadVariable name) -> return (name, id)
+            _ -> Nothing
+
+
 return []
 runTests =  $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |])

src/ShellCheck/Checks/ControlFlow.hs

@@ -0,0 +1,101 @@
+{-
+    Copyright 2022 Vidar Holen
+
+    This file is part of ShellCheck.
+    https://www.shellcheck.net
+
+    ShellCheck is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    ShellCheck is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+-}
+{-# LANGUAGE TemplateHaskell #-}
+
+-- Checks that run on the Control Flow Graph (as opposed to the AST)
+-- This is scaffolding for a work in progress.
+
+module ShellCheck.Checks.ControlFlow (checker, optionalChecks, ShellCheck.Checks.ControlFlow.runTests) where
+
+import ShellCheck.AST
+import ShellCheck.ASTLib
+import ShellCheck.CFG hiding (cfgAnalysis)
+import ShellCheck.CFGAnalysis
+import ShellCheck.AnalyzerLib
+import ShellCheck.Data
+import ShellCheck.Interface
+
+import Control.Monad
+import Control.Monad.Reader
+import Data.Graph.Inductive.Graph
+import qualified Data.Map as M
+import qualified Data.Set as S
+import Data.List
+import Data.Maybe
+
+import Test.QuickCheck.All (forAllProperties)
+import Test.QuickCheck.Test (quickCheckWithResult, stdArgs, maxSuccess)
+
+
+optionalChecks :: [CheckDescription]
+optionalChecks = []
+
+-- A check that runs on the entire graph
+type ControlFlowCheck = Analysis
+-- A check invoked once per node, with its (pre,post) data
+type ControlFlowNodeCheck = LNode CFNode -> (ProgramState, ProgramState) -> Analysis
+-- A check invoked once per effect, with its node's (pre,post) data
+type ControlFlowEffectCheck = IdTagged CFEffect -> Node -> (ProgramState, ProgramState) -> Analysis
+
+
+checker :: AnalysisSpec -> Parameters -> Checker
+checker spec params = Checker {
+    perScript = const $ sequence_ controlFlowChecks,
+    perToken = const $ return ()
+}
+
+controlFlowChecks :: [ControlFlowCheck]
+controlFlowChecks = [
+        runNodeChecks controlFlowNodeChecks
+    ]
+
+controlFlowNodeChecks :: [ControlFlowNodeCheck]
+controlFlowNodeChecks = [
+    runEffectChecks controlFlowEffectChecks
+    ]
+
+controlFlowEffectChecks :: [ControlFlowEffectCheck]
+controlFlowEffectChecks = [
+    ]
+
+runNodeChecks :: [ControlFlowNodeCheck] -> ControlFlowCheck
+runNodeChecks perNode = do
+    cfg <- asks cfgAnalysis
+    runOnAll cfg
+  where
+    getData datas n@(node, label) = do
+        (pre, post) <- M.lookup node datas
+        return (n, (pre, post))
+
+    runOn :: (LNode CFNode, (ProgramState, ProgramState)) -> Analysis
+    runOn (node, prepost) = mapM_ (\c -> c node prepost) perNode
+    runOnAll cfg = mapM_ runOn $ mapMaybe (getData $ nodeToData cfg) $ labNodes (graph cfg)
+
+runEffectChecks :: [ControlFlowEffectCheck] -> ControlFlowNodeCheck
+runEffectChecks list = checkNode
+  where
+    checkNode (node, label) prepost =
+        case label of
+            CFApplyEffects effects -> mapM_ (\effect -> mapM_ (\c -> c effect node prepost) list) effects
+            _ -> return ()
+
+
+return []
+runTests =  $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |])

src/ShellCheck/Checks/ShellSupport.hs

@@ -1,5 +1,5 @@
 {-
-    Copyright 2012-2016 Vidar Holen
+    Copyright 2012-2020 Vidar Holen
 
     This file is part of ShellCheck.
     https://www.shellcheck.net
@@ -25,6 +25,7 @@ import ShellCheck.AST
 import ShellCheck.ASTLib
 import ShellCheck.AnalyzerLib
 import ShellCheck.Interface
+import ShellCheck.Prelude
 import ShellCheck.Regex
 
 import Control.Monad
@@ -59,6 +60,8 @@ checks = [
     ,checkBraceExpansionVars
     ,checkMultiDimensionalArrays
     ,checkPS1Assignments
+    ,checkMultipleBangs
+    ,checkBangAfterPipe
     ]
 
 testChecker (ForShell _ t) =
@@ -134,6 +137,8 @@ prop_checkBashisms49= verify checkBashisms "#!/bin/dash\necho $MACHTYPE"
 prop_checkBashisms50 = verify checkBashisms "#!/bin/sh\ncmd >& file"
 prop_checkBashisms51 = verifyNot checkBashisms "#!/bin/sh\ncmd 2>&1"
 prop_checkBashisms52 = verifyNot checkBashisms "#!/bin/sh\ncmd >&2"
+prop_checkBashisms52b = verifyNot checkBashisms "#!/bin/sh\ncmd >& $var"
+prop_checkBashisms52c = verify checkBashisms "#!/bin/sh\ncmd >& $dir/$var"
 prop_checkBashisms53 = verifyNot checkBashisms "#!/bin/sh\nprintf -- -f\n"
 prop_checkBashisms54 = verify checkBashisms "#!/bin/sh\nfoo+=bar"
 prop_checkBashisms55 = verify checkBashisms "#!/bin/sh\necho ${@%foo}"
@@ -180,7 +185,12 @@ prop_checkBashisms95 = verify checkBashisms "#!/bin/sh\necho $_"
 prop_checkBashisms96 = verifyNot checkBashisms "#!/bin/dash\necho $_"
 prop_checkBashisms97 = verify checkBashisms "#!/bin/sh\necho ${var,}"
 prop_checkBashisms98 = verify checkBashisms "#!/bin/sh\necho ${var^^}"
-prop_checkBashisms99 = verifyNot checkBashisms "#!/bin/dash\necho [^f]oo"
+prop_checkBashisms99 = verify checkBashisms "#!/bin/dash\necho [^f]oo"
+prop_checkBashisms100 = verify checkBashisms "read -r"
+prop_checkBashisms101 = verify checkBashisms "read"
+prop_checkBashisms102 = verifyNot checkBashisms "read -r foo"
+prop_checkBashisms103 = verifyNot checkBashisms "read foo"
+prop_checkBashisms104 = verifyNot checkBashisms "read ''"
 checkBashisms = ForShell [Sh, Dash] $ \t -> do
     params <- ask
     kludge params t
@@ -224,7 +234,8 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do
             warnMsg id 3018 $ filter (/= '|') op ++ " is"
     bashism (TA_Binary id "**" _ _) = warnMsg id 3019 "exponentials are"
     bashism (T_FdRedirect id "&" (T_IoFile _ (T_Greater _) _)) = warnMsg id 3020 "&> is"
-    bashism (T_FdRedirect id "" (T_IoFile _ (T_GREATAND _) _)) = warnMsg id 3021 ">& is"
+    bashism (T_FdRedirect id "" (T_IoFile _ (T_GREATAND _) file)) =
+        unless (all isDigit $ onlyLiteralString file) $ warnMsg id 3021 ">& filename (as opposed to >& fd) is"
     bashism (T_FdRedirect id ('{':_) _) = warnMsg id 3022 "named file descriptors are"
     bashism (T_FdRedirect id num _)
         | all isDigit num && length num > 1 = warnMsg id 3023 "FDs outside 0-9 are"
@@ -235,7 +246,7 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do
         where
             file = onlyLiteralString word
             isNetworked = any (`isPrefixOf` file) ["/dev/tcp", "/dev/udp"]
-    bashism (T_Glob id str) | not isDash && "[^" `isInfixOf` str =
+    bashism (T_Glob id str) | "[^" `isInfixOf` str =
             warnMsg id 3026 "^ in place of ! in glob bracket expressions is"
 
     bashism t@(TA_Variable id str _) | isBashVariable str =
@@ -375,6 +386,9 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do
                 let literal = onlyLiteralString format
                 guard $ "%q" `isInfixOf` literal
                 return $ warnMsg (getId format) 3050 "printf %q is"
+
+            when (name == "read" && all isFlag rest) $
+                warnMsg (getId cmd) 3061 "read without a variable is"
       where
         unsupportedCommands = [
             "let", "caller", "builtin", "complete", "compgen", "declare", "dirs", "disown",
@@ -554,5 +568,29 @@ checkPS1Assignments = ForShell [Bash] f
     escapeRegex = mkRegex "\\\\x1[Bb]|\\\\e|\x1B|\\\\033"
 
 
+prop_checkMultipleBangs1 = verify checkMultipleBangs "! ! true"
+prop_checkMultipleBangs2 = verifyNot checkMultipleBangs "! true"
+checkMultipleBangs = ForShell [Dash, Sh] f
+  where
+    f token = case token of
+        T_Banged id (T_Banged _ _) ->
+            err id 2325 "Multiple ! in front of pipelines are a bash/ksh extension. Use only 0 or 1."
+        _ -> return ()
+
+
+prop_checkBangAfterPipe1 = verify checkBangAfterPipe "true | ! true"
+prop_checkBangAfterPipe2 = verifyNot checkBangAfterPipe "true | ( ! true )"
+prop_checkBangAfterPipe3 = verifyNot checkBangAfterPipe "! ! true | true"
+checkBangAfterPipe = ForShell [Dash, Sh, Bash] f
+  where
+    f token = case token of
+        T_Pipeline _ _ cmds -> mapM_ check cmds
+        _ -> return ()
+
+    check token = case token of
+        T_Banged id _ ->
+            err id 2326 "! is not allowed in the middle of pipelines. Use command group as in cmd | { ! cmd; } if necessary."
+        _ -> return ()
+
 return []
 runTests =  $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |])

src/ShellCheck/Data.hs

@@ -2,33 +2,54 @@ module ShellCheck.Data where
 
 import ShellCheck.Interface
 import Data.Version (showVersion)
-import Paths_ShellCheck (version)
 
+
+{-
+If you are here because you saw an error about Paths_ShellCheck in this file,
+simply comment out the import below and define the version as a constant string.
+
+Instead of:
+
+    import Paths_ShellCheck (version)
     shellcheckVersion = showVersion version
 
+Use:
+
+    -- import Paths_ShellCheck (version)
+    shellcheckVersion = "kludge"
+
+-}
+
+import Paths_ShellCheck (version)
+shellcheckVersion = showVersion version  -- VERSIONSTRING
+
+
 internalVariables = [
     -- Generic
     "", "_", "rest", "REST",
 
     -- Bash
     "BASH", "BASHOPTS", "BASHPID", "BASH_ALIASES", "BASH_ARGC",
-    "BASH_ARGV", "BASH_CMDS", "BASH_COMMAND", "BASH_EXECUTION_STRING",
+    "BASH_ARGV", "BASH_ARGV0", "BASH_CMDS", "BASH_COMMAND",
-    "BASH_LINENO", "BASH_REMATCH", "BASH_SOURCE", "BASH_SUBSHELL",
+    "BASH_EXECUTION_STRING", "BASH_LINENO", "BASH_LOADABLES_PATH",
-    "BASH_VERSINFO", "BASH_VERSION", "COMP_CWORD", "COMP_KEY",
+    "BASH_REMATCH", "BASH_SOURCE", "BASH_SUBSHELL", "BASH_VERSINFO",
-    "COMP_LINE", "COMP_POINT", "COMP_TYPE", "COMP_WORDBREAKS",
+    "BASH_VERSION", "COMP_CWORD", "COMP_KEY", "COMP_LINE", "COMP_POINT",
-    "COMP_WORDS", "COPROC", "DIRSTACK", "EUID", "FUNCNAME", "GROUPS",
+    "COMP_TYPE", "COMP_WORDBREAKS", "COMP_WORDS", "COPROC", "DIRSTACK",
-    "HISTCMD", "HOSTNAME", "HOSTTYPE", "LINENO", "MACHTYPE", "MAPFILE",
+    "EPOCHREALTIME", "EPOCHSECONDS", "EUID", "FUNCNAME", "GROUPS", "HISTCMD",
-    "OLDPWD", "OPTARG", "OPTIND", "OSTYPE", "PIPESTATUS", "PPID", "PWD",
+    "HOSTNAME", "HOSTTYPE", "LINENO", "MACHTYPE", "MAPFILE", "OLDPWD",
-    "RANDOM", "READLINE_LINE", "READLINE_POINT", "REPLY", "SECONDS",
+    "OPTARG", "OPTIND", "OSTYPE", "PIPESTATUS", "PPID", "PWD", "RANDOM",
-    "SHELLOPTS", "SHLVL", "UID", "BASH_ENV", "BASH_XTRACEFD", "CDPATH",
+    "READLINE_ARGUMENT", "READLINE_LINE", "READLINE_MARK", "READLINE_POINT",
-    "COLUMNS", "COMPREPLY", "EMACS", "ENV", "FCEDIT", "FIGNORE",
+    "REPLY", "SECONDS", "SHELLOPTS", "SHLVL", "SRANDOM", "UID", "BASH_COMPAT",
+    "BASH_ENV", "BASH_XTRACEFD", "CDPATH", "CHILD_MAX", "COLUMNS",
+    "COMPREPLY", "EMACS", "ENV", "EXECIGNORE", "FCEDIT", "FIGNORE",
     "FUNCNEST", "GLOBIGNORE", "HISTCONTROL", "HISTFILE", "HISTFILESIZE",
     "HISTIGNORE", "HISTSIZE", "HISTTIMEFORMAT", "HOME", "HOSTFILE", "IFS",
-    "IGNOREEOF", "INPUTRC", "LANG", "LC_ALL", "LC_COLLATE", "LC_CTYPE",
+    "IGNOREEOF", "INPUTRC", "INSIDE_EMACS", "LANG", "LC_ALL", "LC_COLLATE",
-    "LC_MESSAGES", "LC_MONETARY", "LC_NUMERIC", "LC_TIME", "LINES", "MAIL",
+    "LC_CTYPE", "LC_MESSAGES", "LC_MONETARY", "LC_NUMERIC", "LC_TIME",
-    "MAILCHECK", "MAILPATH", "OPTERR", "PATH", "POSIXLY_CORRECT",
+    "LINES", "MAIL", "MAILCHECK", "MAILPATH", "OPTERR", "PATH",
-    "PROMPT_COMMAND", "PROMPT_DIRTRIM", "PS1", "PS2", "PS3", "PS4", "SHELL",
+    "POSIXLY_CORRECT", "PROMPT_COMMAND", "PROMPT_DIRTRIM", "PS0", "PS1",
-    "TIMEFORMAT", "TMOUT", "TMPDIR", "auto_resume", "histchars", "COPROC",
+    "PS2", "PS3", "PS4", "SHELL", "TIMEFORMAT", "TMOUT", "TMPDIR",
+    "auto_resume", "histchars",
 
     -- Other
     "USER", "TZ", "TERM", "LOGNAME", "LD_LIBRARY_PATH", "LANGUAGE", "DISPLAY",
@@ -41,15 +62,104 @@ internalVariables = [
     , "FLAGS_ARGC", "FLAGS_ARGV", "FLAGS_ERROR", "FLAGS_FALSE", "FLAGS_HELP",
     "FLAGS_PARENT", "FLAGS_RESERVED", "FLAGS_TRUE", "FLAGS_VERSION",
     "flags_error", "flags_return"
+  ] ++ portageManualInternalVariables
+
+
+portageManualInternalVariables = [
+    -- toolchain settings
+    "CFLAGS", "CXXFLAGS", "CPPFLAGS", "LDFLAGS", "FFLAGS", "FCFLAGS",
+    "CBUILD", "CHOST", "MAKEOPTS",
+    -- TODO: Delete these if we can handle `tc-export CC` implicit export.
+    "CC", "CPP", "CXX",
+
+    -- portage internals
+    "EBUILD_PHASE", "EBUILD_SH_ARGS", "EMERGE_FROM", "FILESDIR",
+    "MERGE_TYPE", "PM_EBUILD_HOOK_DIR", "PORTAGE_ACTUAL_DISTDIR",
+    "PORTAGE_ARCHLIST", "PORTAGE_BASHRC", "PORTAGE_BINPKG_FILE",
+    "PORTAGE_BINPKG_TAR_OPTS", "PORTAGE_BINPKG_TMPFILE", "PORTAGE_BIN_PATH",
+    "PORTAGE_BUILDDIR", "PORTAGE_BUILD_GROUP", "PORTAGE_BUILD_USER",
+    "PORTAGE_BUNZIP2_COMMAND", "PORTAGE_BZIP2_COMMAND", "PORTAGE_COLORMAP",
+    "PORTAGE_CONFIGROOT", "PORTAGE_DEBUG", "PORTAGE_DEPCACHEDIR",
+    "PORTAGE_EBUILD_EXIT_FILE", "PORTAGE_ECLASS_LOCATIONS", "PORTAGE_GID",
+    "PORTAGE_GRPNAME", "PORTAGE_INST_GID", "PORTAGE_INST_UID",
+    "PORTAGE_INTERNAL_CALLER", "PORTAGE_IPC_DAEMON", "PORTAGE_IUSE",
+    "PORTAGE_LOG_FILE", "PORTAGE_MUTABLE_FILTERED_VARS",
+    "PORTAGE_OVERRIDE_EPREFIX", "PORTAGE_PYM_PATH", "PORTAGE_PYTHON",
+    "PORTAGE_PYTHONPATH", "PORTAGE_READONLY_METADATA", "PORTAGE_READONLY_VARS",
+    "PORTAGE_REPO_NAME", "PORTAGE_REPOSITORIES", "PORTAGE_RESTRICT",
+    "PORTAGE_SAVED_READONLY_VARS", "PORTAGE_SIGPIPE_STATUS", "PORTAGE_TMPDIR",
+    "PORTAGE_UPDATE_ENV", "PORTAGE_USERNAME", "PORTAGE_VERBOSE",
+    "PORTAGE_WORKDIR_MODE", "PORTAGE_XATTR_EXCLUDE", "REPLACING_VERSIONS",
+    "REPLACED_BY_VERSION", "__PORTAGE_HELPER", "__PORTAGE_TEST_HARDLINK_LOCKS",
+
+    -- generic ebuilds
+    "A", "ARCH", "BDEPEND", "BOARD_USE", "BROOT", "CATEGORY", "D",
+    "DEFINED_PHASES", "DEPEND", "DESCRIPTION", "DISTDIR", "DOCS", "EAPI",
+    "ECLASS", "ED", "EPREFIX", "EROOT", "ESYSROOT", "EXTRA_ECONF",
+    "EXTRA_EINSTALL", "EXTRA_MAKE", "FEATURES", "FILESDIR", "HOME", "HOMEPAGE",
+    "HTML_DOCS", "INHERITED", "IUSE", "KEYWORDS", "LICENSE", "P", "PATCHES",
+    "PDEPEND", "PF", "PKG_INSTALL_MASK", "PKGUSE", "PN", "PR", "PROPERTIES",
+    "PROVIDES_EXCLUDE", "PV", "PVR", "QA_AM_MAINTAINER_MODE",
+    "QA_CONFIGURE_OPTIONS", "QA_DESKTOP_FILE", "QA_DT_NEEDED", "QA_EXECSTACK",
+    "QA_FLAGS_IGNORED", "QA_MULTILIB_PATHS", "QA_PREBUILT", "QA_PRESTRIPPED",
+    "QA_SONAME", "QA_SONAME_NO_SYMLINK", "QA_TEXTRELS", "QA_WX_LOAD", "RDEPEND",
+    "REPOSITORY", "REQUIRED_USE", "REQUIRES_EXCLUDE", "RESTRICT", "ROOT", "S",
+    "SLOT", "SRC_TEST", "SRC_URI", "STRIP_MASK", "SUBSLOT", "SYSROOT", "T",
+    "WORKDIR",
+
+    -- autotest.eclass declared incorrectly
+    "AUTOTEST_CLIENT_TESTS", "AUTOTEST_CLIENT_SITE_TESTS",
+    "AUTOTEST_SERVER_TESTS", "AUTOTEST_SERVER_SITE_TESTS", "AUTOTEST_CONFIG",
+    "AUTOTEST_DEPS", "AUTOTEST_PROFILERS", "AUTOTEST_CONFIG_LIST",
+    "AUTOTEST_DEPS_LIST", "AUTOTEST_PROFILERS_LIST",
+
+    -- cros-board.eclass declared incorrectly
+    "CROS_BOARDS",
+
+    -- Undeclared cros-kernel2 vars
+    "AFDO_PROFILE_VERSION",
+
+    -- haskell-cabal.eclass declared incorrectly
+    "CABAL_FEATURES",
+
+    -- Undeclared haskell-cabal.eclass vars
+    "CABAL_CORE_LIB_GHC_PV",
+
+    -- Undeclared readme.gentoo.eclass vars
+    "DOC_CONTENTS",
+
+    -- Backwards compatibility perl-module.eclass vars
+    "MODULE_AUTHOR", "MODULE_VERSION",
+
+    -- Undeclared perl-module.eclass vars
+    "mydoc",
+
+    -- python-utils-r1.eclass declared incorrectly
+    "RESTRICT_PYTHON_ABIS", "PYTHON_MODNAME",
+
+    -- ABI variables
+    "ABI", "DEFAULT_ABI",
+
+    -- AFDO variables
+    "AFDO_LOCATION",
+
+    -- Linguas
+    "LINGUAS"
   ]
 
-specialVariablesWithoutSpaces = [
+
-    "$", "-", "?", "!", "#"
+specialIntegerVariables = [
+    "$", "?", "!", "#"
   ]
+
+specialVariablesWithoutSpaces = "-" : specialIntegerVariables
+
 variablesWithoutSpaces = specialVariablesWithoutSpaces ++ [
-    "BASHPID", "BASH_ARGC", "BASH_LINENO", "BASH_SUBSHELL", "EUID", "LINENO",
+    "BASHPID", "BASH_ARGC", "BASH_LINENO", "BASH_SUBSHELL", "EUID",
-    "OPTIND", "PPID", "RANDOM", "SECONDS", "SHELLOPTS", "SHLVL", "UID",
+    "EPOCHREALTIME", "EPOCHSECONDS", "LINENO", "OPTIND", "PPID", "RANDOM",
-    "COLUMNS", "HISTFILESIZE", "HISTSIZE", "LINES"
+    "READLINE_ARGUMENT", "READLINE_MARK", "READLINE_POINT", "SECONDS",
+    "SHELLOPTS", "SHLVL", "SRANDOM", "UID", "COLUMNS", "HISTFILESIZE",
+    "HISTSIZE", "LINES"
 
     -- shflags
     , "FLAGS_ERROR", "FLAGS_FALSE", "FLAGS_TRUE"
@@ -64,7 +174,9 @@ unbracedVariables = specialVariables ++ [
 arrayVariables = [
     "BASH_ALIASES", "BASH_ARGC", "BASH_ARGV", "BASH_CMDS", "BASH_LINENO",
     "BASH_REMATCH", "BASH_SOURCE", "BASH_VERSINFO", "COMP_WORDS", "COPROC",
-    "DIRSTACK", "FUNCNAME", "GROUPS", "MAPFILE", "PIPESTATUS", "COMPREPLY"
+    "DIRSTACK", "FUNCNAME", "GROUPS", "MAPFILE", "PIPESTATUS", "COMPREPLY",
+    -- For Portage
+    "PATCHES"
   ]
 
 commonCommands = [
@@ -95,10 +207,10 @@ commonCommands = [
 
 nonReadingCommands = [
     "alias", "basename", "bg", "cal", "cd", "chgrp", "chmod", "chown",
-    "cp", "du", "echo", "export", "false", "fg", "fuser", "getconf",
+    "cp", "du", "echo", "export", "fg", "fuser", "getconf",
     "getopt", "getopts", "ipcrm", "ipcs", "jobs", "kill", "ln", "ls",
     "locale", "mv", "printf", "ps", "pwd", "renice", "rm", "rmdir",
-    "set", "sleep", "touch", "trap", "true", "ulimit", "unalias", "uname"
+    "set", "sleep", "touch", "trap", "ulimit", "unalias", "uname"
     ]
 
 sampleWords = [
@@ -124,6 +236,11 @@ unaryTestOps = [
     "-o", "-v", "-R"
   ]
 
+-- Variables inspected by Portage tc-export_build_env
+portageBuildEnvVariables = [
+    "CFLAGS", "CXXFLAGS", "CPPFLAGS", "LDFLAGS"
+  ]
+
 shellForExecutable :: String -> Maybe Shell
 shellForExecutable name =
     case name of
@@ -138,3 +255,6 @@ shellForExecutable name =
         _ -> Nothing
 
 flagsForRead = "sreu:n:N:i:p:a:t:"
+flagsForMapfile = "d:n:O:s:u:C:c:t"
+
+declaringCommands = ["local", "declare", "export", "readonly", "typeset", "let"]

src/ShellCheck/Debug.hs

@@ -0,0 +1,314 @@
+{-
+
+This file contains useful functions for debugging and developing ShellCheck.
+
+To invoke them interactively, run:
+
+    cabal repl
+
+At the ghci prompt, enter:
+
+    :load ShellCheck.Debug
+
+You can now invoke the functions. Here are some examples:
+
+    shellcheckString "echo $1"
+    stringToAst "(( x+1 ))"
+    stringToCfg "if foo; then bar; else baz; fi"
+    writeFile "/tmp/test.dot" $ stringToCfgViz "while foo; do bar; done"
+
+The latter file can be rendered to png with GraphViz:
+
+    dot -Tpng /tmp/test.dot > /tmp/test.png
+
+To run all unit tests in a module:
+
+    ShellCheck.Parser.runTests
+    ShellCheck.Analytics.runTests
+
+To run a specific test:
+
+    :load ShellCheck.Analytics
+    prop_checkUuoc3
+
+If you make code changes, reload in seconds at any time with:
+
+    :r
+
+===========================================================================
+
+Crash course in printf debugging in Haskell:
+
+    import Debug.Trace
+
+    greet 0 = return ()
+    -- Print when a function is invoked
+    greet n | trace ("calling greet " ++ show n) False = undefined
+    greet n = do
+        putStrLn "Enter name"
+        name <- getLine
+        -- Print at some point in any monadic function
+        traceM $ "user entered " ++ name
+        putStrLn $ "Hello " ++ name
+        -- Print a value before passing it on
+        greet $ traceShowId (n - 1)
+
+
+===========================================================================
+
+If you want to invoke `ghci` directly, such as on `shellcheck.hs`, to
+debug all of ShellCheck including I/O, you may see an error like this:
+
+    src/ShellCheck/Data.hs:5:1: error:
+        Could not load module ‘Paths_ShellCheck’
+    it is a hidden module in the package ‘ShellCheck-0.8.0’
+
+This can easily be circumvented by running `./setgitversion` or manually
+editing src/ShellCheck/Data.hs to replace the auto-deduced version number
+with a constant string as indicated.
+
+Afterwards, you can run the ShellCheck tool, as if from the shell, with:
+
+    $ ghci shellcheck.hs
+    ghci> runMain ["-x", "file.sh"]
+
+-}
+
+module ShellCheck.Debug () where
+
+import ShellCheck.Analyzer
+import ShellCheck.AST
+import ShellCheck.CFG
+import ShellCheck.Checker
+import ShellCheck.CFGAnalysis as CF
+import ShellCheck.Interface
+import ShellCheck.Parser
+import ShellCheck.Prelude
+
+import Control.Monad
+import Control.Monad.Identity
+import Control.Monad.RWS
+import Control.Monad.Writer
+import Data.Graph.Inductive.Graph as G
+import Data.List
+import Data.Maybe
+import qualified Data.Map as M
+import qualified Data.Set as S
+
+
+-- Run all of ShellCheck (minus output formatters)
+shellcheckString :: String -> CheckResult
+shellcheckString scriptString =
+    runIdentity $ checkScript dummySystemInterface checkSpec
+  where
+    checkSpec :: CheckSpec
+    checkSpec = emptyCheckSpec {
+        csScript = scriptString
+    }
+
+dummySystemInterface :: SystemInterface Identity
+dummySystemInterface = mockedSystemInterface [
+    -- A tiny, fake filesystem for sourced files
+    ("lib/mylib1.sh", "foo=$(cat $1 | wc -l)"),
+    ("lib/mylib2.sh", "bar=42")
+    ]
+
+-- Parameters used when generating Control Flow Graphs
+cfgParams :: CFGParameters
+cfgParams = CFGParameters {
+    cfLastpipe = False,
+    cfPipefail = False,
+    cfAdditionalInitialVariables = []
+}
+
+-- An example script to play with
+exampleScript :: String
+exampleScript = unlines [
+    "#!/bin/sh",
+    "count=0",
+    "for file in *",
+    "do",
+    "  (( count++ ))",
+    "done",
+    "echo $count"
+    ]
+
+-- Parse the script string into ShellCheck's ParseResult
+parseScriptString :: String -> ParseResult
+parseScriptString scriptString =
+    runIdentity $ parseScript dummySystemInterface parseSpec
+  where
+    parseSpec :: ParseSpec
+    parseSpec = newParseSpec {
+        psFilename = "myscript",
+        psScript = scriptString
+    }
+
+
+-- Parse the script string into an Abstract Syntax Tree
+stringToAst :: String -> Token
+stringToAst scriptString =
+    case maybeRoot of
+        Just root -> root
+        Nothing -> error $ "Script failed to parse: " ++ show parserWarnings
+  where
+    parseResult :: ParseResult
+    parseResult = parseScriptString scriptString
+
+    maybeRoot :: Maybe Token
+    maybeRoot = prRoot parseResult
+
+    parserWarnings :: [PositionedComment]
+    parserWarnings = prComments parseResult
+
+
+astToCfgResult :: Token -> CFGResult
+astToCfgResult = buildGraph cfgParams
+
+astToDfa :: Token -> CFGAnalysis
+astToDfa = analyzeControlFlow cfgParams
+
+astToCfg :: Token -> CFGraph
+astToCfg = cfGraph . astToCfgResult
+
+stringToCfg :: String -> CFGraph
+stringToCfg = astToCfg . stringToAst
+
+stringToDfa :: String -> CFGAnalysis
+stringToDfa = astToDfa . stringToAst
+
+cfgToGraphViz :: CFGraph -> String
+cfgToGraphViz = cfgToGraphVizWith show
+
+stringToCfgViz :: String -> String
+stringToCfgViz = cfgToGraphViz . stringToCfg
+
+stringToDfaViz :: String -> String
+stringToDfaViz = dfaToGraphViz . stringToDfa
+
+-- Dump a Control Flow Graph as GraphViz with extended information
+stringToDetailedCfgViz :: String -> String
+stringToDetailedCfgViz scriptString = cfgToGraphVizWith nodeLabel graph
+  where
+    ast :: Token
+    ast = stringToAst scriptString
+
+    cfgResult :: CFGResult
+    cfgResult = astToCfgResult ast
+
+    graph :: CFGraph
+    graph = cfGraph cfgResult
+
+    idToToken :: M.Map Id Token
+    idToToken = M.fromList $ execWriter $ doAnalysis (\c -> tell [(getId c, c)]) ast
+
+    idToNode :: M.Map Id (Node, Node)
+    idToNode = cfIdToRange cfgResult
+
+    nodeToStartIds :: M.Map Node (S.Set Id)
+    nodeToStartIds =
+        M.fromListWith S.union $
+            map (\(id, (start, _)) -> (start, S.singleton id)) $
+                M.toList idToNode
+
+    nodeToEndIds :: M.Map Node (S.Set Id)
+    nodeToEndIds =
+        M.fromListWith S.union $
+            map (\(id, (_, end)) -> (end, S.singleton id)) $
+                M.toList idToNode
+
+    formatId :: Id -> String
+    formatId id = fromMaybe ("Unknown " ++ show id) $ do
+        (OuterToken _ token) <- M.lookup id idToToken
+        firstWord <- words (show token) !!! 0
+        -- Strip off "Inner_"
+        (_ : tokenName) <- return $ dropWhile (/= '_') firstWord
+        return $ tokenName ++ " " ++ show id
+
+    formatGroup :: S.Set Id -> String
+    formatGroup set = intercalate ", " $ map formatId $ S.toList set
+
+    nodeLabel (node, label) = unlines [
+        show node ++ ". " ++ show label,
+        "Begin: " ++ formatGroup (M.findWithDefault S.empty node nodeToStartIds),
+        "End: " ++ formatGroup (M.findWithDefault S.empty node nodeToEndIds)
+        ]
+
+
+-- Dump a Control Flow Graph with Data Flow Analysis as GraphViz
+dfaToGraphViz :: CF.CFGAnalysis -> String
+dfaToGraphViz analysis = cfgToGraphVizWith label $ CF.graph analysis
+  where
+    label (node, label) =
+        let
+            desc = show node ++ ". " ++ show label
+        in
+            fromMaybe ("No DFA available\n\n" ++ desc) $ do
+                (pre, post) <- M.lookup node $ CF.nodeToData analysis
+                return $ unlines [
+                    "Precondition: " ++ show pre,
+                    "",
+                    desc,
+                    "",
+                    "Postcondition: " ++ show post
+                    ]
+
+
+-- Dump an Control Flow Graph to GraphViz with a given node formatter
+cfgToGraphVizWith :: (LNode CFNode -> String) -> CFGraph -> String
+cfgToGraphVizWith nodeLabel graph = concat [
+    "digraph {\n",
+    concatMap dumpNode (labNodes graph),
+    concatMap dumpLink (labEdges graph),
+    tagVizEntries graph,
+    "}\n"
+    ]
+  where
+    dumpNode l@(node, label) = show node ++ " [label=" ++ quoteViz (nodeLabel l) ++ "]\n"
+    dumpLink (from, to, typ) = show from ++ " -> " ++ show to ++ " [style=" ++ quoteViz (edgeStyle typ)  ++ "]\n"
+    edgeStyle CFEFlow = "solid"
+    edgeStyle CFEExit = "bold"
+    edgeStyle CFEFalseFlow = "dotted"
+
+quoteViz str = "\"" ++ escapeViz str ++ "\""
+escapeViz [] = []
+escapeViz (c:rest) =
+    case c of
+        '\"' -> '\\' : '\"' : escapeViz rest
+        '\n' -> '\\' : 'l' : escapeViz rest
+        '\\' -> '\\' : '\\' : escapeViz rest
+        _ -> c : escapeViz rest
+
+
+-- Dump an Abstract Syntax Tree (or branch thereof) to GraphViz format
+astToGraphViz :: Token -> String
+astToGraphViz token = concat [
+    "digraph {\n",
+    formatTree token,
+    "}\n"
+    ]
+  where
+    formatTree :: Token -> String
+    formatTree t = snd $ execRWS (doStackAnalysis push pop t) () []
+
+    push :: Token -> RWS () String [Int] ()
+    push (OuterToken (Id n) inner) = do
+        stack <- get
+        put (n : stack)
+        case stack of
+            [] -> return ()
+            (top:_) -> tell $ show top ++ " -> " ++ show n ++ "\n"
+        tell $ show n ++ " [label=" ++ quoteViz (show n ++ ": " ++ take 32 (show inner)) ++ "]\n"
+
+    pop :: Token -> RWS () String [Int] ()
+    pop _ = modify tail
+
+
+-- For each entry point, set the rank so that they'll align in the graph
+tagVizEntries :: CFGraph -> String
+tagVizEntries graph = "{ rank=same " ++ rank ++ " }"
+  where
+    entries = mapMaybe find $ labNodes graph
+    find (node, CFEntryPoint name) = return (node, name)
+    find _ = Nothing
+    rank = unwords $ map (\(c, _) -> show c) entries

src/ShellCheck/Fixer.hs

@@ -22,6 +22,8 @@
 module ShellCheck.Fixer (applyFix, removeTabStops, mapPositions, Ranged(..), runTests) where
 
 import ShellCheck.Interface
+import ShellCheck.Prelude
+import Control.Monad
 import Control.Monad.State
 import Data.Array
 import Data.List
@@ -35,7 +37,7 @@ class Ranged a where
     end     :: a -> Position
     overlap :: a -> a -> Bool
     overlap x y =
-        (yStart >= xStart && yStart < xEnd) || (yStart < xStart && yEnd > xStart)
+        xEnd > yStart && yEnd > xStart
         where
             yStart = start y
             yEnd = end y
@@ -86,6 +88,7 @@ instance Ranged Replacement where
 instance Monoid Fix where
     mempty = newFix
     mappend = (<>)
+    mconcat = foldl mappend mempty -- fold left to right since <> discards right on overlap
 
 instance Semigroup Fix where
     f1 <> f2 =
@@ -228,7 +231,7 @@ applyReplacement2 rep string = do
 
     let (l1, l2) = tmap posLine originalPos in
         when (l1 /= 1 || l2 /= 1) $
-            error "ShellCheck internal error, please report: bad cross-line fix"
+            error $ pleaseReport "bad cross-line fix"
 
     let replacer = repString rep
     let shift = (length replacer) - (oldEnd - oldStart)

src/ShellCheck/Formatter/CheckStyle.hs

@@ -48,7 +48,7 @@ outputResults cr sys =
     fileGroups = groupWith sourceFile comments
     outputGroup group = do
         let filename = sourceFile (head group)
-        result <- (siReadFile sys) filename
+        result <- siReadFile sys (Just True) filename
         let contents = either (const "") id result
         outputFile filename contents group
 

src/ShellCheck/Formatter/Diff.hs

@@ -38,9 +38,6 @@ import System.FilePath
 
 import Test.QuickCheck
 
-import Debug.Trace
-ltt x = trace (show x) x
-
 format :: FormatterOptions -> IO Formatter
 format options = do
     foundIssues <- newIORef False
@@ -90,7 +87,7 @@ reportResult foundIssues reportedIssues color result sys = do
     mapM_ output $ M.toList fixmap
   where
     output (name, fix) = do
-        file <- (siReadFile sys) name
+        file <- siReadFile sys (Just True) name
         case file of
             Right contents -> do
                 putStrLn $ formatDoc color $ makeDiff name contents fix
@@ -206,10 +203,9 @@ formatDoc color (DiffDoc name lf regions) =
 buildFixMap :: [Fix] -> M.Map String Fix
 buildFixMap fixes = perFile
   where
-    splitFixes = concatMap splitFixByFile fixes
+    splitFixes = splitFixByFile $ mconcat fixes
     perFile = groupByMap (posFile . repStartPos . head . fixReplacements) splitFixes
 
--- There are currently no multi-file fixes, but let's handle it anyways
 splitFixByFile :: Fix -> [Fix]
 splitFixByFile fix = map makeFix $ groupBy sameFile (fixReplacements fix)
   where

src/ShellCheck/Formatter/Format.hs

@@ -28,6 +28,7 @@ import Data.Array
 import Data.List
 import System.IO
 import System.Info
+import System.Environment
 
 -- A formatter that carries along an arbitrary piece of data
 data Formatter = Formatter {
@@ -68,12 +69,14 @@ makeNonVirtual comments contents =
 
 
 shouldOutputColor :: ColorOption -> IO Bool
-shouldOutputColor colorOption = do
+shouldOutputColor colorOption =
-    term <- hIsTerminalDevice stdout
+    case colorOption of
+        ColorAlways -> return True
+        ColorNever -> return False
+        ColorAuto -> do
+            isTerminal <- hIsTerminalDevice stdout
+            term <- lookupEnv "TERM"
             let windows = "mingw" `isPrefixOf` os
-    let isUsableTty = term && not windows
+            let dumbTerm = term `elem` [Just "dumb", Just "", Nothing]
-    let useColor = case colorOption of
+            let isUsableTty = isTerminal && not windows && not dumbTerm
-                       ColorAlways -> True
+            return isUsableTty
-                       ColorNever -> False
-                       ColorAuto -> isUsableTty
-    return useColor

src/ShellCheck/Formatter/GCC.hs

@@ -43,7 +43,7 @@ outputAll cr sys = mapM_ f groups
     f :: [PositionedComment] -> IO ()
     f group = do
         let filename = sourceFile (head group)
-        result <- (siReadFile sys) filename
+        result <- siReadFile sys (Just True) filename
         let contents = either (const "") id result
         outputResult filename contents group
 

src/ShellCheck/Formatter/JSON.hs

@@ -23,6 +23,7 @@ module ShellCheck.Formatter.JSON (format) where
 import ShellCheck.Interface
 import ShellCheck.Formatter.Format
 
+import Control.DeepSeq
 import Data.Aeson
 import Data.IORef
 import Data.Monoid
@@ -103,7 +104,7 @@ collectResult ref cr sys = mapM_ f groups
     comments = crComments cr
     groups = groupWith sourceFile comments
     f :: [PositionedComment] -> IO ()
-    f group = modifyIORef ref (\x -> comments ++ x)
+    f group = deepseq comments $ modifyIORef ref (\x -> comments ++ x)
 
 finish ref = do
     list <- readIORef ref

src/ShellCheck/Formatter/JSON1.hs

@@ -23,6 +23,7 @@ module ShellCheck.Formatter.JSON1 (format) where
 import ShellCheck.Interface
 import ShellCheck.Formatter.Format
 
+import Control.DeepSeq
 import Data.Aeson
 import Data.IORef
 import Data.Monoid
@@ -117,10 +118,10 @@ collectResult ref cr sys = mapM_ f groups
     f :: [PositionedComment] -> IO ()
     f group = do
         let filename = sourceFile (head group)
-        result <- siReadFile sys filename
+        result <- siReadFile sys (Just True) filename
         let contents = either (const "") id result
         let comments' = makeNonVirtual comments contents
-        modifyIORef ref (\x -> comments' ++ x)
+        deepseq comments' $ modifyIORef ref (\x -> comments' ++ x)
 
 finish ref = do
     list <- readIORef ref

src/ShellCheck/Formatter/TTY.hs

@@ -23,6 +23,7 @@ import ShellCheck.Fixer
 import ShellCheck.Interface
 import ShellCheck.Formatter.Format
 
+import Control.DeepSeq
 import Control.Monad
 import Data.Array
 import Data.Foldable
@@ -88,7 +89,7 @@ rankError err = (ranking, cSeverity $ pcComment err, cCode $ pcComment err)
 appendComments errRef comments max = do
     previous <- readIORef errRef
     let current = map (\x -> (rankError x, cCode $ pcComment x, cMessage $ pcComment x)) comments
-    writeIORef errRef . take max . nubBy equal . sort $ previous ++ current
+    writeIORef errRef $! force . take max . nubBy equal . sort $ previous ++ current
   where
     fst3 (x,_,_) = x
     equal x y = fst3 x == fst3 y
@@ -121,7 +122,7 @@ outputResult options ref result sys = do
 
 outputForFile color sys comments = do
     let fileName = sourceFile (head comments)
-    result <- (siReadFile sys) fileName
+    result <- siReadFile sys (Just True) fileName
     let contents = either (const "") id result
     let fileLinesList = lines contents
     let lineCount = length fileLinesList
@@ -174,7 +175,7 @@ showFixedString color comments lineNum fileLines =
 cuteIndent :: PositionedComment -> String
 cuteIndent comment =
     replicate (fromIntegral $ colNo comment - 1) ' ' ++
-        makeArrow ++ " " ++ code (codeNo comment) ++ ": " ++ messageText comment
+        makeArrow ++ " " ++ code (codeNo comment) ++ " (" ++ severityText comment ++ "): " ++ messageText comment
   where
     arrow n = '^' : replicate (fromIntegral $ n-2) '-' ++ "^"
     makeArrow =

src/ShellCheck/Interface.hs

@@ -25,7 +25,7 @@ module ShellCheck.Interface
     , CheckResult(crFilename, crComments)
     , ParseSpec(psFilename, psScript, psCheckSourced, psIgnoreRC, psShellTypeOverride)
     , ParseResult(prComments, prTokenPositions, prRoot)
-    , AnalysisSpec(asScript, asShellType, asFallbackShell, asExecutionMode, asCheckSourced, asTokenPositions, asOptionalChecks)
+    , AnalysisSpec(..)
     , AnalysisResult(arComments)
     , FormatterOptions(foColorOption, foWikiLinkCount)
     , Shell(Ksh, Sh, Bash, Dash)
@@ -39,11 +39,12 @@ module ShellCheck.Interface
     , ColorOption(ColorAuto, ColorAlways, ColorNever)
     , TokenComment(tcId, tcComment, tcFix)
     , emptyCheckResult
-    , newParseResult
-    , newAnalysisSpec
     , newAnalysisResult
+    , newAnalysisSpec
     , newFormatterOptions
+    , newParseResult
     , newPosition
+    , newSystemInterface
     , newTokenComment
     , mockedSystemInterface
     , mockRcFile
@@ -73,16 +74,22 @@ import qualified Data.Map as Map
 
 
 data SystemInterface m = SystemInterface {
-    -- Read a file by filename, or return an error
+    -- | Given:
-    siReadFile :: String -> m (Either ErrorMessage String),
+    --   What annotations say about including external files (if anything)
-    -- Given:
+    --   A resolved filename from siFindSource
+    --   Read the file or return an error
+    siReadFile :: Maybe Bool -> String -> m (Either ErrorMessage String),
+    -- | Given:
     --   the current script,
+    --   what annotations say about including external files (if anything)
     --   a list of source-path annotations in effect,
     --   and a sourced file,
     --   find the sourced file
-    siFindSource :: String -> [String] -> String -> m FilePath,
+    siFindSource :: String -> Maybe Bool -> [String] -> String -> m FilePath,
-    -- Get the configuration file (name, contents) for a filename
+    -- | Get the configuration file (name, contents) for a filename
-    siGetConfig :: String -> m (Maybe (FilePath, String))
+    siGetConfig :: String -> m (Maybe (FilePath, String)),
+    -- | Look up Portage Eclass variables
+    siGetPortageVariables :: m (Map.Map String [String])
 }
 
 -- ShellCheck input and output
@@ -131,6 +138,15 @@ newParseSpec = ParseSpec {
     psShellTypeOverride = Nothing
 }
 
+newSystemInterface :: Monad m => SystemInterface m
+newSystemInterface =
+    SystemInterface {
+        siReadFile = \_ _ -> return $ Left "Not implemented",
+        siFindSource = \_ _ _ name -> return name,
+        siGetConfig = \_ -> return Nothing,
+        siGetPortageVariables = return Map.empty
+    }
+
 -- Parser input and output
 data ParseSpec = ParseSpec {
     psFilename :: String,
@@ -160,6 +176,7 @@ data AnalysisSpec = AnalysisSpec {
     asFallbackShell :: Maybe Shell,
     asExecutionMode :: ExecutionMode,
     asCheckSourced :: Bool,
+    asIsPortage :: Bool,
     asOptionalChecks :: [String],
     asTokenPositions :: Map.Map Id (Position, Position)
 }
@@ -170,6 +187,7 @@ newAnalysisSpec token = AnalysisSpec {
     asFallbackShell = Nothing,
     asExecutionMode = Executed,
     asCheckSourced = False,
+    asIsPortage = False,
     asOptionalChecks = [],
     asTokenPositions = Map.empty
 }
@@ -307,17 +325,17 @@ data ColorOption =
 
 -- For testing
 mockedSystemInterface :: [(String, String)] -> SystemInterface Identity
-mockedSystemInterface files = SystemInterface {
+mockedSystemInterface files = (newSystemInterface :: SystemInterface Identity) {
     siReadFile = rf,
     siFindSource = fs,
     siGetConfig = const $ return Nothing
 }
   where
-    rf file = return $
+    rf _ file = return $
         case find ((== file) . fst) files of
             Nothing -> Left "File not included in mock."
             Just (_, contents) -> Right contents
-    fs _ _ file = return file
+    fs _ _ _ file = return file
 
 mockRcFile rcfile mock = mock {
     siGetConfig = const . return $ Just (".shellcheckrc", rcfile)

src/ShellCheck/Parser.hs

@@ -1,5 +1,5 @@
 {-
-    Copyright 2012-2019 Vidar Holen
+    Copyright 2012-2022 Vidar Holen
 
     This file is part of ShellCheck.
     https://www.shellcheck.net
@@ -24,9 +24,10 @@
 module ShellCheck.Parser (parseScript, runTests) where
 
 import ShellCheck.AST
-import ShellCheck.ASTLib
+import ShellCheck.ASTLib hiding (runTests)
 import ShellCheck.Data
 import ShellCheck.Interface
+import ShellCheck.Prelude
 
 import Control.Applicative ((<*), (*>))
 import Control.Monad
@@ -37,7 +38,6 @@ import Data.Functor
 import Data.List (isPrefixOf, isInfixOf, isSuffixOf, partition, sortBy, intercalate, nub, find)
 import Data.Maybe
 import Data.Monoid
-import Debug.Trace
 import GHC.Exts (sortWith)
 import Prelude hiding (readList)
 import System.IO
@@ -46,7 +46,7 @@ import Text.Parsec.Error
 import Text.Parsec.Pos
 import qualified Control.Monad.Reader as Mr
 import qualified Control.Monad.State as Ms
-import qualified Data.Map as Map
+import qualified Data.Map.Strict as Map
 
 import Test.QuickCheck.All (quickCheckAll)
 
@@ -66,7 +66,7 @@ doubleQuote = char '"'
 variableStart = upper <|> lower <|> oneOf "_"
 variableChars = upper <|> lower <|> digit <|> oneOf "_"
 -- Chars to allow in function names
-functionChars = variableChars <|> oneOf ":+?-./^@"
+functionChars = variableChars <|> oneOf ":+?-./^@,"
 -- Chars to allow in functions using the 'function' keyword
 extendedFunctionChars = functionChars <|> oneOf "[]*=!"
 specialVariable = oneOf (concat specialVariables)
@@ -87,11 +87,23 @@ extglobStart = oneOf extglobStartChars
 unicodeDoubleQuotes = "\x201C\x201D\x2033\x2036"
 unicodeSingleQuotes = "\x2018\x2019"
 
-prop_spacing = isOk spacing "  \\\n # Comment"
+prop_spacing1 = isOk spacing "  \\\n # Comment"
+prop_spacing2 = isOk spacing "# We can continue lines with \\"
+prop_spacing3 = isWarning spacing "   \\\n #  --verbose=true \\"
 spacing = do
-    x <- many (many1 linewhitespace <|> try (string "\\\n" >> return ""))
+    x <- many (many1 linewhitespace <|> continuation)
     optional readComment
     return $ concat x
+  where
+    continuation = do
+        try (string "\\\n")
+        -- The line was continued. Warn if this next line is a comment with a trailing \
+        whitespace <- many linewhitespace
+        optional $ do
+            x <- readComment
+            when ("\\" `isSuffixOf` x) $
+                parseProblem ErrorC 1143 "This backslash is part of a comment and does not continue the line."
+        return whitespace
 
 spacing1 = do
     spacing <- spacing
@@ -198,7 +210,7 @@ getNextIdSpanningTokenList list =
 -- Get the span covered by an id
 getSpanForId :: Monad m => Id -> SCParser m (SourcePos, SourcePos)
 getSpanForId id =
-    Map.findWithDefault (error "Internal error: no position for id. Please report!") id <$>
+    Map.findWithDefault (error $ pleaseReport "no parser span for id") id <$>
         getMap
 
 -- Create a new id with the same span as an existing one
@@ -445,8 +457,8 @@ called s p = do
     pos <- getPosition
     withContext (ContextName pos s) p
 
-withAnnotations anns =
+withAnnotations anns p =
-    withContext (ContextAnnotation anns)
+    if null anns then p else withContext (ContextAnnotation anns) p
 
 readConditionContents single =
     readCondContents `attempting` lookAhead (do
@@ -544,7 +556,7 @@ readConditionContents single =
         notFollowedBy2 (try (spacing >> string "]"))
         x <- readNormalWord
         pos <- getPosition
-        when (endedWith "]" x && notArrayIndex x) $ do
+        when (notArrayIndex x && endedWith "]" x && not (x `containsLiteral` "[")) $ do
             parseProblemAt pos ErrorC 1020 $
                 "You need a space before the " ++ (if single then "]" else "]]") ++ "."
             fail "Missing space before ]"
@@ -560,6 +572,7 @@ readConditionContents single =
             endedWith _ _ = False
             notArrayIndex (T_NormalWord id s@(_:T_Literal _ t:_)) = t /= "["
             notArrayIndex _ = True
+            containsLiteral x s = s `isInfixOf` onlyLiteralString x
 
     readCondAndOp = readAndOrOp TC_And "&&" False <|> readAndOrOp TC_And "-a" True
 
@@ -807,11 +820,13 @@ readArithmeticContents =
         return $ TA_Expansion id pieces
 
     readGroup = do
+        start <- startSpan
         char '('
         s <- readSequence
         char ')'
+        id <- endSpan start
         spacing
-        return s
+        return $ TA_Parentesis id s
 
     readArithTerm = readGroup <|> readVariable <|> readExpansion
 
@@ -929,6 +944,9 @@ prop_readCondition23 = isOk readCondition "[[ -v arr[$var] ]]"
 prop_readCondition25 = isOk readCondition "[[ lex.yy.c -ot program.l ]]"
 prop_readCondition26 = isOk readScript "[[ foo ]]\\\n && bar"
 prop_readCondition27 = not $ isOk readConditionCommand "[[ x ]] foo"
+prop_readCondition28 = isOk readCondition "[[ x = [\"$1\"] ]]"
+prop_readCondition29 = isOk readCondition "[[ x = [*] ]]"
+
 readCondition = called "test expression" $ do
     opos <- getPosition
     start <- startSpan
@@ -972,12 +990,17 @@ prop_readAnnotation4 = isWarning readAnnotation "# shellcheck cats=dogs disable=
 prop_readAnnotation5 = isOk readAnnotation "# shellcheck disable=SC2002 # All cats are precious\n"
 prop_readAnnotation6 = isOk readAnnotation "# shellcheck disable=SC1234 # shellcheck foo=bar\n"
 prop_readAnnotation7 = isOk readAnnotation "# shellcheck disable=SC1000,SC2000-SC3000,SC1001\n"
+prop_readAnnotation8 = isOk readAnnotation "# shellcheck disable=all\n"
+prop_readAnnotation9 = isOk readAnnotation "# shellcheck source='foo bar' source-path=\"baz etc\"\n"
+prop_readAnnotation10 = isOk readAnnotation "# shellcheck disable='SC1234,SC2345' enable=\"foo\" shell='bash'\n"
+prop_readAnnotation11 = isOk (readAnnotationWithoutPrefix False) "external-sources='true'"
+
 readAnnotation = called "shellcheck directive" $ do
     try readAnnotationPrefix
     many1 linewhitespace
-    readAnnotationWithoutPrefix
+    readAnnotationWithoutPrefix True
 
-readAnnotationWithoutPrefix = do
+readAnnotationWithoutPrefix sandboxed = do
     values <- many1 readKey
     optional readAnyComment
     void linefeed <|> eof <|> do
@@ -987,13 +1010,24 @@ readAnnotationWithoutPrefix = do
     many linewhitespace
     return $ concat values
   where
+    plainOrQuoted p = quoted p <|> p
+    quoted p = do
+        c <- oneOf "'\""
+        start <- getPosition
+        str <- many1 $ noneOf (c:"\n")
+        char c <|> fail "Missing terminating quote for directive."
+        subParse start p str
     readKey = do
         keyPos <- getPosition
         key <- many1 (letter <|> char '-')
         char '=' <|> fail "Expected '=' after directive key"
         annotations <- case key of
-            "disable" -> readRange `sepBy` char ','
+            "disable" -> plainOrQuoted $ readElement `sepBy` char ','
               where
+                readElement = readRange <|> readAll
+                readAll = do
+                    string "all"
+                    return $ DisableComment 0 1000000
                 readRange = do
                     from <- readCode
                     to <- choice [ char '-' *> readCode, return $ from+1 ]
@@ -1003,26 +1037,41 @@ readAnnotationWithoutPrefix = do
                     int <- many1 digit
                     return $ read int
 
-            "enable" -> readName `sepBy` char ','
+            "enable" -> plainOrQuoted $ readName `sepBy` char ','
               where
                 readName = EnableComment <$> many1 (letter <|> char '-')
 
             "source" -> do
-                filename <- many1 $ noneOf " \n"
+                filename <- quoted (many1 anyChar) <|> (many1 $ noneOf " \n")
                 return [SourceOverride filename]
 
             "source-path" -> do
-                dirname <- many1 $ noneOf " \n"
+                dirname <- quoted (many1 anyChar) <|> (many1 $ noneOf " \n")
                 return [SourcePath dirname]
 
             "shell" -> do
                 pos <- getPosition
-                shell <- many1 $ noneOf " \n"
+                shell <- quoted (many1 anyChar) <|> (many1 $ noneOf " \n")
                 when (isNothing $ shellForExecutable shell) $
                     parseNoteAt pos ErrorC 1103
                         "This shell type is unknown. Use e.g. sh or bash."
                 return [ShellOverride shell]
 
+            "external-sources" -> do
+                pos <- getPosition
+                value <- plainOrQuoted $ many1 letter
+                case value of
+                    "true" ->
+                        if sandboxed
+                        then do
+                            parseNoteAt pos ErrorC 1144 "external-sources can only be enabled in .shellcheckrc, not in individual files."
+                            return []
+                        else return [ExternalSources True]
+                    "false" -> return [ExternalSources False]
+                    _ -> do
+                        parseNoteAt pos ErrorC 1145 "Unknown external-sources value. Expected true/false."
+                        return []
+
             _ -> do
                 parseNoteAt keyPos WarningC 1107 "This directive is unknown. It will be ignored."
                 anyChar `reluctantlyTill` whitespace
@@ -1039,6 +1088,7 @@ readComment = do
     unexpecting "shellcheck annotation" readAnnotationPrefix
     readAnyComment
 
+prop_readAnyComment = isOk readAnyComment "# Comment"
 readAnyComment = do
     char '#'
     many $ noneOf "\r\n"
@@ -1359,6 +1409,8 @@ prop_readGlob5 = isOk readGlob "[^[:alpha:]1-9]"
 prop_readGlob6 = isOk readGlob "[\\|]"
 prop_readGlob7 = isOk readGlob "[^[]"
 prop_readGlob8 = isOk readGlob "[*?]"
+prop_readGlob9 = isOk readGlob "[!]^]"
+prop_readGlob10 = isOk readGlob "[]]"
 readGlob = readExtglob <|> readSimple <|> readClass <|> readGlobbyLiteral
     where
         readSimple = do
@@ -1366,22 +1418,25 @@ readGlob = readExtglob <|> readSimple <|> readClass <|> readGlobbyLiteral
             c <- oneOf "*?"
             id <- endSpan start
             return $ T_Glob id [c]
-        -- Doesn't handle weird things like [^]a] and [$foo]. fixme?
         readClass = try $ do
             start <- startSpan
             char '['
-            s <- many1 (predefined <|> readNormalLiteralPart "]" <|> globchars)
+            negation <- charToString (oneOf "!^") <|> return ""
+            leadingBracket <- charToString (oneOf "]") <|> return ""
+            s <- many (predefined <|> readNormalLiteralPart "]" <|> globchars)
+            guard $ not (null leadingBracket) || not (null s)
             char ']'
             id <- endSpan start
-            return $ T_Glob id $ "[" ++ concat s ++ "]"
+            return $ T_Glob id $ "[" ++ concat (negation:leadingBracket:s) ++ "]"
           where
-           globchars = fmap return . oneOf $ "!$[" ++ extglobStartChars
+           globchars = charToString $ oneOf $ "![" ++ extglobStartChars
            predefined = do
               try $ string "[:"
               s <- many1 letter
               string ":]"
               return $ "[:" ++ s ++ ":]"
 
+        charToString = fmap return
         readGlobbyLiteral = do
             start <- startSpan
             c <- extglobStart <|> char '['
@@ -1404,6 +1459,8 @@ readNormalEscaped = called "escaped char" $ do
     do
         next <- quotable <|> oneOf "?*@!+[]{}.,~#"
         when (next == ' ') $ checkTrailingSpaces pos <|> return ()
+        -- Check if this line is followed by a commented line with a trailing backslash
+        when (next == '\n') $ try . lookAhead $ void spacing
         return $ if next == '\n' then "" else [next]
       <|>
         do
@@ -1471,7 +1528,6 @@ readSingleEscaped = do
 
     case x of
         '\'' -> parseProblemAt pos InfoC 1003 "Want to escape a single quote? echo 'This is how it'\\''s done'.";
-        '\n' -> parseProblemAt pos InfoC 1004 "This backslash+linefeed is literal. Break outside single quotes if you just want to break the line."
         _ -> return ()
 
     return [s]
@@ -1875,7 +1931,7 @@ readPendingHereDocs = do
                         -- The end token is just a prefix
                         skipLine
                     | hasTrailer ->
-                        error "ShellCheck bug, please report (here doc trailer)."
+                        error $ pleaseReport "unexpected heredoc trailer"
 
                     -- The following cases assume no trailing text:
                     | dashed == Undashed && (not $ null leadingSpace) -> do
@@ -1981,12 +2037,14 @@ readHereString = called "here string" $ do
     word <- readNormalWord
     return $ T_HereString id word
 
+prop_readNewlineList1 = isOk readScript "&> /dev/null echo foo"
 readNewlineList =
     many1 ((linefeed <|> carriageReturn) `thenSkip` spacing) <* checkBadBreak
   where
     checkBadBreak = optional $ do
                 pos <- getPosition
                 try $ lookAhead (oneOf "|&") --  See if the next thing could be |, || or &&
+                notFollowedBy2 (string "&>") --  Except &> or &>> which is valid
                 parseProblemAt pos ErrorC 1133
                     "Unexpected start of line. If breaking lines, |/||/&& should be at the end of the previous one."
 readLineBreak = optional readNewlineList
@@ -2046,6 +2104,7 @@ prop_readSimpleCommand4 = isOk readSimpleCommand "typeset -a foo=(lol)"
 prop_readSimpleCommand5 = isOk readSimpleCommand "time if true; then echo foo; fi"
 prop_readSimpleCommand6 = isOk readSimpleCommand "time -p ( ls -l; )"
 prop_readSimpleCommand7 = isOk readSimpleCommand "\\ls"
+prop_readSimpleCommand7b = isOk readSimpleCommand "\\:"
 prop_readSimpleCommand8 = isWarning readSimpleCommand "// Lol"
 prop_readSimpleCommand9 = isWarning readSimpleCommand "/* Lolbert */"
 prop_readSimpleCommand10 = isWarning readSimpleCommand "/**** Lolbert */"
@@ -2053,6 +2112,7 @@ prop_readSimpleCommand11 = isOk readSimpleCommand "/\\* foo"
 prop_readSimpleCommand12 = isWarning readSimpleCommand "elsif foo"
 prop_readSimpleCommand13 = isWarning readSimpleCommand "ElseIf foo"
 prop_readSimpleCommand14 = isWarning readSimpleCommand "elseif[$i==2]"
+prop_readSimpleCommand15 = isWarning readSimpleCommand "trap 'foo\"bar' INT"
 readSimpleCommand = called "simple command" $ do
     prefix <- option [] readCmdPrefix
     skipAnnotationAndWarn
@@ -2082,9 +2142,12 @@ readSimpleCommand = called "simple command" $ do
             id2 <- getNewIdFor id1
 
             let result = makeSimpleCommand id1 id2 prefix [cmd] suffix
-            if isCommand ["source", "."] cmd
+            case () of
-                then readSource result
+                _ | isCommand ["source", "."] cmd -> readSource result
-                else return result
+                _ | isCommand ["trap"] cmd -> do
+                        syntaxCheckTrap result
+                        return result
+                _ -> return result
   where
     isCommand strings (T_NormalWord _ [T_Literal _ s]) = s `elem` strings
     isCommand _ _ = False
@@ -2104,6 +2167,17 @@ readSimpleCommand = called "simple command" $ do
                     parseProblemAtId (getId cmd) ErrorC 1131 "Use 'elif' to start another branch."
             _ -> return ()
 
+    syntaxCheckTrap cmd =
+        case cmd of
+            (T_Redirecting _ _ (T_SimpleCommand _ _ (cmd:arg:_))) -> checkArg arg (getLiteralString arg)
+            _ -> return ()
+      where
+        checkArg _ Nothing = return ()
+        checkArg arg (Just ('-':_)) = return ()
+        checkArg arg (Just str) = do
+            (start,end) <- getSpanForId (getId arg)
+            subParse start (tryWithErrors (readCompoundListOrEmpty >> verifyEof) <|> return ()) str
+
     commentWarning id =
         parseProblemAtId id ErrorC 1127 "Was this intended as a comment? Use # in sh."
 
@@ -2153,10 +2227,12 @@ readSource t@(T_Redirecting _ _ (T_SimpleCommand cmdId _ (cmd:file':rest'))) = d
                     if filename == "/dev/null" -- always allow /dev/null
                     then return (Right "", filename)
                     else do
+                        allAnnotations <- getCurrentAnnotations True
                         currentScript <- Mr.asks currentFilename
-                        paths <- mapMaybe getSourcePath <$> getCurrentAnnotations True
+                        let paths = mapMaybe getSourcePath allAnnotations
-                        resolved <- system $ siFindSource sys currentScript paths filename
+                        let externalSources = listToMaybe $ mapMaybe getExternalSources allAnnotations
-                        contents <- system $ siReadFile sys resolved
+                        resolved <- system $ siFindSource sys currentScript externalSources paths filename
+                        contents <- system $ siReadFile sys externalSources resolved
                         return (contents, resolved)
                 case input of
                     Left err -> do
@@ -2190,6 +2266,11 @@ readSource t@(T_Redirecting _ _ (T_SimpleCommand cmdId _ (cmd:file':rest'))) = d
             SourcePath x -> Just x
             _ -> Nothing
 
+    getExternalSources t =
+        case t of
+            ExternalSources b -> Just b
+            _ -> Nothing
+
     -- If the word has a single expansion as the directory, try stripping it
     -- This affects `$foo/bar` but not `${foo}-dir/bar` or `/foo/$file`
     stripDynamicPrefix word =
@@ -2202,22 +2283,31 @@ readSource t@(T_Redirecting _ _ (T_SimpleCommand cmdId _ (cmd:file':rest'))) = d
 
     subRead name script =
         withContext (ContextSource name) $
-            inSeparateContext $
+            inSeparateContext $ do
-                subParse (initialPos name) (readScriptFile True) script
+                oldState <- getState
+                setState $ oldState { pendingHereDocs = [] }
+                result <- subParse (initialPos name) (readScriptFile True) script
+                newState <- getState
+                setState $ newState { pendingHereDocs = pendingHereDocs oldState }
+                return result
 readSource t = return t
 
 
 prop_readPipeline = isOk readPipeline "! cat /etc/issue | grep -i ubuntu"
 prop_readPipeline2 = isWarning readPipeline "!cat /etc/issue | grep -i ubuntu"
 prop_readPipeline3 = isOk readPipeline "for f; do :; done|cat"
+prop_readPipeline4 = isOk readPipeline "! ! true"
+prop_readPipeline5 = isOk readPipeline "true | ! true"
 readPipeline = do
     unexpecting "keyword/token" readKeyword
-    do
+    readBanged readPipeSequence
+
+readBanged parser = do
+    pos <- getPosition
     (T_Bang id) <- g_Bang
-        pipe <- readPipeSequence
+    next <- readBanged parser
-        return $ T_Banged id pipe
+    return $ T_Banged id next
-      <|>
+ <|> parser
-        readPipeSequence
 
 prop_readAndOr = isOk readAndOr "grep -i lol foo || exit 1"
 prop_readAndOr1 = isOk readAndOr "# shellcheck disable=1\nfoo"
@@ -2233,7 +2323,7 @@ readAndOr = do
         parseProblemAt apos ErrorC 1123 "ShellCheck directives are only valid in front of complete compound commands, like 'if', not e.g. individual 'elif' branches."
 
     andOr <- withAnnotations annotations $
-        chainr1 readPipeline $ do
+        chainl1 readPipeline $ do
             op <- g_AND_IF <|> g_OR_IF
             readLineBreak
             return $ case op of T_AND_IF id -> T_AndIf id
@@ -2273,7 +2363,7 @@ readTerm = do
 
 readPipeSequence = do
     start <- startSpan
-    (cmds, pipes) <- sepBy1WithSeparators readCommand
+    (cmds, pipes) <- sepBy1WithSeparators (readBanged readCommand)
                         (readPipe `thenSkip` (spacing >> readLineBreak))
     id <- endSpan start
     spacing
@@ -2303,10 +2393,14 @@ readCommand = choice [
     ]
 
 readCmdName = do
+    -- If the command name is `!` then
+    optional . lookAhead . try $ do
+        char '!'
+        whitespace
     -- Ignore alias suppression
     optional . try $ do
         char '\\'
-        lookAhead $ variableChars
+        lookAhead $ variableChars <|> oneOf ":."
     readCmdWord
 
 readCmdWord = do
@@ -2434,16 +2528,29 @@ readBraceGroup = called "brace group" $ do
     spacing
     return $ T_BraceGroup id list
 
-prop_readBatsTest = isOk readBatsTest "@test 'can parse' {\n  true\n}"
+prop_readBatsTest1 = isOk readBatsTest "@test 'can parse' {\n  true\n}"
+prop_readBatsTest2 = isOk readBatsTest "@test random text !(@*$Y&! {\n  true\n}"
+prop_readBatsTest3 = isOk readBatsTest "@test foo { bar { baz {\n  true\n}"
+prop_readBatsTest4 = isNotOk readBatsTest "@test foo \n{\n true\n}"
 readBatsTest = called "bats @test" $ do
     start <- startSpan
     try $ string "@test "
     spacing
-    name <- readNormalWord
+    name <- readBatsName
     spacing
     test <- readBraceGroup
     id <- endSpan start
     return $ T_BatsTest id name test
+  where
+    readBatsName = do
+        line <- try . lookAhead $ many1 $ noneOf "\n"
+        let name = reverse $ f $ reverse line
+        string name
+
+    -- We want everything before the last " {" in a string, so we find everything after "{ " in its reverse
+    f ('{':' ':rest) = dropWhile isSpace rest
+    f (a:rest) = f rest
+    f [] = ""
 
 prop_readWhileClause = isOk readWhileClause "while [[ -e foo ]]; do sleep 1; done"
 readWhileClause = called "while loop" $ do
@@ -2472,7 +2579,7 @@ readDoGroup kwId = do
         parseProblem ErrorC 1058 "Expected 'do'."
         return "Expected 'do'"
 
-    acceptButWarn g_Semi ErrorC 1059 "No semicolons directly after 'do'."
+    acceptButWarn g_Semi ErrorC 1059 "Semicolon is not allowed directly after 'do'. You can just delete it."
     allspacing
 
     optional (do
@@ -2792,7 +2899,7 @@ readLetSuffix = many1 (readIoRedirect <|> try readLetExpression <|> readCmdWord)
         startPos <- getPosition
         expression <- readStringForParser readCmdWord
         let (unQuoted, newPos) = kludgeAwayQuotes expression startPos
-        subParse newPos readArithmeticContents unQuoted
+        subParse newPos (readArithmeticContents <* eof) unQuoted
 
     kludgeAwayQuotes :: String -> SourcePos -> (String, SourcePos)
     kludgeAwayQuotes s p =
@@ -3179,7 +3286,7 @@ prop_readConfigKVs4 = isOk readConfigKVs "\n\n\n\n\t \n"
 prop_readConfigKVs5 = isOk readConfigKVs "# shellcheck accepts annotation-like comments in rc files\ndisable=1234"
 readConfigKVs = do
     anySpacingOrComment
-    annotations <- many (readAnnotationWithoutPrefix <* anySpacingOrComment)
+    annotations <- many (readAnnotationWithoutPrefix False <* anySpacingOrComment)
     eof
     return $ concat annotations
 anySpacingOrComment =
@@ -3191,23 +3298,30 @@ prop_readScript3 = isWarning readScript "#!/bin/bash\necho hello\xA0world"
 prop_readScript4 = isWarning readScript "#!/usr/bin/perl\nfoo=("
 prop_readScript5 = isOk readScript "#!/bin/bash\n#This is an empty script\n\n"
 prop_readScript6 = isOk readScript "#!/usr/bin/env -S X=FOO bash\n#This is an empty script\n\n"
+prop_readScript7 = isOk readScript "#!/bin/zsh\n# shellcheck disable=SC1071\nfor f (a b); echo $f\n"
 readScriptFile sourced = do
     start <- startSpan
     pos <- getPosition
-    optional $ do
-        readUtf8Bom
-        parseProblem ErrorC 1082
-            "This file has a UTF-8 BOM. Remove it with: LC_CTYPE=C sed '1s/^...//' < yourscript ."
-    shebang <- readShebang <|> readEmptyLiteral
-    let (T_Literal _ shebangString) = shebang
-    allspacing
-    annotationStart <- startSpan
-    fileAnnotations <- readAnnotations
     rcAnnotations <- if sourced
                      then return []
                      else do
                         filename <- Mr.asks currentFilename
                         readConfigFile filename
+
+    -- Put the rc annotations on the stack so that one can ignore e.g. SC1084 in .shellcheckrc
+    withAnnotations rcAnnotations $ do
+        hasBom <- wasIncluded readUtf8Bom
+        shebang <- readShebang <|> readEmptyLiteral
+        let (T_Literal _ shebangString) = shebang
+        allspacing
+        annotationStart <- startSpan
+        fileAnnotations <- readAnnotations
+
+        -- Similarly put the filewide annotations on the stack to allow earlier suppression
+        withAnnotations fileAnnotations $ do
+            when (hasBom) $
+                parseProblemAt pos ErrorC 1082
+                    "This file has a UTF-8 BOM. Remove it with: LC_CTYPE=C sed '1s/^...//' < yourscript ."
             let annotations = fileAnnotations ++ rcAnnotations
             annotationId <- endSpan annotationStart
             let shellAnnotationSpecified =
@@ -3216,11 +3330,12 @@ readScriptFile sourced = do
             let ignoreShebang = shellAnnotationSpecified || shellFlagSpecified
 
             unless ignoreShebang $
-        verifyShebang pos (getShell shebangString)
+                verifyShebang pos (executableFromShebang shebangString)
-    if ignoreShebang || isValidShell (getShell shebangString) /= Just False
+            if ignoreShebang || isValidShell (executableFromShebang shebangString) /= Just False
               then do
-            commands <- withAnnotations annotations readCompoundListOrEmpty
+                    commands <- readCompoundListOrEmpty
                     id <- endSpan start
+                    readPendingHereDocs
                     verifyEof
                     let script = T_Annotation annotationId annotations $
                                     T_Script id shebang commands
@@ -3231,17 +3346,6 @@ readScriptFile sourced = do
                     return $ T_Script id shebang []
 
   where
-    basename s = reverse . takeWhile (/= '/') . reverse $ s
-    skipFlags = dropWhile ("-" `isPrefixOf`)
-    getShell sb =
-        case words sb of
-            [] -> ""
-            [x] -> basename x
-            (first:args) ->
-                if basename first == "env"
-                    then fromMaybe "" $ find (notElem '=') $ skipFlags args
-                    else basename first
-
     verifyShebang pos s = do
         case isValidShell s of
             Just True -> return ()
@@ -3270,6 +3374,7 @@ readScriptFile sourced = do
         "awk",
         "csh",
         "expect",
+        "fish",
         "perl",
         "python",
         "ruby",
@@ -3332,16 +3437,6 @@ parsesCleanly parser string = runIdentity $ do
             return $ Just . null $ parseNotes userState ++ parseProblems systemState
         (Left _, _) -> return Nothing
 
--- For printf debugging: print the value of an expression
--- Example: return $ dump $ T_Literal id [c]
-dump :: Show a => a -> a
-dump x = trace (show x) x
-
--- Like above, but print a specific expression:
--- Example: return $ dumps ("Returning: " ++ [c])  $ T_Literal id [c]
-dumps :: Show x => x -> a -> a
-dumps t = trace (show t)
-
 parseWithNotes parser = do
     item <- parser
     state <- getState
@@ -3421,9 +3516,9 @@ notesForContext list = zipWith ($) [first, second] $ filter isName list
 
 -- Go over all T_UnparsedIndex and reparse them as either arithmetic or text
 -- depending on declare -A statements.
-reparseIndices root =
+reparseIndices root = process root
-   analyze blank blank f root
   where
+    process = analyze blank blank f
     associative = getAssociativeArrays root
     isAssociative s = s `elem` associative
     f (T_Assignment id mode name indices value) = do
@@ -3448,8 +3543,9 @@ reparseIndices root =
 
     fixAssignmentIndex name word =
         case word of
-            T_UnparsedIndex id pos src ->
+            T_UnparsedIndex id pos src -> do
-                parsed name pos src
+                idx <- parsed name pos src
+                process idx -- Recursively parse for cases like x[y[z=1]]=1
             _ -> return word
 
     parsed name pos src =

src/ShellCheck/PortageVariables.hs

@@ -0,0 +1,138 @@
+{-# LANGUAGE ApplicativeDo #-}
+{-# LANGUAGE LambdaCase #-}
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE TupleSections #-}
+
+module ShellCheck.PortageVariables (
+    readPortageVariables
+  ) where
+
+import ShellCheck.Regex
+
+import Control.Exception
+import Control.Monad
+import Data.Maybe
+import System.Directory (listDirectory)
+import System.Exit (ExitCode(..))
+import System.FilePath
+import System.IO
+import System.Process
+
+import qualified Data.ByteString as B
+import qualified Data.Map as M
+
+type RepoName = String
+type RepoPath = String
+type EclassName = String
+type EclassVar = String
+
+-- | This is used for looking up what eclass variables are inherited,
+--   keyed by the name of the eclass.
+type EclassMap = M.Map EclassName [EclassVar]
+
+data Repository = Repository
+    { repositoryName :: RepoName
+    , repositoryLocation :: RepoPath
+    , repositoryEclasses :: [Eclass]
+    } deriving (Show, Eq, Ord)
+
+data Eclass = Eclass
+    { eclassName :: EclassName
+    , eclassVars :: [EclassVar]
+    } deriving (Show, Eq, Ord)
+
+readPortageVariables :: IO (M.Map String [String])
+readPortageVariables = portageVariables <$> scanRepos
+
+-- | Map from eclass names to a list of eclass variables
+portageVariables :: [Repository] -> EclassMap
+portageVariables = foldMap $ foldMap go . repositoryEclasses
+  where
+    go e = M.singleton (eclassName e) (eclassVars e)
+
+-- | Run @portageq@ to gather a list of repo names and paths, then scan each
+--   one for eclasses and ultimately eclass metadata.
+scanRepos :: IO [Repository]
+scanRepos = do
+    let cmd = "portageq"
+    let args = ["repos_config", "/"]
+    out <- runOrDie cmd args
+    forM (reposParser $ lines out) $ \(n,p) -> Repository n p <$> getEclasses p
+
+-- | Get the name of the repo and its path from blocks outputted by
+--   @portageq@. If the path doesn't exist, this will return @Nothing@.
+reposParser :: [String] -> [(RepoName, RepoPath)]
+reposParser = f ""
+  where
+    segmentRegex = mkRegex "^\\[(.*)\\].*"
+    locationRegex = mkRegex "^[[:space:]]*location[[:space:]]*=[[:space:]]*(.*)[[:space:]]*$"
+    f name [] = []
+    f name (line:rest) =
+        case (matchRegex segmentRegex line, matchRegex locationRegex line) of
+            (Just [next], _) -> f next rest
+            (_, Just [location]) -> (name, location) : f name rest
+            _ -> f name rest
+
+-- | Scan the repo path for @*.eclass@ files in @eclass/@, then run
+--   'eclassParser' on each of them to produce @[Eclass]@.
+--
+--   If the @eclass/@ directory doesn't exist, the scan is skipped for that
+--   repo.
+getEclasses :: RepoPath -> IO [Eclass]
+getEclasses repoLoc = do
+    let eclassDir = repoLoc </> "eclass"
+
+    files <- handle catcher $ listDirectory eclassDir
+    let names = filter (\(_, e) -> e == ".eclass") $ map splitExtension files
+
+    forM (names :: [(String, String)]) $ \(name, ext) -> do
+        contents <- withFile (eclassDir </> name <.> ext) ReadMode readFully
+        return $ Eclass name $ eclassParser (lines contents)
+
+  where
+    catcher :: IOException -> IO [String]
+    catcher e = do
+        hPutStrLn stderr $ "Unable to find .eclass files: " ++ show e
+        return []
+
+-- | Scan a @.eclass@ file for any @@@ECLASS_VARIABLE:@ comments, generating
+--   a list of eclass variables.
+eclassParser :: [String] -> [String]
+eclassParser lines = mapMaybe match lines
+  where
+    varRegex = mkRegex "^[[:space:]]*#[[:space:]]*@ECLASS_VARIABLE:[[:space:]]*([^[:space:]]*)[[:space:]]*$"
+    match str = head <$> matchRegex varRegex str
+
+-- | Run the command and return the full stdout string (stdin is ignored).
+--
+--   If the command exits with a non-zero exit code, this will throw an
+--   error including the captured contents of stdout and stderr.
+runOrDie :: FilePath -> [String] -> IO String
+runOrDie cmd args = bracket acquire release $ \(_,o,e,p) -> do
+    ot <- readFully (fromJust o)
+    et <- readFully (fromJust e)
+    ec <- waitForProcess p
+    case ec of
+        ExitSuccess -> pure ot
+        ExitFailure i -> fail $ unlines $ map unwords
+            $   [ [ show cmd ]
+                    ++ map show args
+                    ++ [ "failed with exit code", show i]
+                , [ "stdout:" ], [ ot ]
+                , [ "stderr:" ], [ et ]
+                ]
+  where
+    acquire = createProcess (proc cmd args)
+        { std_in = NoStream
+        , std_out = CreatePipe
+        , std_err = CreatePipe
+        }
+    release (i,o,e,p) = do
+        _ <- waitForProcess p
+        forM_ [i,o,e] $ mapM_ hClose
+
+readFully :: Handle -> IO String
+readFully handle = do
+    hSetBinaryMode handle True
+    str <- hGetContents handle
+    length str `seq` return str

src/ShellCheck/Prelude.hs

@@ -0,0 +1,51 @@
+{-
+    Copyright 2022 Vidar Holen
+
+    This file is part of ShellCheck.
+    https://www.shellcheck.net
+
+    ShellCheck is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    ShellCheck is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+-}
+
+-- Generic basic utility functions
+module ShellCheck.Prelude where
+
+import Data.Semigroup
+
+
+-- Get element 0 or a default. Like `head` but safe.
+headOrDefault _ (a:_) = a
+headOrDefault def _   = def
+
+-- Get the last element or a default. Like `last` but safe.
+lastOrDefault def [] = def
+lastOrDefault _ list = last list
+
+--- Get element n of a list, or Nothing. Like `!!` but safe.
+(!!!) list i =
+    case drop i list of
+        []    -> Nothing
+        (r:_) -> Just r
+
+
+-- Like mconcat but for Semigroups
+sconcat1 :: (Semigroup t) => [t] -> t
+sconcat1 [x] = x
+sconcat1 (x:xs) = x <> sconcat1 xs
+
+sconcatOrDefault def [] = def
+sconcatOrDefault _ list = sconcat1 list
+
+-- For more actionable "impossible" errors
+pleaseReport str = "ShellCheck internal error, please report: " ++ str

stack.yaml

@@ -2,7 +2,7 @@
 # For more information, see: https://docs.haskellstack.org/en/stable/yaml_configuration/
 
 # Specifies the GHC version and set of packages available (e.g., lts-3.5, nightly-2015-09-21, ghc-7.10.2)
-resolver: lts-13.26
+resolver: lts-18.15
 
 # Local packages, usually specified by relative directory name
 packages:

striptests

@@ -29,6 +29,7 @@ detestify() {
       state = 0;
     }
 
+    /STRIP/ { next; }
     /LANGUAGE TemplateHaskell/ { next; }
     /^import.*Test\./ { next; }
 
@@ -75,4 +76,3 @@ find . -name '.git' -prune -o -type f -name '*.hs' -print |
   do
     modify "$file" detestify
   done
-

test/buildtest

@@ -22,6 +22,7 @@ fi
 
 cabal install --dependencies-only --enable-tests "${flags[@]}" ||
   cabal install --dependencies-only "${flags[@]}" ||
+    cabal install --dependencies-only --max-backjumps -1 "${flags[@]}" ||
       die "can't install dependencies"
 cabal configure --enable-tests "${flags[@]}" ||
   die "configure failed"
@@ -29,6 +30,8 @@ cabal build ||
   die "build failed"
 cabal test ||
   die "test failed"
+cabal haddock ||
+  die "haddock failed"
 
 sc="$(find . -name shellcheck -type f -perm -111)"
 [ -x "$sc" ] || die "Can't find executable"

test/check_release

@@ -56,19 +56,19 @@ cat << EOF
 Manual Checklist
 
 $((i++)). Make sure none of the automated checks above failed
-$((i++)). Make sure Travis build currently passes: https://travis-ci.org/koalaman/shellcheck
+$((i++)). Make sure GitHub Build currently passes: https://github.com/koalaman/shellcheck/actions
 $((i++)). Make sure SnapCraft build currently works: https://build.snapcraft.io/user/koalaman
 $((i++)). Run test/distrotest to ensure that most distros can build OOTB.
 $((i++)). Format and read over the manual for bad formatting and outdated info.
-$((i++)). Make sure the Hackage package builds, so that all files are 
+$((i++)). Make sure the Hackage package builds.
 
 Release Steps
 
 $((j++)). \`cabal sdist\` to generate a Hackage package
 $((j++)). \`git push --follow-tags\` to push commit
-$((j++)). Wait for Travis to build
+$((j++)). Wait for GitHub Actions to build.
 $((j++)). Verify release:
-  a. Check that the new versions are uploaded: https://shellcheck.storage.googleapis.com/index.html
+  a. Check that the new versions are uploaded: https://github.com/koalaman/shellcheck/tags
   b. Check that the docker images have version tags: https://hub.docker.com/u/koalaman
 $((j++)). If no disaster, upload to Hackage: http://hackage.haskell.org/upload
 $((j++)). Push a new commit that updates CHANGELOG.md

test/distrotest

@@ -25,6 +25,13 @@ exit 0
 echo "Deleting 'dist' and 'dist-newstyle'..."
 rm -rf dist dist-newstyle
 
+execs=$(find . -name shellcheck)
+
+if [ -n "$execs" ]
+then
+  die "Found unexpected executables. Remove and try again: $execs"
+fi
+
 log=$(mktemp) || die "Can't create temp file"
 date >> "$log" || die "Can't write to log"
 
@@ -63,14 +70,17 @@ debian:testing        apt-get update && apt-get install -y cabal-install
 ubuntu:latest         apt-get update && apt-get install -y cabal-install
 haskell:latest        true
 opensuse/leap:latest  zypper install -y cabal-install ghc
-fedora:latest         dnf install -y cabal-install ghc-template-haskell-devel findutils
+fedora:latest         dnf install -y cabal-install ghc-template-haskell-devel findutils libstdc++-static gcc-c++
-archlinux/base:latest pacman -S -y --noconfirm cabal-install ghc-static base-devel
+archlinux:latest      pacman -S -y --noconfirm cabal-install ghc-static base-devel
 
-# Other versions we want to support
+# Ubuntu LTS
+ubuntu:22.04          apt-get update && apt-get install -y cabal-install
+ubuntu:20.04          apt-get update && apt-get install -y cabal-install
 ubuntu:18.04          apt-get update && apt-get install -y cabal-install
+ubuntu:16.04          apt-get update && apt-get install -y cabal-install
 
-# Misc Haskell including current and latest Stack build
+# Stack on Ubuntu LTS
-ubuntu:18.04          set -e; apt-get update && apt-get install -y curl && curl -sSL https://get.haskellstack.org/ | sh -s - -f && cd /mnt && exec test/stacktest
+ubuntu:22.04          set -e; apt-get update && apt-get install -y curl && curl -sSL https://get.haskellstack.org/ | sh -s - -f && cd /mnt && exec test/stacktest
 EOF
 
 exit "$final"

test/shellcheck.hs

@@ -4,8 +4,12 @@ import Control.Monad
 import System.Exit
 import qualified ShellCheck.Analytics
 import qualified ShellCheck.AnalyzerLib
+import qualified ShellCheck.ASTLib
+import qualified ShellCheck.CFG
+import qualified ShellCheck.CFGAnalysis
 import qualified ShellCheck.Checker
 import qualified ShellCheck.Checks.Commands
+import qualified ShellCheck.Checks.ControlFlow
 import qualified ShellCheck.Checks.Custom
 import qualified ShellCheck.Checks.ShellSupport
 import qualified ShellCheck.Fixer
@@ -17,8 +21,12 @@ main = do
     results <- sequence [
         ShellCheck.Analytics.runTests
         ,ShellCheck.AnalyzerLib.runTests
+        ,ShellCheck.ASTLib.runTests
+        ,ShellCheck.CFG.runTests
+        ,ShellCheck.CFGAnalysis.runTests
         ,ShellCheck.Checker.runTests
         ,ShellCheck.Checks.Commands.runTests
+        ,ShellCheck.Checks.ControlFlow.runTests
         ,ShellCheck.Checks.Custom.runTests
         ,ShellCheck.Checks.ShellSupport.runTests
         ,ShellCheck.Fixer.runTests

test/stacktest

@@ -3,7 +3,7 @@
 # various resolvers. It's run via distrotest.
 
 resolvers=(
-  nightly-"$(date -d "3 days ago" +"%Y-%m-%d")"
+#  nightly-"$(date -d "3 days ago" +"%Y-%m-%d")"
 )
 
 die() { echo "$*" >&2; exit 1; }
@@ -18,10 +18,11 @@ command -v stack ||
 stack setup        || die "Failed to setup with default resolver"
 stack build --test || die "Failed to build/test with default resolver"
 
+# Nice to haves, but not necessary
 for resolver in "${resolvers[@]}"
 do
-  stack --resolver="$resolver" setup        || die "Failed to setup $resolver"
+  stack --resolver="$resolver" setup        || die "Failed to setup $resolver. This probably doesn't matter."
-  stack --resolver="$resolver" build --test || die "Failed build/test with $resolver!"
+  stack --resolver="$resolver" build --test || die "Failed build/test with $resolver! This probably doesn't matter."
 done
 
 echo "Success"