Stable version 0.8.0

This release is dedicated to dibblego, who pushed me down the Haskell
rabbit hole. In 2006 I thought you were crazy. Today I *know* you are.

.github/workflows/build.yml

@@ -1,4 +1,4 @@
-name: Build Lol
+name: Build ShellCheck
 
 # Run this workflow every time a new commit pushed to your repository
 on: push
@@ -16,22 +16,25 @@ jobs:
 
       - name: Checkout repository
         uses: actions/checkout@v2
-
+        with:
-      - name: Package Source
+          fetch-depth: 0
-        run: |
-          mkdir source
-          cabal sdist
-          mv dist/*.tar.gz source/source.tar.gz
 
       - name: Deduce tags
         run: |
-          exec > source/tags
+          mkdir source
-          echo "latest"
+          echo "latest" > source/tags
           if tag=$(git describe --exact-match --tags)
           then
-            echo "stable"
+            echo "stable" >> source/tags
-            echo "$tag"
+            echo "$tag" >> source/tags
           fi
+          cat source/tags
+
+      - name: Package Source
+        run: |
+          grep "stable" source/tags || ./setgitversion
+          cabal sdist
+          mv dist-newstyle/sdist/*.tar.gz source/source.tar.gz
 
       - name: Upload artifact
         uses: actions/upload-artifact@v2
@@ -113,6 +116,10 @@ jobs:
           export TAGS="$(cat source/tags)"
           ./.github_deploy
 
+      - name: Waiting for GitHub to replicate uploaded releases
+        run: |
+          sleep 300
+
       - name: Upload to Docker Hub
         env:
           DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}

.github_deploy

@@ -26,4 +26,3 @@ do
   done
   gh release upload "$tag" "${files[@]}" --clobber || exit 1
 done
-

CHANGELOG.md

@@ -1,13 +1,53 @@
-## Git
+## v0.8.0 - 2021-11-06
+### Added
+- `disable=all` now conveniently disables all warnings
+- `external-sources=true` directive can be added to .shellcheckrc to make
+  shellcheck behave as if `-x` was specified.
+- Optional `check-extra-masked-returns` for pointing out commands with
+  suppressed exit codes (SC2312).
+- Optional `require-double-brackets` for recommending \[\[ ]] (SC2292).
+- SC2286-SC2288: Warn when command name ends in a symbol like `/.)'"`
+- SC2289: Warn when command name contains tabs or linefeeds
+- SC2291: Warn about repeated unquoted spaces between words in echo
+- SC2292: Suggest [[ over [ in Bash/Ksh scripts (optional)
+- SC2293/SC2294: Warn when calling `eval` with arrays
+- SC2295: Warn about "${x#$y}" treating $y as a pattern when not quoted
+- SC2296-SC2301: Improved warnings for bad parameter expansions
+- SC2302/SC2303: Warn about loops over array values when using them as keys
+- SC2304-SC2306: Warn about unquoted globs in expr arguments
+- SC2307: Warn about insufficient number of arguments to expr
+- SC2308: Suggest other approaches for non-standard expr extensions
+- SC2313: Warn about `read` with unquoted, array indexed variable
+
+### Fixed
+- SC2102 about repetitions in ranges no longer triggers on [[ -v arr[xx] ]]
+- SC2155 now recognizes `typeset` and local read-only `declare` statements
+- SC2181 now tries to avoid triggering for error handling functions
+- SC2290: Warn about misused = in declare & co, which were not caught by SC2270+
+- The flag --color=auto no longer outputs color when TERM is "dumb" or unset
+
+### Changed
+- SC2048: Warning about $\* now also applies to ${array[\*]}
+- SC2181 now only triggers on single condition tests like `[ $? = 0 ]`.
+- Quote warnings are now emitted for declaration utilities in sh
+- Leading `_` can now be used to suppress warnings about unused variables
+- TTY output now includes warning level in text as well as color
+
+### Removed
+- SC1004: Literal backslash+linefeed in '' was found to be usually correct
+
+
+## v0.7.2 - 2021-04-19
 ### Added
 - `disable` directives can now be a range, e.g. `disable=SC3000-SC4000`
+- SC1143: Warn about line continuations in comments
 - SC2259/SC2260: Warn when redirections override pipes
 - SC2261: Warn about multiple competing redirections
 - SC2262/SC2263: Warn about aliases declared and used in the same parsing unit
 - SC2264: Warn about wrapper functions that blatantly recurse
 - SC2265/SC2266: Warn when using & or | with test statements
 - SC2267: Warn when using xargs -i instead of -I
-- Optional avoid-x-comparisons: Style warning SC2268 for `[ x$var = xval ]`
+- SC2268: Warn about unnecessary x-comparisons like `[ x$var = xval ]`
 
 ### Fixed
 - SC1072/SC1073 now respond to disable annotations, though ignoring parse errors
@@ -21,7 +61,7 @@
 - POSIX/dash unsupported feature warnings now have individual SC3xxx codes
 - SC1090: A leading `$x/` or `$(x)/` is now treated as `./` when locating files
 - SC2154: Variables appearing in -z/-n tests are no longer considered unassigned
-- SC2270-SC2285: Improved warnings about misused =, e.g. `${var}=42`
+- SC2270-SC2285: Improved warnings about misused `=`, e.g. `${var}=42`
 
 
 ## v0.7.1 - 2020-04-04
@@ -164,7 +204,7 @@
 - SC2204/SC2205: Warn about `( -z foo )` and `( foo -eq bar )`
 - SC2200/SC2201: Warn about brace expansion in [/[[
 - SC2198/SC2199: Warn about arrays in [/[[
-- SC2196/SC2197: Warn about deprected egrep/fgrep
+- SC2196/SC2197: Warn about deprecated egrep/fgrep
 - SC2195: Warn about unmatchable case branches
 - SC2194: Warn about constant 'case' statements
 - SC2193: Warn about `[[ file.png == *.mp3 ]]` and other unmatchables
@@ -181,7 +221,7 @@
 ### Fixed
 - `-c` no longer suggested when using `grep -o | wc`
 - Comments and whitespace are now allowed before filewide directives
-- Here doc delimters with esoteric quoting like `foo""` are now handled
+- Here doc delimiters with esoteric quoting like `foo""` are now handled
 - SC2095 about `ssh` in while read loops is now suppressed when using `-n`
 - `%(%Y%M%D)T` now recognized as a single formatter in `printf` checks
 - `grep -F` now suppresses regex related suggestions

LICENSE

@@ -681,4 +681,4 @@ into proprietary programs.  If your program is a subroutine library, you
 may consider it more useful to permit linking proprietary applications with
 the library.  If this is what you want to do, use the GNU Lesser General
 Public License instead of this License.  But first, please read
-<https://www.gnu.org/philosophy/why-not-lgpl.html>.
+<https://www.gnu.org/licenses/why-not-lgpl.html>.

README.md

@@ -1,4 +1,5 @@
-[![Build Status](https://travis-ci.org/koalaman/shellcheck.svg?branch=master)](https://travis-ci.org/koalaman/shellcheck)
+[![Build Status](https://github.com/koalaman/shellcheck/actions/workflows/build.yml/badge.svg)](https://github.com/koalaman/shellcheck/actions/workflows/build.yml)
+
 
 # ShellCheck - A shell script static analysis tool
 
@@ -112,10 +113,6 @@ Services and platforms that have ShellCheck pre-installed and ready to use:
 * [CircleCI](https://circleci.com) via the [ShellCheck Orb](https://circleci.com/orbs/registry/orb/circleci/shellcheck)
 * [Github](https://github.com/features/actions) (only Linux)
 
-Services and platforms with third party plugins:
-
-* [SonarQube](https://www.sonarqube.org/) through [sonar-shellcheck-plugin](https://github.com/emerald-squad/sonar-shellcheck-plugin)
-
 Most other services, including [GitLab](https://about.gitlab.com/), let you install
 ShellCheck yourself, either through the system's package manager (see [Installing](#installing)),
 or by downloading and unpacking a [binary release](#installing-a-pre-compiled-binary).
@@ -143,7 +140,7 @@ On systems with Stack (installs to `~/.local/bin`):
 
 On Debian based distros:
 
-    apt-get install shellcheck
+    sudo apt install shellcheck
 
 On Arch Linux based distros:
 
@@ -157,8 +154,8 @@ On Gentoo based distros:
 
 On EPEL based distros:
 
-    yum -y install epel-release
+    sudo yum -y install epel-release
-    yum install ShellCheck
+    sudo yum install ShellCheck
 
 On Fedora based distros:
 
@@ -242,6 +239,19 @@ pandoc -s -f markdown-smart -t man shellcheck.1.md -o shellcheck.1
 sudo mv shellcheck.1 /usr/share/man/man1
 ```
 
+### pre-commit
+
+To run ShellCheck via [pre-commit](https://pre-commit.com/), add the hook to your `.pre-commit-config.yaml`:
+
+```
+repos:
+-   repo: https://github.com/koalaman/shellcheck-precommit
+    rev: v0.7.2
+    hooks:
+    -   id: shellcheck
+#       args: ["--severity=warning"]  # Optionally only show errors and warnings
+```
+
 ### Travis CI
 
 Travis CI has now integrated ShellCheck by default, so you don't need to manually install it.
@@ -348,6 +358,7 @@ echo 'Don't forget to restart!'   # Singlequote closed by apostrophe
 echo 'Don\'t try this at home'    # Attempting to escape ' in ''
 echo 'Path is $PATH'              # Variables in single quotes
 trap "echo Took ${SECONDS}s" 0    # Prematurely expanded trap
+unset var[i]                      # Array index treated as glob
 ```
 
 ### Conditionals
@@ -366,6 +377,7 @@ ShellCheck can recognize many types of incorrect test statements.
 [ grep -q foo file ]              # Command without $(..)
 [[ "$$file" == *.jpg ]]           # Comparisons that can't succeed
 (( 1 -lt 2 ))                     # Using test operators in ((..))
+[ x ] & [ y ] | [ z ]             # Accidental backgrounding and piping
 ```
 
 ### Frequently misused commands
@@ -437,6 +449,8 @@ echo "Hello $name"                # Unassigned lowercase variables
 cmd | read bar; echo $bar         # Assignments in subshells
 cat foo | cp bar                  # Piping to commands that don't read
 printf '%s: %s\n' foo             # Mismatches in printf argument count
+eval "${array[@]}"                # Lost word boundaries in array eval
+for i in "${x[@]}"; do ${x[$i]}   # Using array value as key
 ```
 
 ### Robustness
@@ -461,6 +475,7 @@ ShellCheck will warn when using features not supported by the shebang. For examp
 echo {1..$n}                     # Works in ksh, but not bash/dash/sh
 echo {1..10}                     # Works in ksh and bash, but not dash/sh
 echo -n 42                       # Works in ksh, bash and dash, undefined in sh
+expr match str regex             # Unportable alias for `expr str : regex`
 trap 'exit 42' sigint            # Unportable signal spec
 cmd &> file                      # Unportable redirection operator
 read foo < /dev/tcp/host/22      # Unportable intercepted files
@@ -481,10 +496,15 @@ rm “file”                         # Unicode quotes
 echo "Hello world"                # Carriage return / DOS line endings
 echo hello \                      # Trailing spaces after \
 var=42 echo $var                  # Expansion of inlined environment
-#!/bin/bash -x -e                 # Common shebang errors
+!# bin/bash -x -e                 # Common shebang errors
 echo $((n/180*100))               # Unnecessary loss of precision
 ls *[:digit:].txt                 # Bad character class globs
 sed 's/foo/bar/' file > file      # Redirecting to input
+var2=$var2                        # Variable assigned to itself
+[ x$var = xval ]                  # Antiquated x-comparisons
+ls() { ls -l "$@"; }              # Infinitely recursive wrapper
+alias ls='ls -l'; ls foo          # Alias used before it takes effect
+for x; do for x; do               # Nested loop uses same variable
 while getopts "a" f; do case $f in "b") # Unhandled getopts flags
 ```
 

ShellCheck.cabal

@@ -1,5 +1,5 @@
 Name:             ShellCheck
-Version:          0.7.1
+Version:          0.8.0
 Synopsis:         Shell script analysis tool
 License:          GPL-3
 License-file:     LICENSE
@@ -8,7 +8,7 @@ Author:           Vidar Holen
 Maintainer:       vidar@vidarholen.net
 Homepage:         https://www.shellcheck.net/
 Build-Type:       Simple
-Cabal-Version:    >= 1.8
+Cabal-Version:    1.18
 Bug-reports:      https://github.com/koalaman/shellcheck/issues
 Description:
   The goals of ShellCheck are:
@@ -22,9 +22,11 @@ Description:
   * To point out subtle caveats, corner cases and pitfalls, that may cause an
     advanced user's otherwise working script to fail under future circumstances.
 
+Extra-Doc-Files:
+    README.md
+    CHANGELOG.md
 Extra-Source-Files:
     -- documentation
-    README.md
     shellcheck.1.md
     -- A script to build the man page using pandoc
     manpage
@@ -83,6 +85,7 @@ library
       ShellCheck.Regex
     other-modules:
       Paths_ShellCheck
+    default-language: Haskell98
 
 executable shellcheck
     if impl(ghc < 8.0)
@@ -103,6 +106,7 @@ executable shellcheck
       QuickCheck >= 2.7.4,
       regex-tdfa,
       ShellCheck
+    default-language: Haskell98
     main-is: shellcheck.hs
 
 test-suite test-shellcheck
@@ -122,5 +126,5 @@ test-suite test-shellcheck
       QuickCheck >= 2.7.4,
       regex-tdfa,
       ShellCheck
+    default-language: Haskell98
     main-is: test/shellcheck.hs
-

build/darwin.x86_64/Dockerfile

@@ -22,7 +22,7 @@ RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.2.0.0/cabal-in
 ENV CABALOPTS "--with-ghc=$TARGET-ghc;--with-hc-pkg=$TARGET-ghc-pkg"
 
 # Prebuild the dependencies
-RUN cabal update && IFS=';' && cabal install $CABALOPTS --lib Diff-0.4.0 base-compat-0.11.2 base-orphans-0.8.4 dlist-1.0 hashable-1.3.0.0 indexed-traversable-0.1.1 integer-logarithms-1.0.3.1 primitive-0.7.1.0 regex-base-0.94.0.0 splitmix-0.1.0.3 tagged-0.8.6.1 th-abstraction-0.4.2.0 transformers-compat-0.6.6 base-compat-batteries-0.11.2 time-compat-1.9.5 unordered-containers-0.2.13.0 data-fix-0.3.1 vector-0.12.2.0 scientific-0.3.6.2 regex-tdfa-1.3.1.0 random-1.2.0 distributive-0.6.2.1 attoparsec-0.13.2.5 uuid-types-1.0.3 comonad-5.0.8 bifunctors-5.5.10 assoc-1.0.2 these-1.1.1.1 strict-0.4.0.1 aeson-1.5.5.1
+RUN cabal update && IFS=';' && cabal install --dependencies-only $CABALOPTS ShellCheck
 
 # Copy the build script
 COPY build /usr/bin

build/darwin.x86_64/build

@@ -2,7 +2,7 @@
 set -xe
 {
   tar xzv --strip-components=1
-  ./striptests
+  chmod +x striptests && ./striptests
   mkdir "$TARGETNAME"
   cabal update
   ( IFS=';'; cabal build $CABALOPTS )

build/linux.aarch64/Dockerfile

@@ -21,7 +21,7 @@ RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.2.0.0/cabal-in
 ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections;--with-ghc=$TARGET-ghc;--with-hc-pkg=$TARGET-ghc-pkg"
 
 # Prebuild the dependencies
-RUN cabal update && IFS=';' && cabal install $CABALOPTS --lib Diff-0.4.0 base-compat-0.11.2 base-orphans-0.8.4 dlist-1.0 hashable-1.3.0.0 indexed-traversable-0.1.1 integer-logarithms-1.0.3.1 primitive-0.7.1.0 regex-base-0.94.0.0 splitmix-0.1.0.3 tagged-0.8.6.1 th-abstraction-0.4.2.0 transformers-compat-0.6.6 base-compat-batteries-0.11.2 time-compat-1.9.5 unordered-containers-0.2.13.0 data-fix-0.3.1 vector-0.12.2.0 scientific-0.3.6.2 regex-tdfa-1.3.1.0 random-1.2.0 distributive-0.6.2.1 attoparsec-0.13.2.5 uuid-types-1.0.3 comonad-5.0.8 bifunctors-5.5.10 assoc-1.0.2 these-1.1.1.1 strict-0.4.0.1 aeson-1.5.5.1
+RUN cabal update && IFS=';' && cabal install --dependencies-only $CABALOPTS ShellCheck
 
 # Copy the build script
 COPY build /usr/bin

build/linux.aarch64/build

@@ -2,7 +2,7 @@
 set -xe
 {
   tar xzv --strip-components=1
-  ./striptests
+  chmod +x striptests && ./striptests
   mkdir "$TARGETNAME"
   cabal update
   ( IFS=';'; cabal build $CABALOPTS --enable-executable-static )

build/linux.armv6hf/Dockerfile

@@ -51,7 +51,7 @@ RUN pirun apt-get install -y ghc cabal-install
 # Finally we can build the current dependencies. This takes hours.
 ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections;--gcc-options;-Os -Wl,--gc-sections -ffunction-sections -fdata-sections"
 RUN pirun cabal update
-RUN IFS=";" && pirun cabal install --lib $CABALOPTS Diff-0.4.0 base-compat-0.11.2 base-orphans-0.8.4 dlist-1.0 hashable-1.3.1.0 indexed-traversable-0.1.1 integer-logarithms-1.0.3.1 primitive-0.7.1.0 regex-base-0.94.0.1 splitmix-0.1.0.3 tagged-0.8.6.1 th-abstraction-0.4.2.0 transformers-compat-0.6.6 base-compat-batteries-0.11.2 time-compat-1.9.5 unordered-containers-0.2.13.0 data-fix-0.3.1 vector-0.12.2.0 scientific-0.3.6.2 regex-tdfa-1.3.1.0 random-1.2.0 distributive-0.6.2.1 attoparsec-0.13.2.5 uuid-types-1.0.4 comonad-5.0.8 bifunctors-5.5.10 assoc-1.0.2 these-1.1.1.1 strict-0.4.0.1 aeson-1.5.6.0
+RUN IFS=";" && pirun cabal install --dependencies-only $CABALOPTS ShellCheck
 
 # Copy the build script
 WORKDIR /pi/scratch

build/linux.armv6hf/build

@@ -3,7 +3,7 @@ set -xe
 cd /scratch
 {
   tar xzv --strip-components=1
-  ./striptests
+  chmod +x striptests && ./striptests
   mkdir "$TARGETNAME"
   # This script does not cabal update because compiling anything new is slow
   ( IFS=';'; cabal build $CABALOPTS --enable-executable-static )

build/linux.x86_64/build

@@ -2,7 +2,7 @@
 set -xe
 {
   tar xzv --strip-components=1
-  ./striptests
+  chmod +x striptests && ./striptests
   mkdir "$TARGETNAME"
   cabal update
   ( IFS=';'; cabal build $CABALOPTS --enable-executable-static )

build/windows.x86_64/Dockerfile

@@ -5,7 +5,7 @@ ENV TARGETNAME windows.x86_64
 # We don't need wine32, even though it complains
 USER root
 ENV DEBIAN_FRONTEND noninteractive
-RUN apt-get update && apt-get install -y curl busybox wine
+RUN apt-get update && apt-get install -y curl busybox wine winbind
 
 # Fetch Windows version, will be available under z:\haskell
 WORKDIR /haskell
@@ -19,8 +19,8 @@ ENV WINEPATH /haskell/bin
 # that necessitated this but I don't care enough to find out
 ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections"
 
-# Precompile some deps to speed up later builds. This list is just copied from `cabal build`
+# Precompile some deps to speed up later builds
-RUN wine /haskell/bin/cabal.exe update && IFS=';' && wine /haskell/bin/cabal.exe install $CABALOPTS --lib Diff-0.4.0 base-compat-0.11.2 base-orphans-0.8.4 dlist-1.0 hashable-1.3.0.0 indexed-traversable-0.1.1 integer-logarithms-1.0.3.1 primitive-0.7.1.0 regex-base-0.94.0.0 splitmix-0.1.0.3 tagged-0.8.6.1 th-abstraction-0.4.2.0 transformers-compat-0.6.6 base-compat-batteries-0.11.2 time-compat-1.9.5 unordered-containers-0.2.13.0 data-fix-0.3.1 vector-0.12.2.0 scientific-0.3.6.2 regex-tdfa-1.3.1.0 random-1.2.0 distributive-0.6.2.1 attoparsec-0.13.2.5 uuid-types-1.0.3 comonad-5.0.8 bifunctors-5.5.10 assoc-1.0.2 these-1.1.1.1 strict-0.4.0.1 aeson-1.5.5.1
+RUN wine /haskell/bin/cabal.exe update && IFS=';' && wine /haskell/bin/cabal.exe install --lib --dependencies-only $CABALOPTS ShellCheck
 
 COPY build /usr/bin
 WORKDIR /scratch

build/windows.x86_64/build

@@ -6,7 +6,7 @@ cabal() {
 set -xe
 {
   tar xzv --strip-components=1
-  ./striptests
+  chmod +x striptests && ./striptests
   mkdir "$TARGETNAME"
   cabal update
   ( IFS=';'; cabal build $CABALOPTS )

quickrun

@@ -2,4 +2,12 @@
 # quickrun runs ShellCheck in an interpreted mode.
 # This allows testing changes without recompiling.
 
-runghc -isrc -idist/build/autogen shellcheck.hs "$@"
+path=$(find . -type f -path './dist*/Paths_ShellCheck.hs' | sort | head -n 1)
+if [ -z "$path" ]
+then
+  echo >&2 "Unable to find Paths_ShellCheck.hs. Please 'cabal build' once."
+  exit 1
+fi
+path="${path%/*}"
+
+exec runghc -isrc -i"$path" shellcheck.hs "$@"

quicktest

@@ -3,8 +3,17 @@
 # This allows running tests without compiling, which can be faster.
 # 'cabal test' remains the source of truth.
 
+path=$(find . -type f -path './dist*/Paths_ShellCheck.hs' | sort | head -n 1)
+if [ -z "$path" ]
+then
+    echo >&2 "Unable to find Paths_ShellCheck.hs. Please 'cabal build' once."
+      exit 1
+fi
+path="${path%/*}"
+
+
 (
-var=$(echo 'main' | ghci test/shellcheck.hs 2>&1 | tee /dev/stderr)
+var=$(echo 'main' | ghci -isrc -i"$path" test/shellcheck.hs 2>&1 | tee /dev/stderr)
 if [[ $var == *ExitSuccess* ]]
 then
   exit 0

setgitversion

@@ -0,0 +1,11 @@
+#!/bin/sh -xe
+# This script hardcodes the `git describe` version as ShellCheck's version number.
+# This is done to allow shellcheck --version to differ from the cabal version when
+# building git snapshots.
+
+file="src/ShellCheck/Data.hs"
+test -e "$file"
+tmp=$(mktemp)
+version=$(git describe)
+sed -e "s/=.*VERSIONSTRING.*/= \"$version\" -- VERSIONSTRING, DO NOT SUBMIT/" "$file" > "$tmp"
+mv "$tmp" "$file"

shellcheck.1.md

@@ -112,6 +112,9 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts.
     line (plus `/dev/null`). This option allows following any file the script
     may `source`.
 
+    This option may also be enabled using `external-sources=true` in
+    `.shellcheckrc`. This flag takes precedence.
+
 **FILES...**
 
 :   One or more script files to check, or "-" for standard input.
@@ -234,11 +237,20 @@ Valid keys are:
     The command can be a simple command like `echo foo`, or a compound command
     like a function definition, subshell block or loop. A range can be
     be specified with a dash, e.g. `disable=SC3000-SC4000` to exclude 3xxx.
+    All warnings can be disabled with `disable=all`.
 
 **enable**
 :   Enable an optional check by name, as listed with **--list-optional**.
     Only file-wide `enable` directives are considered.
 
+**external-sources**
+:   Set to `true` in `.shellcheckrc` to always allow ShellCheck to open
+    arbitrary files from 'source' statements (the way most tools do).
+
+    This option defaults to `false` only due to ShellCheck's origin as a
+    remote service for checking untrusted scripts. It can safely be enabled
+    for normal development.
+
 **source**
 :   Overrides the filename included by a `source`/`.` statement. This can be
     used to tell shellcheck where to look for a file whose name is determined
@@ -270,6 +282,9 @@ Here is an example `.shellcheckrc`:
     source-path=SCRIPTDIR
     source-path=/mnt/chroot
 
+    # Allow opening any 'source'd file, even if not specified as input
+    external-sources=true
+
     # Turn on warnings for unquoted variables with safe values
     enable=quote-safe-variables
 
@@ -320,10 +335,32 @@ locales where encoding is unspecified (such as the `C` locale).
 Windows users seeing `commitBuffer: invalid argument (invalid character)`
 should set their terminal to use UTF-8 with `chcp 65001`.
 
-# AUTHORS
+# KNOWN INCOMPATIBILITIES
 
-ShellCheck is developed and maintained by Vidar Holen, with assistance from a
+(If nothing in this section makes sense, you are unlikely to be affected by it)
-long list of wonderful contributors.
+
+To avoid confusing and misguided suggestions, ShellCheck requires function
+bodies to be either `{ brace groups; }` or `( subshells )`, and function names
+containing `[]*=!` are only recognized after a `function` keyword.
+
+The following unconventional function definitions are identical in Bash,
+but ShellCheck only recognizes the latter.
+
+    [x!=y] () [[ $1 ]]
+    function [x!=y] () { [[ $1 ]]; }
+
+Shells without the `function` keyword do not allow these characters in function
+names to begin with.  Function names containing `{}` are not supported at all.
+
+Further, if ShellCheck sees `[x!=y]` it will assume this is an invalid
+comparison. To invoke the above function, quote the command as in `'[x!=y]'`,
+or to retain the same globbing behavior, use `command [x!=y]`.
+
+ShellCheck imposes additional restrictions on the `[` command to help diagnose
+common invalid uses. While `[ $x= 1 ]` is defined in POSIX, ShellCheck will
+assume it was intended as the much more likely comparison `[ "$x" = 1 ]` and
+fail accordingly. For unconventional or dynamic uses of the `[` command, use
+`test` or `\[` instead.
 
 # REPORTING BUGS
 
@@ -331,9 +368,14 @@ Bugs and issues can be reported on GitHub:
 
 https://github.com/koalaman/shellcheck/issues
 
+# AUTHORS
+
+ShellCheck is developed and maintained by Vidar Holen, with assistance from a
+long list of wonderful contributors.
+
 # COPYRIGHT
 
-Copyright 2012-2019, Vidar Holen and contributors.
+Copyright 2012-2021, Vidar Holen and contributors.
 Licensed under the GNU General Public License version 3 or later,
 see https://gnu.org/licenses/gpl.html
 

shellcheck.hs

@@ -234,7 +234,7 @@ runFormatter sys format options files = do
 
     process :: FilePath -> IO Status
     process filename = do
-        input <- siReadFile sys filename
+        input <- siReadFile sys Nothing filename
         either (reportFailure filename) check input
       where
         check contents = do
@@ -389,6 +389,7 @@ parseOption flag options =
             throwError SyntaxFailure
         return (Prelude.read num :: Integer)
 
+ioInterface :: Options -> [FilePath] -> IO (SystemInterface IO)
 ioInterface options files = do
     inputs <- mapM normalize files
     cache <- newIORef emptyCache
@@ -402,14 +403,14 @@ ioInterface options files = do
     emptyCache :: Map.Map FilePath String
     emptyCache = Map.empty
 
-    get cache inputs file = do
+    get cache inputs rcSuggestsExternal file = do
         map <- readIORef cache
         case Map.lookup file map of
             Just x  -> return $ Right x
-            Nothing -> fetch cache inputs file
+            Nothing -> fetch cache inputs rcSuggestsExternal file
 
-    fetch cache inputs file = do
+    fetch cache inputs rcSuggestsExternal file = do
-        ok <- allowable inputs file
+        ok <- allowable rcSuggestsExternal inputs file
         if ok
           then (do
             (contents, shouldCache) <- inputFile file
@@ -417,13 +418,16 @@ ioInterface options files = do
                 modifyIORef cache $ Map.insert file contents
             return $ Right contents
             ) `catch` handler
-          else return $ Left (file ++ " was not specified as input (see shellcheck -x).")
+          else
+            if rcSuggestsExternal == Just False
+            then return $ Left (file ++ " was not specified as input, and external files were disabled via directive.")
+            else return $ Left (file ++ " was not specified as input (see shellcheck -x).")
       where
         handler :: IOException -> IO (Either ErrorMessage String)
         handler ex = return . Left $ show ex
 
-    allowable inputs x =
+    allowable rcSuggestsExternal inputs x =
-        if externalSources options
+        if fromMaybe (externalSources options) rcSuggestsExternal
         then return True
         else do
             path <- normalize x
@@ -497,7 +501,7 @@ ioInterface options files = do
             b <- p x
             if b then pure (Just x) else acc
 
-    findSourceFile inputs sourcePathFlag currentScript sourcePathAnnotation original =
+    findSourceFile inputs sourcePathFlag currentScript rcSuggestsExternal sourcePathAnnotation original =
         if isAbsolute original
         then
             let (_, relative) = splitDrive original
@@ -506,7 +510,7 @@ ioInterface options files = do
             find original original
       where
         find filename deflt = do
-            sources <- findM ((allowable inputs) `andM` doesFileExist) $
+            sources <- findM ((allowable rcSuggestsExternal inputs) `andM` doesFileExist) $
                         (adjustPath filename):(map ((</> filename) . adjustPath) $ sourcePathFlag ++ sourcePathAnnotation)
             case sources of
                 Nothing -> return deflt

src/ShellCheck/AST.hs

@@ -150,6 +150,7 @@ data Annotation =
     | SourceOverride String
     | ShellOverride String
     | SourcePath String
+    | ExternalSources Bool
     deriving (Show, Eq)
 data ConditionType = DoubleBracket | SingleBracket deriving (Show, Eq)
 

src/ShellCheck/ASTLib.hs

@@ -1,5 +1,5 @@
 {-
-    Copyright 2012-2019 Vidar Holen
+    Copyright 2012-2021 Vidar Holen
 
     This file is part of ShellCheck.
     https://www.shellcheck.net
@@ -17,9 +17,11 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 -}
+{-# LANGUAGE TemplateHaskell #-}
 module ShellCheck.ASTLib where
 
 import ShellCheck.AST
+import ShellCheck.Regex
 
 import Control.Monad.Writer
 import Control.Monad
@@ -31,6 +33,8 @@ import Data.Maybe
 import qualified Data.Map as Map
 import Numeric (showHex)
 
+import Test.QuickCheck
+
 arguments (T_SimpleCommand _ _ (cmd:args)) = args
 
 -- Is this a type of loop?
@@ -55,10 +59,28 @@ willSplit x =
     T_NormalWord _ l -> any willSplit l
     _ -> False
 
-isGlob T_Extglob {} = True
+isGlob t = case t of
-isGlob T_Glob {} = True
+    T_Extglob {} -> True
-isGlob (T_NormalWord _ l) = any isGlob l
+    T_Glob {} -> True
-isGlob _ = False
+    T_NormalWord _ l -> any isGlob l || hasSplitRange l
+    _ -> False
+  where
+    -- foo[x${var}y] gets parsed as foo,[,x,$var,y],
+    -- so check if there's such an interval
+    hasSplitRange l =
+        let afterBracket = dropWhile (not . isHalfOpenRange) l
+        in any isClosingRange afterBracket
+
+    isHalfOpenRange t =
+        case t of
+            T_Literal _ "[" -> True
+            _ -> False
+
+    isClosingRange t =
+        case t of
+            T_Literal _ str -> ']' `elem` str
+            _ -> False
+
 
 -- Is this shell word a constant?
 isConstant token =
@@ -224,6 +246,39 @@ getOpts (gnu, arbitraryLongOpts) string longopts args = process args
 
     listToArgs = map (\x -> ("", (x, x)))
 
+
+-- Generic getOpts that doesn't rely on a format string, but may also be inaccurate.
+-- This provides a best guess interpretation instead of failing when new options are added.
+--
+--    "--" is treated as end of arguments
+--    "--anything[=foo]" is treated as a long option without argument
+--    "-any" is treated as -a -n -y, with the next arg as an option to -y unless it starts with -
+--    anything else is an argument
+getGenericOpts :: [Token] -> [(String, (Token, Token))]
+getGenericOpts = process
+  where
+    process (token:rest) =
+        case getLiteralStringDef "\0" token of
+            "--" -> map (\c -> ("", (c,c))) rest
+            '-':'-':word -> (takeWhile (`notElem` "\0=") word, (token, token)) : process rest
+            '-':optString ->
+                let opts = takeWhile (/= '\0') optString
+                in
+                    case rest of
+                        next:_ | "-" `isPrefixOf` getLiteralStringDef "\0" next  ->
+                            map (\c -> ([c], (token, token))) opts ++ process rest
+                        next:remainder ->
+                            case reverse opts of
+                                last:initial ->
+                                    map (\c -> ([c], (token, token))) (reverse initial)
+                                        ++ [([last], (token, next))]
+                                        ++ process remainder
+                                [] -> process remainder
+                        [] -> map (\c -> ([c], (token, token))) opts
+            _ -> ("", (token, token)) : process rest
+    process [] = []
+
+
 -- Is this an expansion of multiple items of an array?
 isArrayExpansion (T_DollarBraced _ _ l) =
     let string = concat $ oversimplify l in
@@ -232,14 +287,14 @@ isArrayExpansion (T_DollarBraced _ _ l) =
 isArrayExpansion _ = False
 
 -- Is it possible that this arg becomes multiple args?
-mayBecomeMultipleArgs t = willBecomeMultipleArgs t || f t
+mayBecomeMultipleArgs t = willBecomeMultipleArgs t || f False t
   where
-    f (T_DollarBraced _ _ l) =
+    f quoted (T_DollarBraced _ _ l) =
         let string = concat $ oversimplify l in
-            "!" `isPrefixOf` string
+            not quoted || "!" `isPrefixOf` string
-    f (T_DoubleQuoted _ parts) = any f parts
+    f quoted (T_DoubleQuoted _ parts) = any (f True) parts
-    f (T_NormalWord _ parts) = any f parts
+    f quoted (T_NormalWord _ parts) = any (f quoted) parts
-    f _ = False
+    f _ _ = False
 
 -- Is it certain that this word will becomes multiple words?
 willBecomeMultipleArgs t = willConcatInAssignment t || f t
@@ -247,7 +302,6 @@ willBecomeMultipleArgs t = willConcatInAssignment t || f t
     f T_Extglob {} = True
     f T_Glob {} = True
     f T_BraceExpansion {} = True
-    f (T_DoubleQuoted _ parts) = any f parts
     f (T_NormalWord _ parts) = any f parts
     f _ = False
 
@@ -672,3 +726,43 @@ isAnnotationIgnoringCode code t =
   where
     hasNum (DisableComment from to) = code >= from && code < to
     hasNum _                   = False
+
+prop_executableFromShebang1 = executableFromShebang "/bin/sh" == "sh"
+prop_executableFromShebang2 = executableFromShebang "/bin/bash" == "bash"
+prop_executableFromShebang3 = executableFromShebang "/usr/bin/env ksh" == "ksh"
+prop_executableFromShebang4 = executableFromShebang "/usr/bin/env -S foo=bar bash -x" == "bash"
+prop_executableFromShebang5 = executableFromShebang "/usr/bin/env --split-string=bash -x" == "bash"
+prop_executableFromShebang6 = executableFromShebang "/usr/bin/env --split-string=foo=bar bash -x" == "bash"
+prop_executableFromShebang7 = executableFromShebang "/usr/bin/env --split-string bash -x" == "bash"
+prop_executableFromShebang8 = executableFromShebang "/usr/bin/env --split-string foo=bar bash -x" == "bash"
+prop_executableFromShebang9 = executableFromShebang "/usr/bin/env foo=bar dash" == "dash"
+prop_executableFromShebang10 = executableFromShebang "/bin/busybox sh" == "ash"
+prop_executableFromShebang11 = executableFromShebang "/bin/busybox ash" == "ash"
+
+-- Get the shell executable from a string like '/usr/bin/env bash'
+executableFromShebang :: String -> String
+executableFromShebang = shellFor
+  where
+    re = mkRegex "/env +(-S|--split-string=?)? *(.*)"
+    shellFor s | s `matches` re =
+        case matchRegex re s of
+            Just [flag, shell] -> fromEnvArgs (words shell)
+            _ -> ""
+    shellFor sb =
+        case words sb of
+            [] -> ""
+            [x] -> basename x
+            (first:second:args) | basename first == "busybox" ->
+                case basename second of
+                   "sh" -> "ash" -- busybox sh is ash
+                   x -> x
+            (first:args) | basename first == "env" ->
+                fromEnvArgs args
+            (first:_) -> basename first
+
+    fromEnvArgs args = fromMaybe "" $ find (notElem '=') $ skipFlags args
+    basename s = reverse . takeWhile (/= '/') . reverse $ s
+    skipFlags = dropWhile ("-" `isPrefixOf`)
+
+return []
+runTests = $quickCheckAll

src/ShellCheck/AnalyzerLib.hs

@@ -1,5 +1,5 @@
 {-
-    Copyright 2012-2019 Vidar Holen
+    Copyright 2012-2021 Vidar Holen
 
     This file is part of ShellCheck.
     https://www.shellcheck.net
@@ -79,8 +79,12 @@ composeAnalyzers f g x = f x >> g x
 data Parameters = Parameters {
     -- Whether this script has the 'lastpipe' option set/default.
     hasLastpipe        :: Bool,
+    -- Whether this script has the 'inherit_errexit' option set/default.
+    hasInheritErrexit  :: Bool,
     -- Whether this script has 'set -e' anywhere.
     hasSetE            :: Bool,
+    -- Whether this script has 'set -o pipefail' anywhere.
+    hasPipefail        :: Bool,
     -- A linear (bad) analysis of data flow
     variableFlow       :: [StackData],
     -- A map from Id to parent Token
@@ -142,7 +146,7 @@ producesComments c s = do
         prRoot pr
         let spec = defaultSpec pr
         let params = makeParameters spec
-        return . not . null $ runChecker params c
+        return . not . null $ filterByAnnotation spec params $ runChecker params c
 
 makeComment :: Severity -> Id -> Code -> String -> TokenComment
 makeComment severity id code note =
@@ -167,6 +171,8 @@ errWithFix :: MonadWriter [TokenComment] m => Id -> Code -> String -> Fix -> m (
 errWithFix  = addCommentWithFix ErrorC
 warnWithFix :: MonadWriter [TokenComment] m => Id -> Code -> String -> Fix -> m ()
 warnWithFix  = addCommentWithFix WarningC
+infoWithFix :: MonadWriter [TokenComment] m => Id -> Code -> String -> Fix -> m ()
+infoWithFix = addCommentWithFix InfoC
 styleWithFix :: MonadWriter [TokenComment] m => Id -> Code -> String -> Fix -> m ()
 styleWithFix = addCommentWithFix StyleC
 
@@ -178,7 +184,8 @@ makeCommentWithFix :: Severity -> Id -> Code -> String -> Fix -> TokenComment
 makeCommentWithFix severity id code str fix =
     let comment = makeComment severity id code str
         withFix = comment {
-            tcFix = Just fix
+            -- If fix is empty, pretend it wasn't there.
+            tcFix = if null (fixReplacements fix) then Nothing else Just fix
         }
     in force withFix
 
@@ -193,7 +200,18 @@ makeParameters spec =
                 Dash -> False
                 Sh   -> False
                 Ksh  -> True,
-
+        hasInheritErrexit =
+            case shellType params of
+                Bash -> containsInheritErrexit root
+                Dash -> True
+                Sh   -> True
+                Ksh  -> False,
+        hasPipefail =
+            case shellType params of
+                Bash -> containsPipefail root
+                Dash -> True
+                Sh   -> True
+                Ksh  -> containsPipefail root,
         shellTypeSpecified = isJust (asShellType spec) || isJust (asFallbackShell spec),
         parentMap = getParentTree root,
         variableFlow = getVariableFlow params root,
@@ -216,18 +234,33 @@ containsSetE root = isNothing $ doAnalysis (guard . not . isSetE) root
             _ -> False
     re = mkRegex "[[:space:]]-[^-]*e"
 
--- Does this script mention 'shopt -s lastpipe' anywhere?
+containsPipefail root = isNothing $ doAnalysis (guard . not . isPipefail) root
--- Also used as a hack.
+  where
-containsLastpipe root =
+    isPipefail t =
+        case t of
+            T_SimpleCommand {}  ->
+                t `isUnqualifiedCommand` "set" &&
+                    ("pipefail" `elem` oversimplify t ||
+                        "o" `elem` map snd (getAllFlags t))
+            _ -> False
+
+containsShopt shopt root =
         isNothing $ doAnalysis (guard . not . isShoptLastPipe) root
     where
         isShoptLastPipe t =
             case t of
                 T_SimpleCommand {}  ->
                     t `isUnqualifiedCommand` "shopt" &&
-                        ("lastpipe" `elem` oversimplify t)
+                        (shopt `elem` oversimplify t)
                 _ -> False
 
+-- Does this script mention 'shopt -s inherit_errexit' anywhere?
+containsInheritErrexit = containsShopt "inherit_errexit"
+
+-- Does this script mention 'shopt -s lastpipe' anywhere?
+-- Also used as a hack.
+containsLastpipe = containsShopt "lastpipe"
+
 
 prop_determineShell0 = determineShellTest "#!/bin/sh" == Sh
 prop_determineShell1 = determineShellTest "#!/usr/bin/env ksh" == Ksh
@@ -240,6 +273,8 @@ prop_determineShell7 = determineShellTest "#! /bin/ash" == Dash
 prop_determineShell8 = determineShellTest' (Just Ksh) "#!/bin/sh" == Sh
 prop_determineShell9 = determineShellTest "#!/bin/env -S dash -x" == Dash
 prop_determineShell10 = determineShellTest "#!/bin/env --split-string= dash -x" == Dash
+prop_determineShell11 = determineShellTest "#!/bin/busybox sh" == Dash -- busybox sh is a specific shell, not posix sh
+prop_determineShell12 = determineShellTest "#!/bin/busybox ash" == Dash
 
 determineShellTest = determineShellTest' Nothing
 determineShellTest' fallbackShell = determineShell fallbackShell . fromJust . prRoot . pScript
@@ -253,19 +288,6 @@ determineShell fallbackShell t = fromMaybe Bash $
         headOrDefault (fromShebang s) [s | ShellOverride s <- annotations]
     fromShebang (T_Script _ (T_Literal _ s) _) = executableFromShebang s
 
--- Given a string like "/bin/bash" or "/usr/bin/env dash",
--- return the shell basename like "bash" or "dash"
-executableFromShebang :: String -> String
-executableFromShebang = shellFor
-  where
-    shellFor s | "/env " `isInfixOf` s = case matchRegex re s of
-        Just [flag, shell] -> shell
-        _ -> ""
-    shellFor s | ' ' `elem` s = shellFor $ takeWhile (/= ' ') s
-    shellFor s = reverse . takeWhile (/= '/') . reverse $ s
-    re = mkRegex "/env +(-S|--split-string=?)? *([^ ]*)"
-
-
 -- Given a root node, make a map from Id to parent Token.
 -- This is used to populate parentMap in Parameters
 getParentTree :: Token -> Map.Map Id Token
@@ -298,14 +320,14 @@ isStrictlyQuoteFree = isQuoteFreeNode True
 isQuoteFree = isQuoteFreeNode False
 
 
-isQuoteFreeNode strict tree t =
+isQuoteFreeNode strict shell tree t =
     isQuoteFreeElement t ||
-        headOrDefault False (mapMaybe isQuoteFreeContext (drop 1 $ getPath tree t))
+        (fromMaybe False $ msum $ map isQuoteFreeContext $ drop 1 $ getPath tree t)
   where
     -- Is this node self-quoting in itself?
     isQuoteFreeElement t =
         case t of
-            T_Assignment {} -> True
+            T_Assignment {} -> assignmentIsQuoting t
             T_FdRedirect {} -> True
             _               -> False
 
@@ -317,7 +339,7 @@ isQuoteFreeNode strict tree t =
             TC_Binary _ DoubleBracket _ _ _ -> return True
             TA_Sequence {}                  -> return True
             T_Arithmetic {}                 -> return True
-            T_Assignment {}                 -> return True
+            T_Assignment {}                 -> return $ assignmentIsQuoting t
             T_Redirecting {}                -> return False
             T_DoubleQuoted _ _              -> return True
             T_DollarDoubleQuoted _ _        -> return True
@@ -329,6 +351,18 @@ isQuoteFreeNode strict tree t =
             T_SelectIn {}                   -> return (not strict)
             _                               -> Nothing
 
+    -- Check whether this assigment is self-quoting due to being a recognized
+    -- assignment passed to a Declaration Utility. This will soon be required
+    -- by POSIX: https://austingroupbugs.net/view.php?id=351
+    assignmentIsQuoting t = shellParsesParamsAsAssignments || not (isAssignmentParamToCommand t)
+    shellParsesParamsAsAssignments = shell /= Sh
+
+    -- Is this assignment a parameter to a command like export/typeset/etc?
+    isAssignmentParamToCommand (T_Assignment id _ _ _ _) =
+        case Map.lookup id tree of
+            Just (T_SimpleCommand _ _ (_:args)) -> id `elem` (map getId args)
+            _ -> False
+
 -- Check if a token is a parameter to a certain command by name:
 -- Example: isParamTo (parentMap params) "sed" t
 isParamTo :: Map.Map Id Token -> String -> Token -> Bool
@@ -493,14 +527,8 @@ getModifiedVariables t =
 
         -- Count [[ -v foo ]] as an "assignment".
         -- This is to prevent [ -v foo ] being unassigned or unused.
-        TC_Unary id _ "-v" token -> do
+        TC_Unary id _ "-v" token -> maybeToList $ do
-            str <- fmap (takeWhile (/= '[')) $ -- Quoted index
+            str <- getVariableForTestDashV token
-                    flip getLiteralStringExt token $ \x ->
-                case x of
-                    T_Glob _ s -> return s -- Unquoted index
-                    _          -> []
-
-            guard . not . null $ str
             return (t, token, str, DataString SourceChecked)
 
         TC_Unary _ _ "-n" token -> markAsChecked t token
@@ -543,14 +571,12 @@ isClosingFileOp op =
 -- Consider 'export/declare -x' a reference, since it makes the var available
 getReferencedVariableCommand base@(T_SimpleCommand _ _ (T_NormalWord _ (T_Literal _ x:_):rest)) =
     case x of
+        "declare" -> forDeclare
+        "typeset" -> forDeclare
+
         "export" -> if "f" `elem` flags
             then []
             else concatMap getReference rest
-        "declare" -> if
-                any (`elem` flags) ["x", "p"] &&
-                    (not $ any (`elem` flags) ["f", "F"])
-            then concatMap getReference rest
-            else []
         "local" -> if "x" `elem` flags
             then concatMap getReference rest
             else []
@@ -561,6 +587,13 @@ getReferencedVariableCommand base@(T_SimpleCommand _ _ (T_NormalWord _ (T_Litera
         "alias" -> [(base, token, name) | token <- rest, name <- getVariablesFromLiteralToken token]
         _ -> []
   where
+    forDeclare =
+            if
+                any (`elem` flags) ["x", "p"] &&
+                    (not $ any (`elem` flags) ["f", "F"])
+            then concatMap getReference rest
+            else []
+
     getReference t@(T_Assignment _ _ name _ value) = [(t, t, name)]
     getReference t@(T_NormalWord _ [T_Literal _ name]) | not ("-" `isPrefixOf` name) = [(t, t, name)]
     getReference _ = []
@@ -600,8 +633,8 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T
         "export" ->
             if "f" `elem` flags then [] else concatMap getModifierParamString rest
 
-        "declare" -> if any (`elem` flags) ["F", "f", "p"] then [] else declaredVars
+        "declare" -> forDeclare
-        "typeset" -> declaredVars
+        "typeset" -> forDeclare
 
         "local" -> concatMap getModifierParamString rest
         "readonly" ->
@@ -613,6 +646,7 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T
             return (base, base, "@", DataString $ SourceFrom params)
 
         "printf" -> maybeToList $ getPrintfVariable rest
+        "wait" ->   maybeToList $ getWaitVariable rest
 
         "mapfile" -> maybeToList $ getMapfileArray base rest
         "readarray" -> maybeToList $ getMapfileArray base rest
@@ -632,6 +666,8 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T
         T_NormalWord id1 [T_DoubleQuoted id2 [T_Literal id3 (stripEquals s)]]
     stripEqualsFrom t = t
 
+    forDeclare = if any (`elem` flags) ["F", "f", "p"] then [] else declaredVars
+
     declaredVars = concatMap (getModifierParam defaultType) rest
       where
         defaultType = if any (`elem` flags) ["a", "A"] then DataArray else DataString
@@ -670,15 +706,15 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T
                 _            -> return (t:fromMaybe [] (getSetParams rest))
     getSetParams [] = Nothing
 
-    getPrintfVariable list = f $ map (\x -> (x, getLiteralString x)) list
+    getPrintfVariable list = getFlagAssignedVariable "v" (SourceFrom list) $ getBsdOpts "v:" list
-      where
+    getWaitVariable   list = getFlagAssignedVariable "p" SourceInteger     $ return $ getGenericOpts list
-        f ((_, Just "-v") : (t, Just var) : _) = return (base, t, varName, varType $ SourceFrom list)
+
-            where
+    getFlagAssignedVariable str dataSource maybeFlags = do
-                (varName, varType) = case elemIndex '[' var of
+        flags <- maybeFlags
-                    Just i -> (take i var, DataArray)
+        (_, (flag, value)) <- find ((== str) . fst) flags
-                    Nothing -> (var, DataString)
+        variableName <- getLiteralStringExt (const $ return "!") value
-        f (_:rest) = f rest
+        let (baseName, index) = span (/= '[') variableName
-        f [] = fail "not found"
+        return (base, value, baseName, (if null index then DataString else DataArray) dataSource)
 
     -- mapfile has some curious syntax allowing flags plus 0..n variable names
     -- where only the first non-option one is used if any.
@@ -719,6 +755,20 @@ getIndexReferences s = fromMaybe [] $ do
   where
     re = mkRegex "(\\[.*\\])"
 
+-- Given a NormalWord like foo or foo[$bar], get foo.
+-- Primarily used to get references for [[ -v foo[bar] ]]
+getVariableForTestDashV :: Token -> Maybe String
+getVariableForTestDashV t = do
+    str <- takeWhile ('[' /=) <$> getLiteralStringExt toStr t
+    guard $ isVariableName str
+    return str
+  where
+    -- foo[bar] gets parsed with [bar] as a glob, so undo that
+    toStr (T_Glob _ s) = return s
+    -- Turn foo[$x] into foo[\0] so that we can get the constant array name
+    -- in a non-constant expression (while filtering out foo$x[$y])
+    toStr _ = return "\0"
+
 prop_getOffsetReferences1 = getOffsetReferences ":bar" == ["bar"]
 prop_getOffsetReferences2 = getOffsetReferences ":bar:baz" == ["bar", "baz"]
 prop_getOffsetReferences3 = getOffsetReferences "[foo]:bar" == ["bar"]
@@ -748,7 +798,7 @@ getReferencedVariables parents t =
         TC_Unary id _ "-v" token -> getIfReference t token
         TC_Unary id _ "-R" token -> getIfReference t token
         TC_Binary id DoubleBracket op lhs rhs ->
-            if isDereferencing op
+            if isDereferencingBinaryOp op
             then concatMap (getIfReference t) [lhs, rhs]
             else []
 
@@ -777,17 +827,16 @@ getReferencedVariables parents t =
         T_Glob _ s -> return s    -- Also when parsed as globs
         _          -> []
 
-    getIfReference context token = do
+    getIfReference context token = maybeToList $ do
-            str@(h:_) <- getLiteralStringExt literalizer token
+            str <- getVariableForTestDashV token
-            when (isDigit h) $ fail "is a number"
             return (context, token, getBracedReference str)
 
-    isDereferencing = (`elem` ["-eq", "-ne", "-lt", "-le", "-gt", "-ge"])
-
     isArithmeticAssignment t = case getPath parents t of
         this: TA_Assignment _ "=" lhs _ :_ -> lhs == t
         _                                  -> False
 
+isDereferencingBinaryOp = (`elem` ["-eq", "-ne", "-lt", "-le", "-gt", "-ge"])
+
 dataTypeFrom defaultType v = (case v of T_Array {} -> DataArray; _ -> defaultType) $ SourceFrom [v]
 
 
@@ -812,6 +861,7 @@ isConfusedGlobRegex _       = False
 
 isVariableStartChar x = x == '_' || isAsciiLower x || isAsciiUpper x
 isVariableChar x = isVariableStartChar x || isDigit x
+isSpecialVariableChar = (`elem` "*@#?-$!")
 variableNameRegex = mkRegex "[_a-zA-Z][_a-zA-Z0-9]*"
 
 prop_isVariableName1 = isVariableName "_fo123"
@@ -857,7 +907,7 @@ getBracedReference s = fromMaybe s $
         let name = takeWhile isVariableChar s
         guard . not $ null name
         return name
-    getSpecial (c:_) | c `elem` "*@#?-$!" = return [c]
+    getSpecial (c:_) | isSpecialVariableChar c = return [c]
     getSpecial _ = fail "empty or not special"
 
     nameExpansion ('!':next:rest) = do -- e.g. ${!foo*bar*}
@@ -870,6 +920,8 @@ getBracedReference s = fromMaybe s $
 prop_getBracedModifier1 = getBracedModifier "foo:bar:baz" == ":bar:baz"
 prop_getBracedModifier2 = getBracedModifier "!var:-foo" == ":-foo"
 prop_getBracedModifier3 = getBracedModifier "foo[bar]" == "[bar]"
+prop_getBracedModifier4 = getBracedModifier "foo[@]@Q" == "[@]@Q"
+prop_getBracedModifier5 = getBracedModifier "@@Q" == "@Q"
 getBracedModifier s = headOrDefault "" $ do
     let var = getBracedReference s
     a <- dropModifier s
@@ -888,6 +940,10 @@ getBracedModifier s = headOrDefault "" $ do
 headOrDefault _ (a:_) = a
 headOrDefault def _   = def
 
+-- Get the last element or a default. Like `last` but safe.
+lastOrDefault def [] = def
+lastOrDefault _ list = last list
+
 --- Get element n of a list, or Nothing. Like `!!` but safe.
 (!!!) list i =
     case drop i list of
@@ -945,5 +1001,34 @@ isBashLike params =
         Dash -> False
         Sh -> False
 
+-- Returns whether a token is a parameter expansion without any modifiers.
+-- True for $var ${var} $1 $#
+-- False for ${#var} ${var[x]} ${var:-0}
+isUnmodifiedParameterExpansion t =
+    case t of
+        T_DollarBraced _ False _ -> True
+        T_DollarBraced _ _ list ->
+            let str = concat $ oversimplify list
+            in getBracedReference str == str
+        _ -> False
+
+isTrueAssignmentSource c =
+    case c of
+        DataString SourceChecked -> False
+        DataString SourceDeclaration -> False
+        DataArray SourceChecked -> False
+        DataArray SourceDeclaration -> False
+        _ -> True
+
+modifiesVariable params token name =
+    or $ map check flow
+  where
+    flow = getVariableFlow params token
+    check t =
+        case t of
+            Assignment (_, _, n, source) -> isTrueAssignmentSource source && n == name
+            _ -> False
+
+
 return []
 runTests =  $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |])

src/ShellCheck/Checker.hs

@@ -1,5 +1,5 @@
 {-
-    Copyright 2012-2019 Vidar Holen
+    Copyright 2012-2020 Vidar Holen
 
     This file is part of ShellCheck.
     https://www.shellcheck.net
@@ -156,6 +156,11 @@ checkWithIncludesAndSourcePath includes mapper = getErrors
         siFindSource = mapper
     }
 
+checkWithRcIncludesAndSourcePath rc includes mapper = getErrors
+    (mockRcFile rc $ mockedSystemInterface includes) {
+        siFindSource = mapper
+    }
+
 prop_findsParseIssue = check "echo \"$12\"" == [1037]
 
 prop_commentDisablesParseIssue1 =
@@ -301,6 +306,13 @@ prop_canDisableShebangWarning = null $ result
         csScript = "#shellcheck disable=SC2148\nfoo"
     }
 
+prop_canDisableAllWarnings = result == [2086]
+  where
+    result = checkWithSpec [] emptyCheckSpec {
+        csFilename = "file.sh",
+        csScript = "#!/bin/sh\necho $1\n#shellcheck disable=all\necho `echo $1`"
+    }
+
 prop_canDisableParseErrors = null $ result
   where
     result = checkWithSpec [] emptyCheckSpec {
@@ -384,7 +396,7 @@ prop_canEnableOptionalsWithRc = result == [2244]
 
 prop_sourcePathRedirectsName = result == [2086]
   where
-    f "dir/myscript" _ "lib" = return "foo/lib"
+    f "dir/myscript" _ _ "lib" = return "foo/lib"
     result = checkWithIncludesAndSourcePath [("foo/lib", "echo $1")] f emptyCheckSpec {
         csScript = "#!/bin/bash\nsource lib",
         csFilename = "dir/myscript",
@@ -393,7 +405,7 @@ prop_sourcePathRedirectsName = result == [2086]
 
 prop_sourcePathAddsAnnotation = result == [2086]
   where
-    f "dir/myscript" ["mypath"] "lib" = return "foo/lib"
+    f "dir/myscript" _ ["mypath"] "lib" = return "foo/lib"
     result = checkWithIncludesAndSourcePath [("foo/lib", "echo $1")] f emptyCheckSpec {
         csScript = "#!/bin/bash\n# shellcheck source-path=mypath\nsource lib",
         csFilename = "dir/myscript",
@@ -402,13 +414,75 @@ prop_sourcePathAddsAnnotation = result == [2086]
 
 prop_sourcePathRedirectsDirective = result == [2086]
   where
-    f "dir/myscript" _ "lib" = return "foo/lib"
+    f "dir/myscript" _ _ "lib" = return "foo/lib"
-    f _ _ _ = return "/dev/null"
+    f _ _ _ _ = return "/dev/null"
     result = checkWithIncludesAndSourcePath [("foo/lib", "echo $1")] f emptyCheckSpec {
         csScript = "#!/bin/bash\n# shellcheck source=lib\nsource kittens",
         csFilename = "dir/myscript",
         csCheckSourced = True
     }
 
+prop_rcCanAllowExternalSources = result == [2086]
+  where
+    f "dir/myscript" (Just True) _ "mylib" = return "resolved/mylib"
+    f a b c d = error $ show ("Unexpected", a, b, c, d)
+    result = checkWithRcIncludesAndSourcePath "external-sources=true" [("resolved/mylib", "echo $1")] f emptyCheckSpec {
+        csScript = "#!/bin/bash\nsource mylib",
+        csFilename = "dir/myscript",
+        csCheckSourced = True
+    }
+
+prop_rcCanDenyExternalSources = result == [2086]
+  where
+    f "dir/myscript" (Just False) _ "mylib" = return "resolved/mylib"
+    f a b c d = error $ show ("Unexpected", a, b, c, d)
+    result = checkWithRcIncludesAndSourcePath "external-sources=false" [("resolved/mylib", "echo $1")] f emptyCheckSpec {
+        csScript = "#!/bin/bash\nsource mylib",
+        csFilename = "dir/myscript",
+        csCheckSourced = True
+    }
+
+prop_rcCanLeaveExternalSourcesUnspecified = result == [2086]
+  where
+    f "dir/myscript" Nothing _ "mylib" = return "resolved/mylib"
+    f a b c d = error $ show ("Unexpected", a, b, c, d)
+    result = checkWithRcIncludesAndSourcePath "" [("resolved/mylib", "echo $1")] f emptyCheckSpec {
+        csScript = "#!/bin/bash\nsource mylib",
+        csFilename = "dir/myscript",
+        csCheckSourced = True
+    }
+
+prop_fileCanDisableExternalSources = result == [2006, 2086]
+  where
+    f "dir/myscript" (Just True) _ "withExternal" = return "withExternal"
+    f "dir/myscript" (Just False) _ "withoutExternal" = return "withoutExternal"
+    f a b c d = error $ show ("Unexpected", a, b, c, d)
+    result = checkWithRcIncludesAndSourcePath "external-sources=true" [("withExternal", "echo $1"), ("withoutExternal", "_=`foo`")] f emptyCheckSpec {
+        csScript = "#!/bin/bash\ntrue\nsource withExternal\n# shellcheck external-sources=false\nsource withoutExternal",
+        csFilename = "dir/myscript",
+        csCheckSourced = True
+    }
+
+prop_fileCannotEnableExternalSources = result == [1144]
+  where
+    f "dir/myscript" Nothing _ "foo" = return "foo"
+    f a b c d = error $ show ("Unexpected", a, b, c, d)
+    result = checkWithRcIncludesAndSourcePath "" [("foo", "true")] f emptyCheckSpec {
+        csScript = "#!/bin/bash\n# shellcheck external-sources=true\nsource foo",
+        csFilename = "dir/myscript",
+        csCheckSourced = True
+    }
+
+prop_fileCannotEnableExternalSources2 = result == [1144]
+  where
+    f "dir/myscript" (Just False) _ "foo" = return "foo"
+    f a b c d = error $ show ("Unexpected", a, b, c, d)
+    result = checkWithRcIncludesAndSourcePath "external-sources=false" [("foo", "true")] f emptyCheckSpec {
+        csScript = "#!/bin/bash\n# shellcheck external-sources=true\nsource foo",
+        csFilename = "dir/myscript",
+        csCheckSourced = True
+    }
+
+
 return []
 runTests = $quickCheckAll

src/ShellCheck/Checks/Commands.hs

@@ -1,5 +1,5 @@
 {-
-    Copyright 2012-2019 Vidar Holen
+    Copyright 2012-2021 Vidar Holen
 
     This file is part of ShellCheck.
     https://www.shellcheck.net
@@ -57,7 +57,7 @@ commandChecks :: [CommandCheck]
 commandChecks = [
     checkTr
     ,checkFindNameGlob
-    ,checkNeedlessExpr
+    ,checkExpr
     ,checkGrepRe
     ,checkTrapQuotes
     ,checkReturn
@@ -95,7 +95,12 @@ commandChecks = [
     ,checkSourceArgs
     ,checkChmodDashr
     ,checkXargsDashi
+    ,checkUnquotedEchoSpaces
+    ,checkEvalArray
     ]
+    ++ map checkArgComparison declaringCommands
+    ++ map checkMaskedReturns declaringCommands
+
 
 optionalChecks = map fst optionalCommandChecks
 optionalCommandChecks :: [(CheckDescription, CommandCheck)]
@@ -131,18 +136,30 @@ prop_checkGetOptsS3 = checkGetOpts "-f -x" ["f", "x"] [] $ getOpts (True, True)
 prop_checkGetOptsS4 = checkGetOpts "-f -x" ["f"] [] $ getOpts (True, True) "f:" []
 prop_checkGetOptsS5 = checkGetOpts "-fx" [] [] $ getOpts (True, True) "fx:" []
 
+prop_checkGenericOptsS1 = checkGetOpts "-f x" ["f"] [] $ return . getGenericOpts
+prop_checkGenericOptsS2 = checkGetOpts "-abc x" ["a", "b", "c"] [] $ return . getGenericOpts
+prop_checkGenericOptsS3 = checkGetOpts "-abc -x" ["a", "b", "c", "x"] [] $ return . getGenericOpts
+prop_checkGenericOptsS4 = checkGetOpts "-x" ["x"] [] $ return . getGenericOpts
+
 -- Long options
 prop_checkGetOptsL1 = checkGetOpts "--foo=bar baz" ["foo"] ["baz"] $ getOpts (True, False) "" [("foo", True)]
 prop_checkGetOptsL2 = checkGetOpts "--foo bar baz" ["foo"] ["baz"] $ getOpts (True, False) "" [("foo", True)]
 prop_checkGetOptsL3 = checkGetOpts "--foo baz" ["foo"] ["baz"] $ getOpts (True, True) "" []
 prop_checkGetOptsL4 = checkGetOpts "--foo baz" [] [] $ getOpts (True, False) "" []
 
+prop_checkGenericOptsL1 = checkGetOpts "--foo=bar" ["foo"] [] $ return . getGenericOpts
+prop_checkGenericOptsL2 = checkGetOpts "--foo bar" ["foo"] ["bar"] $ return . getGenericOpts
+prop_checkGenericOptsL3 = checkGetOpts "-x --foo" ["x", "foo"] [] $ return . getGenericOpts
+
 -- Know when to terminate
 prop_checkGetOptsT1 = checkGetOpts "-a x -b" ["a", "b"] ["x"] $ getOpts (True, True) "ab" []
 prop_checkGetOptsT2 = checkGetOpts "-a x -b" ["a"] ["x","-b"] $ getOpts (False, True) "ab" []
 prop_checkGetOptsT3 = checkGetOpts "-a -- -b" ["a"] ["-b"] $ getOpts (True, True) "ab" []
 prop_checkGetOptsT4 = checkGetOpts "-a -- -b" ["a", "b"] [] $ getOpts (True, True) "a:b" []
 
+prop_checkGenericOptsT1 = checkGetOpts "-x -- -y" ["x"] ["-y"] $ return . getGenericOpts
+prop_checkGenericOptsT2 = checkGetOpts "-xy --" ["x", "y"] [] $ return . getGenericOpts
+
 
 buildCommandMap :: [CommandCheck] -> Map.Map CommandName (Token -> Analysis)
 buildCommandMap = foldl' addCheck Map.empty
@@ -235,19 +252,74 @@ checkFindNameGlob = CommandCheck (Basename "find") (f . arguments)  where
         acc b
 
 
-prop_checkNeedlessExpr = verify checkNeedlessExpr "foo=$(expr 3 + 2)"
+prop_checkExpr = verify checkExpr "foo=$(expr 3 + 2)"
-prop_checkNeedlessExpr2 = verify checkNeedlessExpr "foo=`echo \\`expr 3 + 2\\``"
+prop_checkExpr2 = verify checkExpr "foo=`echo \\`expr 3 + 2\\``"
-prop_checkNeedlessExpr3 = verifyNot checkNeedlessExpr "foo=$(expr foo : regex)"
+prop_checkExpr3 = verifyNot checkExpr "foo=$(expr foo : regex)"
-prop_checkNeedlessExpr4 = verifyNot checkNeedlessExpr "foo=$(expr foo \\< regex)"
+prop_checkExpr4 = verifyNot checkExpr "foo=$(expr foo \\< regex)"
-checkNeedlessExpr = CommandCheck (Basename "expr") f where
+prop_checkExpr5 = verify checkExpr "# shellcheck disable=SC2003\nexpr match foo bar"
-    f t =
+prop_checkExpr6 = verify checkExpr "# shellcheck disable=SC2003\nexpr foo : fo*"
+prop_checkExpr7 = verify checkExpr "# shellcheck disable=SC2003\nexpr 5 -3"
+prop_checkExpr8 = verifyNot checkExpr "# shellcheck disable=SC2003\nexpr \"$@\""
+prop_checkExpr9 = verifyNot checkExpr "# shellcheck disable=SC2003\nexpr 5 $rest"
+prop_checkExpr10 = verify checkExpr "# shellcheck disable=SC2003\nexpr length \"$var\""
+prop_checkExpr11 = verify checkExpr "# shellcheck disable=SC2003\nexpr foo > bar"
+prop_checkExpr12 = verify checkExpr "# shellcheck disable=SC2003\nexpr 1 | 2"
+prop_checkExpr13 = verify checkExpr "# shellcheck disable=SC2003\nexpr 1 * 2"
+prop_checkExpr14 = verify checkExpr "# shellcheck disable=SC2003\nexpr \"$x\" >=  \"$y\""
+
+checkExpr = CommandCheck (Basename "expr") f where
+    f t = do
         when (all (`notElem` exceptions) (words $ arguments t)) $
             style (getId $ getCommandTokenOrThis t) 2003
                 "expr is antiquated. Consider rewriting this using $((..)), ${} or [[ ]]."
+
+        case arguments t of
+            [lhs, op, rhs] -> do
+                checkOp lhs
+                case getWordParts op of
+                    [T_Glob _ "*"] ->
+                        err (getId op) 2304
+                            "* must be escaped to multiply: \\*. Modern $((x * y)) avoids this issue."
+                    [T_Literal _ ":"] | isGlob rhs ->
+                        warn (getId rhs) 2305
+                            "Quote regex argument to expr to avoid it expanding as a glob."
+                    _ -> return ()
+
+            [single] | not (willSplit single) ->
+                warn (getId single) 2307
+                    "'expr' expects 3+ arguments but sees 1. Make sure each operator/operand is a separate argument, and escape <>&|."
+
+            [first, second] |
+                (fromMaybe "" $ getLiteralString first) /= "length"
+                  && not (willSplit first || willSplit second) -> do
+                    checkOp first
+                    warn (getId t) 2307
+                        "'expr' expects 3+ arguments, but sees 2. Make sure each operator/operand is a separate argument, and escape <>&|."
+
+            (first:rest) -> do
+                checkOp first
+                forM_ rest $ \t ->
+                    -- We already find 95%+ of multiplication and regex earlier, so don't bother classifying this further.
+                    when (isGlob t) $ warn (getId t) 2306 "Escape glob characters in arguments to expr to avoid pathname expansion."
+
+            _ -> return ()
+
     -- These operators are hard to replicate in POSIX
-    exceptions = [ ":", "<", ">", "<=", ">=" ]
+    exceptions = [ ":", "<", ">", "<=", ">=",
+        -- We can offer better suggestions for these
+        "match", "length", "substr", "index"]
     words = mapMaybe getLiteralString
 
+    checkOp side =
+        case getLiteralString side of
+            Just "match" -> msg "'expr match' has unspecified results. Prefer 'expr str : regex'."
+            Just "length" -> msg "'expr length' has unspecified results. Prefer ${#var}."
+            Just "substr" -> msg "'expr substr' has unspecified results. Prefer 'cut' or ${var#???}."
+            Just "index" -> msg "'expr index' has unspecified results. Prefer x=${var%%[chars]*}; $((${#x}+1))."
+            _ -> return ()
+      where
+        msg = info (getId side) 2308
+
 
 prop_checkGrepRe1 = verify checkGrepRe "cat foo | grep *.mp3"
 prop_checkGrepRe2 = verify checkGrepRe "grep -Ev cow*test *.mp3"
@@ -718,6 +790,7 @@ prop_checkReadExpansions5 = verify checkReadExpansions "read \"$var\""
 prop_checkReadExpansions6 = verify checkReadExpansions "read -a $var"
 prop_checkReadExpansions7 = verifyNot checkReadExpansions "read $1"
 prop_checkReadExpansions8 = verifyNot checkReadExpansions "read ${var?}"
+prop_checkReadExpansions9 = verify checkReadExpansions "read arr[val]"
 checkReadExpansions = CommandCheck (Exactly "read") check
   where
     options = getGnuOpts flagsForRead
@@ -725,13 +798,26 @@ checkReadExpansions = CommandCheck (Exactly "read") check
         opts <- options $ arguments cmd
         return [y | (x,(_, y)) <- opts, null x || x == "a"]
 
-    check cmd = mapM_ warning $ getVars cmd
+    check cmd = do
-    warning t = sequence_ $ do
+        mapM_ dollarWarning $ getVars cmd
+        mapM_ arrayWarning $ arguments cmd
+
+    dollarWarning t = sequence_ $ do
         name <- getSingleUnmodifiedBracedString t
         guard $ isVariableName name   -- e.g. not $1
         return . warn (getId t) 2229 $
             "This does not read '" ++ name ++ "'. Remove $/${} for that, or use ${var?} to quiet."
 
+    arrayWarning word =
+        when (any isUnquotedBracket $ getWordParts word) $
+            warn (getId word) 2313 $
+                "Quote array indices to avoid them expanding as globs."
+
+    isUnquotedBracket t =
+        case t of
+            T_Glob _ ('[':_) -> True
+            _ -> False
+
 -- Return the single variable expansion that makes up this word, if any.
 -- e.g. $foo -> $foo, "$foo"'' -> $foo , "hello $name" -> Nothing
 getSingleUnmodifiedBracedString :: Token -> Maybe String
@@ -771,6 +857,9 @@ checkAliasesExpandEarly = CommandCheck (Exactly "alias") (f . arguments)
 
 prop_checkUnsetGlobs1 = verify checkUnsetGlobs "unset foo[1]"
 prop_checkUnsetGlobs2 = verifyNot checkUnsetGlobs "unset foo"
+prop_checkUnsetGlobs3 = verify checkUnsetGlobs "unset foo[$i]"
+prop_checkUnsetGlobs4 = verify checkUnsetGlobs "unset foo[x${i}y]"
+prop_checkUnsetGlobs5 = verifyNot checkUnsetGlobs "unset foo]["
 checkUnsetGlobs = CommandCheck (Exactly "unset") (mapM_ check . arguments)
   where
     check arg =
@@ -870,15 +959,29 @@ prop_checkWhileGetoptsCase2 = verify checkWhileGetoptsCase "while getopts 'a:' x
 prop_checkWhileGetoptsCase3 = verifyNot checkWhileGetoptsCase "while getopts 'a:b' x; do case $x in a) foo;; b) bar;; *) :;esac; done"
 prop_checkWhileGetoptsCase4 = verifyNot checkWhileGetoptsCase "while getopts 'a:123' x; do case $x in a) foo;; [0-9]) bar;; esac; done"
 prop_checkWhileGetoptsCase5 = verifyNot checkWhileGetoptsCase "while getopts 'a:' x; do case $x in a) foo;; \\?) bar;; *) baz;; esac; done"
+prop_checkWhileGetoptsCase6 = verifyNot checkWhileGetoptsCase "while getopts 'a:b' x; do case $y in a) foo;; esac; done"
+prop_checkWhileGetoptsCase7 = verifyNot checkWhileGetoptsCase "while getopts 'a:b' x; do case x$x in xa) foo;; xb) foo;; esac; done"
+prop_checkWhileGetoptsCase8 = verifyNot checkWhileGetoptsCase "while getopts 'a:b' x; do x=a; case $x in a) foo;; esac; done"
 checkWhileGetoptsCase = CommandCheck (Exactly "getopts") f
   where
     f :: Token -> Analysis
-    f t@(T_SimpleCommand _ _ (cmd:arg1:_))  = do
+    f t@(T_SimpleCommand _ _ (cmd:arg1:name:_))  = do
         path <- getPathM t
+        params <- ask
         sequence_ $ do
             options <- getLiteralString arg1
+            getoptsVar <- getLiteralString name
             (T_WhileExpression _ _ body) <- findFirst whileLoop path
-            caseCmd <- mapMaybe findCase body !!! 0
+            caseCmd@(T_CaseExpression _ var _) <- mapMaybe findCase body !!! 0
+
+            -- Make sure getopts name and case variable matches
+            [T_DollarBraced _ _ bracedWord] <- return $ getWordParts var
+            [T_Literal _ caseVar] <- return $ getWordParts bracedWord
+            guard $ caseVar == getoptsVar
+
+            -- Make sure the variable isn't modified
+            guard . not $ modifiesVariable params (T_BraceGroup (Id 0) body) getoptsVar
+
             return $ check (getId arg1) (map (:[]) $ filter (/= ':') options) caseCmd
     f _ = return ()
 
@@ -1056,7 +1159,7 @@ checkFindRedirections = CommandCheck (Basename "find") f
 
 prop_checkWhich = verify checkWhich "which '.+'"
 checkWhich = CommandCheck (Basename "which") $
-    \t -> info (getId $ getCommandTokenOrThis t) 2230 "which is non-standard. Use builtin 'command -v' instead."
+    \t -> info (getId $ getCommandTokenOrThis t) 2230 "'which' is non-standard. Use builtin 'command -v' instead."
 
 prop_checkSudoRedirect1 = verify checkSudoRedirect "sudo echo 3 > /proc/file"
 prop_checkSudoRedirect2 = verify checkSudoRedirect "sudo cmd < input"
@@ -1143,5 +1246,145 @@ checkXargsDashi = CommandCheck (Basename "xargs") f
         return $ info (getId option) 2267 "GNU xargs -i is deprecated in favor of -I{}"
     parseOpts = getBsdOpts "0oprtxadR:S:J:L:l:n:P:s:e:E:i:I:"
 
+
+prop_checkArgComparison1 = verify (checkArgComparison "declare") "declare a = b"
+prop_checkArgComparison2 = verify (checkArgComparison "declare") "declare a =b"
+prop_checkArgComparison3 = verifyNot (checkArgComparison "declare") "declare a=b"
+prop_checkArgComparison4 = verify (checkArgComparison "export") "export a +=b"
+prop_checkArgComparison7 = verifyNot (checkArgComparison "declare") "declare -a +i foo"
+prop_checkArgComparison8 = verify (checkArgComparison "let") "let x = 0"
+-- This mirrors checkSecondArgIsComparison but for arguments to local/readonly/declare/export
+checkArgComparison cmd = CommandCheck (Exactly cmd) wordsWithEqual
+  where
+    wordsWithEqual t = mapM_ check $ arguments t
+    check arg = do
+      sequence_ $ do
+        str <- getLeadingUnquotedString arg
+        case str of
+            '=':_ ->
+                return $ err (headId arg) 2290 $
+                    "Remove spaces around = to assign."
+            '+':'=':_ ->
+                return $ err (headId arg) 2290 $
+                    "Remove spaces around += to append."
+            _ -> Nothing
+
+       -- 'let' is parsed as a sequence of arithmetic expansions,
+       -- so we want the additional warning for "x="
+      when (cmd == "let") $ sequence_ $ do
+        token <- getTrailingUnquotedLiteral arg
+        str <- getLiteralString token
+        guard $ "=" `isSuffixOf` str
+        return $ err (getId token) 2290 $
+            "Remove spaces around = to assign."
+
+    headId t =
+        case t of
+            T_NormalWord _ (x:_) -> getId x
+            _ -> getId t
+
+
+prop_checkMaskedReturns1 = verify (checkMaskedReturns "local") "f() { local a=$(false); }"
+prop_checkMaskedReturns2 = verify (checkMaskedReturns "declare") "declare a=$(false)"
+prop_checkMaskedReturns3 = verify (checkMaskedReturns "declare") "declare a=\"`false`\""
+prop_checkMaskedReturns4 = verify (checkMaskedReturns "readonly") "readonly a=$(false)"
+prop_checkMaskedReturns5 = verify (checkMaskedReturns "readonly") "readonly a=\"`false`\""
+prop_checkMaskedReturns6 = verifyNot (checkMaskedReturns "declare") "declare a; a=$(false)"
+prop_checkMaskedReturns7 = verifyNot (checkMaskedReturns "local") "f() { local -r a=$(false); }"
+prop_checkMaskedReturns8 = verifyNot (checkMaskedReturns "readonly") "a=$(false); readonly a"
+prop_checkMaskedReturns9 = verify (checkMaskedReturns "typeset") "#!/bin/ksh\n f() { typeset -r x=$(false); }"
+prop_checkMaskedReturns10 = verifyNot (checkMaskedReturns "typeset") "#!/bin/ksh\n function f { typeset -r x=$(false); }"
+prop_checkMaskedReturns11 = verifyNot (checkMaskedReturns "typeset") "#!/bin/bash\n f() { typeset -r x=$(false); }"
+prop_checkMaskedReturns12 = verify (checkMaskedReturns "typeset") "typeset -r x=$(false);"
+prop_checkMaskedReturns13 = verify (checkMaskedReturns "typeset") "f() { typeset -g x=$(false); }"
+prop_checkMaskedReturns14 = verify (checkMaskedReturns "declare") "declare x=${ false; }"
+prop_checkMaskedReturns15 = verify (checkMaskedReturns "declare") "f() { declare x=$(false); }"
+checkMaskedReturns str = CommandCheck (Exactly str) checkCmd
+  where
+    checkCmd t = do
+        path <- getPathM t
+        shell <- asks shellType
+        sequence_ $ do
+            name <- getCommandName t
+
+            let flags = map snd (getAllFlags t)
+            let hasDashR =  "r" `elem` flags
+            let hasDashG =  "g" `elem` flags
+            let isInScopedFunction = any (isScopedFunction shell) path
+
+            let isLocal = not hasDashG && isLocalInFunction name && isInScopedFunction
+            let isReadOnly = name == "readonly" || hasDashR
+
+            -- Don't warn about local variables that are declared readonly,
+            -- because the workaround `local x; x=$(false); local -r x;` is annoying
+            guard . not $ isLocal && isReadOnly
+
+            return $ mapM_ checkArgs $ arguments t
+
+    checkArgs (T_Assignment id _ _ _ word) | any hasReturn $ getWordParts word =
+        warn id 2155 "Declare and assign separately to avoid masking return values."
+    checkArgs _ = return ()
+
+    isLocalInFunction = (`elem` ["local", "declare", "typeset"])
+    isScopedFunction shell t =
+        case t of
+            T_BatsTest {} -> True
+            -- In ksh, only functions declared with 'function' have their own scope
+            T_Function _ (FunctionKeyword hasFunction) _ _ _ -> shell /= Ksh || hasFunction
+            _ -> False
+
+    hasReturn t = case t of
+        T_Backticked {} -> True
+        T_DollarExpansion {} -> True
+        T_DollarBraceCommandExpansion {} -> True
+        _ -> False
+
+
+prop_checkUnquotedEchoSpaces1 = verify checkUnquotedEchoSpaces "echo foo         bar"
+prop_checkUnquotedEchoSpaces2 = verifyNot checkUnquotedEchoSpaces "echo       foo"
+prop_checkUnquotedEchoSpaces3 = verifyNot checkUnquotedEchoSpaces "echo foo  bar"
+prop_checkUnquotedEchoSpaces4 = verifyNot checkUnquotedEchoSpaces "echo 'foo          bar'"
+prop_checkUnquotedEchoSpaces5 = verifyNot checkUnquotedEchoSpaces "echo a > myfile.txt b"
+prop_checkUnquotedEchoSpaces6 = verifyNot checkUnquotedEchoSpaces "        echo foo\\\n        bar"
+checkUnquotedEchoSpaces = CommandCheck (Basename "echo") check
+  where
+    check t = do
+        let args = arguments t
+        m <- asks tokenPositions
+        redir <- getClosestCommandM t
+        sequence_ $ do
+            let positions = mapMaybe (\c -> Map.lookup (getId c) m) args
+            let pairs = zip positions (drop 1 positions)
+            (T_Redirecting _ redirTokens _) <- redir
+            let redirPositions = mapMaybe (\c -> fst <$> Map.lookup (getId c) m) redirTokens
+            guard $ any (hasSpacesBetween redirPositions) pairs
+            return $ info (getId t) 2291 "Quote repeated spaces to avoid them collapsing into one."
+
+    hasSpacesBetween redirs ((a,b), (c,d)) =
+        posLine a == posLine d
+        && ((posColumn c) - (posColumn b)) >= 4
+        && not (any (\x -> b < x && x < c) redirs)
+
+
+prop_checkEvalArray1 = verify checkEvalArray  "eval $@"
+prop_checkEvalArray2 = verify checkEvalArray  "eval \"${args[@]}\""
+prop_checkEvalArray3 = verify checkEvalArray  "eval \"${args[@]@Q}\""
+prop_checkEvalArray4 = verifyNot checkEvalArray  "eval \"${args[*]@Q}\""
+prop_checkEvalArray5 = verifyNot checkEvalArray  "eval \"$*\""
+checkEvalArray = CommandCheck (Exactly "eval") (mapM_ check . concatMap getWordParts . arguments)
+  where
+    check t =
+        when (isArrayExpansion t) $
+            if isEscaped t
+            then style (getId t) 2293 "When eval'ing @Q-quoted words, use * rather than @ as the index."
+            else warn (getId t) 2294 "eval negates the benefit of arrays. Drop eval to preserve whitespace/symbols (or eval as string)."
+
+    isEscaped q =
+        case q of
+            -- Match ${arr[@]@Q} and ${@@Q} and such
+            T_DollarBraced _ _ l -> 'Q' `elem` getBracedModifier (concat $ oversimplify l)
+            _ -> False
+
+
 return []
 runTests =  $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |])

src/ShellCheck/Checks/ShellSupport.hs

@@ -1,5 +1,5 @@
 {-
-    Copyright 2012-2016 Vidar Holen
+    Copyright 2012-2020 Vidar Holen
 
     This file is part of ShellCheck.
     https://www.shellcheck.net
@@ -180,7 +180,7 @@ prop_checkBashisms95 = verify checkBashisms "#!/bin/sh\necho $_"
 prop_checkBashisms96 = verifyNot checkBashisms "#!/bin/dash\necho $_"
 prop_checkBashisms97 = verify checkBashisms "#!/bin/sh\necho ${var,}"
 prop_checkBashisms98 = verify checkBashisms "#!/bin/sh\necho ${var^^}"
-prop_checkBashisms99 = verifyNot checkBashisms "#!/bin/dash\necho [^f]oo"
+prop_checkBashisms99 = verify checkBashisms "#!/bin/dash\necho [^f]oo"
 checkBashisms = ForShell [Sh, Dash] $ \t -> do
     params <- ask
     kludge params t
@@ -235,7 +235,7 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do
         where
             file = onlyLiteralString word
             isNetworked = any (`isPrefixOf` file) ["/dev/tcp", "/dev/udp"]
-    bashism (T_Glob id str) | not isDash && "[^" `isInfixOf` str =
+    bashism (T_Glob id str) | "[^" `isInfixOf` str =
             warnMsg id 3026 "^ in place of ! in glob bracket expressions is"
 
     bashism t@(TA_Variable id str _) | isBashVariable str =

src/ShellCheck/Data.hs

@@ -4,7 +4,7 @@ import ShellCheck.Interface
 import Data.Version (showVersion)
 import Paths_ShellCheck (version)
 
-shellcheckVersion = showVersion version
+shellcheckVersion = showVersion version -- VERSIONSTRING
 
 internalVariables = [
     -- Generic
@@ -138,3 +138,5 @@ shellForExecutable name =
         _ -> Nothing
 
 flagsForRead = "sreu:n:N:i:p:a:t:"
+
+declaringCommands = ["local", "declare", "export", "readonly", "typeset", "let"]

src/ShellCheck/Formatter/CheckStyle.hs

@@ -48,7 +48,7 @@ outputResults cr sys =
     fileGroups = groupWith sourceFile comments
     outputGroup group = do
         let filename = sourceFile (head group)
-        result <- (siReadFile sys) filename
+        result <- siReadFile sys (Just True) filename
         let contents = either (const "") id result
         outputFile filename contents group
 

src/ShellCheck/Formatter/Diff.hs

@@ -38,9 +38,6 @@ import System.FilePath
 
 import Test.QuickCheck
 
-import Debug.Trace
-ltt x = trace (show x) x
-
 format :: FormatterOptions -> IO Formatter
 format options = do
     foundIssues <- newIORef False
@@ -90,7 +87,7 @@ reportResult foundIssues reportedIssues color result sys = do
     mapM_ output $ M.toList fixmap
   where
     output (name, fix) = do
-        file <- (siReadFile sys) name
+        file <- siReadFile sys (Just True) name
         case file of
             Right contents -> do
                 putStrLn $ formatDoc color $ makeDiff name contents fix

src/ShellCheck/Formatter/Format.hs

@@ -28,6 +28,7 @@ import Data.Array
 import Data.List
 import System.IO
 import System.Info
+import System.Environment
 
 -- A formatter that carries along an arbitrary piece of data
 data Formatter = Formatter {
@@ -68,12 +69,14 @@ makeNonVirtual comments contents =
 
 
 shouldOutputColor :: ColorOption -> IO Bool
-shouldOutputColor colorOption = do
+shouldOutputColor colorOption =
-    term <- hIsTerminalDevice stdout
+    case colorOption of
-    let windows = "mingw" `isPrefixOf` os
+        ColorAlways -> return True
-    let isUsableTty = term && not windows
+        ColorNever -> return False
-    let useColor = case colorOption of
+        ColorAuto -> do
-                       ColorAlways -> True
+            isTerminal <- hIsTerminalDevice stdout
-                       ColorNever -> False
+            term <- lookupEnv "TERM"
-                       ColorAuto -> isUsableTty
+            let windows = "mingw" `isPrefixOf` os
-    return useColor
+            let dumbTerm = term `elem` [Just "dumb", Just "", Nothing]
+            let isUsableTty = isTerminal && not windows && not dumbTerm
+            return isUsableTty

src/ShellCheck/Formatter/GCC.hs

@@ -43,7 +43,7 @@ outputAll cr sys = mapM_ f groups
     f :: [PositionedComment] -> IO ()
     f group = do
         let filename = sourceFile (head group)
-        result <- (siReadFile sys) filename
+        result <- siReadFile sys (Just True) filename
         let contents = either (const "") id result
         outputResult filename contents group
 

src/ShellCheck/Formatter/JSON1.hs

@@ -117,7 +117,7 @@ collectResult ref cr sys = mapM_ f groups
     f :: [PositionedComment] -> IO ()
     f group = do
         let filename = sourceFile (head group)
-        result <- siReadFile sys filename
+        result <- siReadFile sys (Just True) filename
         let contents = either (const "") id result
         let comments' = makeNonVirtual comments contents
         modifyIORef ref (\x -> comments' ++ x)

src/ShellCheck/Formatter/TTY.hs

@@ -121,7 +121,7 @@ outputResult options ref result sys = do
 
 outputForFile color sys comments = do
     let fileName = sourceFile (head comments)
-    result <- (siReadFile sys) fileName
+    result <- siReadFile sys (Just True) fileName
     let contents = either (const "") id result
     let fileLinesList = lines contents
     let lineCount = length fileLinesList
@@ -174,7 +174,7 @@ showFixedString color comments lineNum fileLines =
 cuteIndent :: PositionedComment -> String
 cuteIndent comment =
     replicate (fromIntegral $ colNo comment - 1) ' ' ++
-        makeArrow ++ " " ++ code (codeNo comment) ++ ": " ++ messageText comment
+        makeArrow ++ " " ++ code (codeNo comment) ++ " (" ++ severityText comment ++ "): " ++ messageText comment
   where
     arrow n = '^' : replicate (fromIntegral $ n-2) '-' ++ "^"
     makeArrow =

src/ShellCheck/Interface.hs

@@ -73,15 +73,19 @@ import qualified Data.Map as Map
 
 
 data SystemInterface m = SystemInterface {
-    -- Read a file by filename, or return an error
+    -- | Given:
-    siReadFile :: String -> m (Either ErrorMessage String),
+    --   What annotations say about including external files (if anything)
-    -- Given:
+    --   A resolved filename from siFindSource
+    --   Read the file or return an error
+    siReadFile :: Maybe Bool -> String -> m (Either ErrorMessage String),
+    -- | Given:
     --   the current script,
+    --   what annotations say about including external files (if anything)
     --   a list of source-path annotations in effect,
     --   and a sourced file,
     --   find the sourced file
-    siFindSource :: String -> [String] -> String -> m FilePath,
+    siFindSource :: String -> Maybe Bool -> [String] -> String -> m FilePath,
-    -- Get the configuration file (name, contents) for a filename
+    -- | Get the configuration file (name, contents) for a filename
     siGetConfig :: String -> m (Maybe (FilePath, String))
 }
 
@@ -313,11 +317,11 @@ mockedSystemInterface files = SystemInterface {
     siGetConfig = const $ return Nothing
 }
   where
-    rf file = return $
+    rf _ file = return $
         case find ((== file) . fst) files of
             Nothing -> Left "File not included in mock."
             Just (_, contents) -> Right contents
-    fs _ _ file = return file
+    fs _ _ _ file = return file
 
 mockRcFile rcfile mock = mock {
     siGetConfig = const . return $ Just (".shellcheckrc", rcfile)

src/ShellCheck/Parser.hs

@@ -1,5 +1,5 @@
 {-
-    Copyright 2012-2019 Vidar Holen
+    Copyright 2012-2021 Vidar Holen
 
     This file is part of ShellCheck.
     https://www.shellcheck.net
@@ -24,7 +24,7 @@
 module ShellCheck.Parser (parseScript, runTests) where
 
 import ShellCheck.AST
-import ShellCheck.ASTLib
+import ShellCheck.ASTLib hiding (runTests)
 import ShellCheck.Data
 import ShellCheck.Interface
 
@@ -37,7 +37,7 @@ import Data.Functor
 import Data.List (isPrefixOf, isInfixOf, isSuffixOf, partition, sortBy, intercalate, nub, find)
 import Data.Maybe
 import Data.Monoid
-import Debug.Trace
+import Debug.Trace -- STRIP
 import GHC.Exts (sortWith)
 import Prelude hiding (readList)
 import System.IO
@@ -66,7 +66,7 @@ doubleQuote = char '"'
 variableStart = upper <|> lower <|> oneOf "_"
 variableChars = upper <|> lower <|> digit <|> oneOf "_"
 -- Chars to allow in function names
-functionChars = variableChars <|> oneOf ":+?-./^@"
+functionChars = variableChars <|> oneOf ":+?-./^@,"
 -- Chars to allow in functions using the 'function' keyword
 extendedFunctionChars = functionChars <|> oneOf "[]*=!"
 specialVariable = oneOf (concat specialVariables)
@@ -87,11 +87,23 @@ extglobStart = oneOf extglobStartChars
 unicodeDoubleQuotes = "\x201C\x201D\x2033\x2036"
 unicodeSingleQuotes = "\x2018\x2019"
 
-prop_spacing = isOk spacing "  \\\n # Comment"
+prop_spacing1 = isOk spacing "  \\\n # Comment"
+prop_spacing2 = isOk spacing "# We can continue lines with \\"
+prop_spacing3 = isWarning spacing "   \\\n #  --verbose=true \\"
 spacing = do
-    x <- many (many1 linewhitespace <|> try (string "\\\n" >> return ""))
+    x <- many (many1 linewhitespace <|> continuation)
     optional readComment
     return $ concat x
+  where
+    continuation = do
+        try (string "\\\n")
+        -- The line was continued. Warn if this next line is a comment with a trailing \
+        whitespace <- many linewhitespace
+        optional $ do
+            x <- readComment
+            when ("\\" `isSuffixOf` x) $
+                parseProblem ErrorC 1143 "This backslash is part of a comment and does not continue the line."
+        return whitespace
 
 spacing1 = do
     spacing <- spacing
@@ -972,12 +984,13 @@ prop_readAnnotation4 = isWarning readAnnotation "# shellcheck cats=dogs disable=
 prop_readAnnotation5 = isOk readAnnotation "# shellcheck disable=SC2002 # All cats are precious\n"
 prop_readAnnotation6 = isOk readAnnotation "# shellcheck disable=SC1234 # shellcheck foo=bar\n"
 prop_readAnnotation7 = isOk readAnnotation "# shellcheck disable=SC1000,SC2000-SC3000,SC1001\n"
+prop_readAnnotation8 = isOk readAnnotation "# shellcheck disable=all\n"
 readAnnotation = called "shellcheck directive" $ do
     try readAnnotationPrefix
     many1 linewhitespace
-    readAnnotationWithoutPrefix
+    readAnnotationWithoutPrefix True
 
-readAnnotationWithoutPrefix = do
+readAnnotationWithoutPrefix sandboxed = do
     values <- many1 readKey
     optional readAnyComment
     void linefeed <|> eof <|> do
@@ -992,8 +1005,12 @@ readAnnotationWithoutPrefix = do
         key <- many1 (letter <|> char '-')
         char '=' <|> fail "Expected '=' after directive key"
         annotations <- case key of
-            "disable" -> readRange `sepBy` char ','
+            "disable" -> readElement `sepBy` char ','
               where
+                readElement = readRange <|> readAll
+                readAll = do
+                    string "all"
+                    return $ DisableComment 0 1000000
                 readRange = do
                     from <- readCode
                     to <- choice [ char '-' *> readCode, return $ from+1 ]
@@ -1023,6 +1040,21 @@ readAnnotationWithoutPrefix = do
                         "This shell type is unknown. Use e.g. sh or bash."
                 return [ShellOverride shell]
 
+            "external-sources" -> do
+                pos <- getPosition
+                value <- many1 letter
+                case value of
+                    "true" ->
+                        if sandboxed
+                        then do
+                            parseNoteAt pos ErrorC 1144 "external-sources can only be enabled in .shellcheckrc, not in individual files."
+                            return []
+                        else return [ExternalSources True]
+                    "false" -> return [ExternalSources False]
+                    _ -> do
+                        parseNoteAt pos ErrorC 1145 "Unknown external-sources value. Expected true/false."
+                        return []
+
             _ -> do
                 parseNoteAt keyPos WarningC 1107 "This directive is unknown. It will be ignored."
                 anyChar `reluctantlyTill` whitespace
@@ -1039,6 +1071,7 @@ readComment = do
     unexpecting "shellcheck annotation" readAnnotationPrefix
     readAnyComment
 
+prop_readAnyComment = isOk readAnyComment "# Comment"
 readAnyComment = do
     char '#'
     many $ noneOf "\r\n"
@@ -1359,6 +1392,8 @@ prop_readGlob5 = isOk readGlob "[^[:alpha:]1-9]"
 prop_readGlob6 = isOk readGlob "[\\|]"
 prop_readGlob7 = isOk readGlob "[^[]"
 prop_readGlob8 = isOk readGlob "[*?]"
+prop_readGlob9 = isOk readGlob "[!]^]"
+prop_readGlob10 = isOk readGlob "[]]"
 readGlob = readExtglob <|> readSimple <|> readClass <|> readGlobbyLiteral
     where
         readSimple = do
@@ -1366,22 +1401,25 @@ readGlob = readExtglob <|> readSimple <|> readClass <|> readGlobbyLiteral
             c <- oneOf "*?"
             id <- endSpan start
             return $ T_Glob id [c]
-        -- Doesn't handle weird things like [^]a] and [$foo]. fixme?
         readClass = try $ do
             start <- startSpan
             char '['
-            s <- many1 (predefined <|> readNormalLiteralPart "]" <|> globchars)
+            negation <- charToString (oneOf "!^") <|> return ""
+            leadingBracket <- charToString (oneOf "]") <|> return ""
+            s <- many (predefined <|> readNormalLiteralPart "]" <|> globchars)
+            guard $ not (null leadingBracket) || not (null s)
             char ']'
             id <- endSpan start
-            return $ T_Glob id $ "[" ++ concat s ++ "]"
+            return $ T_Glob id $ "[" ++ concat (negation:leadingBracket:s) ++ "]"
           where
-           globchars = fmap return . oneOf $ "!$[" ++ extglobStartChars
+           globchars = charToString $ oneOf $ "![" ++ extglobStartChars
            predefined = do
               try $ string "[:"
               s <- many1 letter
               string ":]"
               return $ "[:" ++ s ++ ":]"
 
+        charToString = fmap return
         readGlobbyLiteral = do
             start <- startSpan
             c <- extglobStart <|> char '['
@@ -1404,6 +1442,8 @@ readNormalEscaped = called "escaped char" $ do
     do
         next <- quotable <|> oneOf "?*@!+[]{}.,~#"
         when (next == ' ') $ checkTrailingSpaces pos <|> return ()
+        -- Check if this line is followed by a commented line with a trailing backslash
+        when (next == '\n') $ try . lookAhead $ void spacing
         return $ if next == '\n' then "" else [next]
       <|>
         do
@@ -1471,7 +1511,6 @@ readSingleEscaped = do
 
     case x of
         '\'' -> parseProblemAt pos InfoC 1003 "Want to escape a single quote? echo 'This is how it'\\''s done'.";
-        '\n' -> parseProblemAt pos InfoC 1004 "This backslash+linefeed is literal. Break outside single quotes if you just want to break the line."
         _ -> return ()
 
     return [s]
@@ -1981,12 +2020,14 @@ readHereString = called "here string" $ do
     word <- readNormalWord
     return $ T_HereString id word
 
+prop_readNewlineList1 = isOk readScript "&> /dev/null echo foo"
 readNewlineList =
     many1 ((linefeed <|> carriageReturn) `thenSkip` spacing) <* checkBadBreak
   where
     checkBadBreak = optional $ do
                 pos <- getPosition
                 try $ lookAhead (oneOf "|&") --  See if the next thing could be |, || or &&
+                notFollowedBy2 (string "&>") --  Except &> or &>> which is valid
                 parseProblemAt pos ErrorC 1133
                     "Unexpected start of line. If breaking lines, |/||/&& should be at the end of the previous one."
 readLineBreak = optional readNewlineList
@@ -2046,6 +2087,7 @@ prop_readSimpleCommand4 = isOk readSimpleCommand "typeset -a foo=(lol)"
 prop_readSimpleCommand5 = isOk readSimpleCommand "time if true; then echo foo; fi"
 prop_readSimpleCommand6 = isOk readSimpleCommand "time -p ( ls -l; )"
 prop_readSimpleCommand7 = isOk readSimpleCommand "\\ls"
+prop_readSimpleCommand7b = isOk readSimpleCommand "\\:"
 prop_readSimpleCommand8 = isWarning readSimpleCommand "// Lol"
 prop_readSimpleCommand9 = isWarning readSimpleCommand "/* Lolbert */"
 prop_readSimpleCommand10 = isWarning readSimpleCommand "/**** Lolbert */"
@@ -2153,10 +2195,12 @@ readSource t@(T_Redirecting _ _ (T_SimpleCommand cmdId _ (cmd:file':rest'))) = d
                     if filename == "/dev/null" -- always allow /dev/null
                     then return (Right "", filename)
                     else do
+                        allAnnotations <- getCurrentAnnotations True
                         currentScript <- Mr.asks currentFilename
-                        paths <- mapMaybe getSourcePath <$> getCurrentAnnotations True
+                        let paths = mapMaybe getSourcePath allAnnotations
-                        resolved <- system $ siFindSource sys currentScript paths filename
+                        let externalSources = listToMaybe $ mapMaybe getExternalSources allAnnotations
-                        contents <- system $ siReadFile sys resolved
+                        resolved <- system $ siFindSource sys currentScript externalSources paths filename
+                        contents <- system $ siReadFile sys externalSources resolved
                         return (contents, resolved)
                 case input of
                     Left err -> do
@@ -2190,6 +2234,11 @@ readSource t@(T_Redirecting _ _ (T_SimpleCommand cmdId _ (cmd:file':rest'))) = d
             SourcePath x -> Just x
             _ -> Nothing
 
+    getExternalSources t =
+        case t of
+            ExternalSources b -> Just b
+            _ -> Nothing
+
     -- If the word has a single expansion as the directory, try stripping it
     -- This affects `$foo/bar` but not `${foo}-dir/bar` or `/foo/$file`
     stripDynamicPrefix word =
@@ -2306,7 +2355,7 @@ readCmdName = do
     -- Ignore alias suppression
     optional . try $ do
         char '\\'
-        lookAhead $ variableChars
+        lookAhead $ variableChars <|> oneOf ":."
     readCmdWord
 
 readCmdWord = do
@@ -2792,7 +2841,7 @@ readLetSuffix = many1 (readIoRedirect <|> try readLetExpression <|> readCmdWord)
         startPos <- getPosition
         expression <- readStringForParser readCmdWord
         let (unQuoted, newPos) = kludgeAwayQuotes expression startPos
-        subParse newPos readArithmeticContents unQuoted
+        subParse newPos (readArithmeticContents <* eof) unQuoted
 
     kludgeAwayQuotes :: String -> SourcePos -> (String, SourcePos)
     kludgeAwayQuotes s p =
@@ -3179,7 +3228,7 @@ prop_readConfigKVs4 = isOk readConfigKVs "\n\n\n\n\t \n"
 prop_readConfigKVs5 = isOk readConfigKVs "# shellcheck accepts annotation-like comments in rc files\ndisable=1234"
 readConfigKVs = do
     anySpacingOrComment
-    annotations <- many (readAnnotationWithoutPrefix <* anySpacingOrComment)
+    annotations <- many (readAnnotationWithoutPrefix False <* anySpacingOrComment)
     eof
     return $ concat annotations
 anySpacingOrComment =
@@ -3216,8 +3265,8 @@ readScriptFile sourced = do
     let ignoreShebang = shellAnnotationSpecified || shellFlagSpecified
 
     unless ignoreShebang $
-        verifyShebang pos (getShell shebangString)
+        verifyShebang pos (executableFromShebang shebangString)
-    if ignoreShebang || isValidShell (getShell shebangString) /= Just False
+    if ignoreShebang || isValidShell (executableFromShebang shebangString) /= Just False
       then do
             commands <- withAnnotations annotations readCompoundListOrEmpty
             id <- endSpan start
@@ -3231,17 +3280,6 @@ readScriptFile sourced = do
             return $ T_Script id shebang []
 
   where
-    basename s = reverse . takeWhile (/= '/') . reverse $ s
-    skipFlags = dropWhile ("-" `isPrefixOf`)
-    getShell sb =
-        case words sb of
-            [] -> ""
-            [x] -> basename x
-            (first:args) ->
-                if basename first == "env"
-                    then fromMaybe "" $ find (notElem '=') $ skipFlags args
-                    else basename first
-
     verifyShebang pos s = do
         case isValidShell s of
             Just True -> return ()
@@ -3334,13 +3372,13 @@ parsesCleanly parser string = runIdentity $ do
 
 -- For printf debugging: print the value of an expression
 -- Example: return $ dump $ T_Literal id [c]
-dump :: Show a => a -> a
+dump :: Show a => a -> a     -- STRIP
-dump x = trace (show x) x
+dump x = trace (show x) x    -- STRIP
 
 -- Like above, but print a specific expression:
 -- Example: return $ dumps ("Returning: " ++ [c])  $ T_Literal id [c]
-dumps :: Show x => x -> a -> a
+dumps :: Show x => x -> a -> a -- STRIP
-dumps t = trace (show t)
+dumps t = trace (show t)       -- STRIP
 
 parseWithNotes parser = do
     item <- parser

stack.yaml

@@ -2,7 +2,7 @@
 # For more information, see: https://docs.haskellstack.org/en/stable/yaml_configuration/
 
 # Specifies the GHC version and set of packages available (e.g., lts-3.5, nightly-2015-09-21, ghc-7.10.2)
-resolver: lts-13.26
+resolver: lts-18.15
 
 # Local packages, usually specified by relative directory name
 packages:

striptests

@@ -29,6 +29,7 @@ detestify() {
       state = 0;
     }
 
+    /STRIP/ { next; }
     /LANGUAGE TemplateHaskell/ { next; }
     /^import.*Test\./ { next; }
 
@@ -75,4 +76,3 @@ find . -name '.git' -prune -o -type f -name '*.hs' -print |
   do
     modify "$file" detestify
   done
-

test/buildtest

@@ -29,6 +29,8 @@ cabal build ||
   die "build failed"
 cabal test ||
   die "test failed"
+cabal haddock ||
+  die "haddock failed"
 
 sc="$(find . -name shellcheck -type f -perm -111)"
 [ -x "$sc" ] || die "Can't find executable"

test/check_release

@@ -56,19 +56,19 @@ cat << EOF
 Manual Checklist
 
 $((i++)). Make sure none of the automated checks above failed
-$((i++)). Make sure Travis build currently passes: https://travis-ci.org/koalaman/shellcheck
+$((i++)). Make sure GitHub Build currently passes: https://github.com/koalaman/shellcheck/actions
 $((i++)). Make sure SnapCraft build currently works: https://build.snapcraft.io/user/koalaman
 $((i++)). Run test/distrotest to ensure that most distros can build OOTB.
 $((i++)). Format and read over the manual for bad formatting and outdated info.
-$((i++)). Make sure the Hackage package builds, so that all files are 
+$((i++)). Make sure the Hackage package builds.
 
 Release Steps
 
 $((j++)). \`cabal sdist\` to generate a Hackage package
 $((j++)). \`git push --follow-tags\` to push commit
-$((j++)). Wait for Travis to build
+$((j++)). Wait for GitHub Actions to build.
 $((j++)). Verify release:
-  a. Check that the new versions are uploaded: https://shellcheck.storage.googleapis.com/index.html
+  a. Check that the new versions are uploaded: https://github.com/koalaman/shellcheck/tags
   b. Check that the docker images have version tags: https://hub.docker.com/u/koalaman
 $((j++)). If no disaster, upload to Hackage: http://hackage.haskell.org/upload
 $((j++)). Push a new commit that updates CHANGELOG.md

test/distrotest

@@ -64,13 +64,13 @@ ubuntu:latest         apt-get update && apt-get install -y cabal-install
 haskell:latest        true
 opensuse/leap:latest  zypper install -y cabal-install ghc
 fedora:latest         dnf install -y cabal-install ghc-template-haskell-devel findutils
-archlinux/base:latest pacman -S -y --noconfirm cabal-install ghc-static base-devel
+archlinux:latest      pacman -S -y --noconfirm cabal-install ghc-static base-devel
 
-# Other versions we want to support
+# Ubuntu LTS
-ubuntu:18.04          apt-get update && apt-get install -y cabal-install
+ubuntu:20.04          apt-get update && apt-get install -y cabal-install
 
-# Misc Haskell including current and latest Stack build
+# Stack on Ubuntu LTS
-ubuntu:18.04          set -e; apt-get update && apt-get install -y curl && curl -sSL https://get.haskellstack.org/ | sh -s - -f && cd /mnt && exec test/stacktest
+ubuntu:20.04          set -e; apt-get update && apt-get install -y curl && curl -sSL https://get.haskellstack.org/ | sh -s - -f && cd /mnt && exec test/stacktest
 EOF
 
 exit "$final"

test/shellcheck.hs

@@ -4,6 +4,7 @@ import Control.Monad
 import System.Exit
 import qualified ShellCheck.Analytics
 import qualified ShellCheck.AnalyzerLib
+import qualified ShellCheck.ASTLib
 import qualified ShellCheck.Checker
 import qualified ShellCheck.Checks.Commands
 import qualified ShellCheck.Checks.Custom
@@ -17,6 +18,7 @@ main = do
     results <- sequence [
         ShellCheck.Analytics.runTests
         ,ShellCheck.AnalyzerLib.runTests
+        ,ShellCheck.ASTLib.runTests
         ,ShellCheck.Checker.runTests
         ,ShellCheck.Checks.Commands.runTests
         ,ShellCheck.Checks.Custom.runTests

test/stacktest

@@ -18,10 +18,11 @@ command -v stack ||
 stack setup        || die "Failed to setup with default resolver"
 stack build --test || die "Failed to build/test with default resolver"
 
+# Nice to haves, but not necessary
 for resolver in "${resolvers[@]}"
 do
-  stack --resolver="$resolver" setup        || die "Failed to setup $resolver"
+  stack --resolver="$resolver" setup        || die "Failed to setup $resolver. This probably doesn't matter."
-  stack --resolver="$resolver" build --test || die "Failed build/test with $resolver!"
+  stack --resolver="$resolver" build --test || die "Failed build/test with $resolver! This probably doesn't matter."
 done
 
 echo "Success"