Drop attoparsec/text dependencies

improve short description for SC2038

.github/dependabot.yml

@@ -0,0 +1,7 @@
+version: 2
+
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"

.multi_arch_docker

@@ -3,28 +3,10 @@
 # binaries previously built and deployed to GitHub.
 
 function multi_arch_docker::install_docker_buildx() {
-  # Install up-to-date version of docker, with buildx support.
-  local -r docker_apt_repo='https://download.docker.com/linux/ubuntu'
-  curl -fsSL "${docker_apt_repo}/gpg" | sudo apt-key add -
-  local -r os="$(lsb_release -cs)"
-  sudo add-apt-repository "deb [arch=amd64] $docker_apt_repo $os stable"
-  sudo apt-get update
-  sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
-
-  # Enable docker daemon experimental support (for 'pull --platform').
-  local -r config='/etc/docker/daemon.json'
-  if [[ -e "$config" ]]; then
-    sudo sed -i -e 's/{/{ "experimental": true, /' "$config"
-  else
-    echo '{ "experimental": true }' | sudo tee "$config"
-  fi
-  sudo systemctl restart docker
-
   # Install QEMU multi-architecture support for docker buildx.
   docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
 
   # Instantiate docker buildx builder with multi-architecture support.
-  export DOCKER_CLI_EXPERIMENTAL=enabled
   docker buildx create --name mybuilder
   docker buildx use mybuilder
   # Start up buildx and verify that all is OK.

CHANGELOG.md

@@ -1,3 +1,15 @@
+## Git
+### Added
+- SC2324: Warn when x+=1 appends instead of increments
+- SC2325: Warn about multiple `!`s in dash/sh.
+- SC2326: Warn about `foo | ! bar` in bash/dash/sh.
+
+### Fixed
+- source statements with here docs now work correctly
+
+### Changed
+
+
 ## v0.9.0 - 2022-12-12
 ### Added
 - SC2316: Warn about 'local readonly foo' and similar (thanks, patrickxia!)

README.md

@@ -112,6 +112,7 @@ Services and platforms that have ShellCheck pre-installed and ready to use:
 * [Code Factor](https://www.codefactor.io/)
 * [CircleCI](https://circleci.com) via the [ShellCheck Orb](https://circleci.com/orbs/registry/orb/circleci/shellcheck)
 * [Github](https://github.com/features/actions) (only Linux)
+* [Trunk Check](https://trunk.io/products/check) (universal linter; [allows you to explicitly version your shellcheck install](https://github.com/trunk-io/plugins/blob/bcbb361dcdbe4619af51ea7db474d7fb87540d20/.trunk/trunk.yaml#L32)) via the [shellcheck plugin](https://github.com/trunk-io/plugins/blob/main/linters/shellcheck/plugin.yaml)
 
 Most other services, including [GitLab](https://about.gitlab.com/), let you install
 ShellCheck yourself, either through the system's package manager (see [Installing](#installing)),
@@ -232,6 +233,9 @@ Alternatively, you can download pre-compiled binaries for the latest release her
 or see the [GitHub Releases](https://github.com/koalaman/shellcheck/releases) for other releases
 (including the [latest](https://github.com/koalaman/shellcheck/releases/tag/latest) meta-release for daily git builds).
 
+There are currently no official binaries for Apple Silicon, but third party builds are available via
+[ShellCheck for Visual Studio Code](https://github.com/vscode-shellcheck/shellcheck-binaries/releases).
+
 Distro packages already come with a `man` page. If you are building from source, it can be installed with:
 
 ```console

ShellCheck.cabal

@@ -46,7 +46,7 @@ library
         semigroups
     build-depends:
       -- The lower bounds are based on GHC 7.10.3
-      -- The upper bounds are based on GHC 9.4.3
+      -- The upper bounds are based on GHC 9.6.1
       aeson                >= 1.4.0 && < 2.2,
       array                >= 0.5.1 && < 0.6,
       base                 >= 4.8.0.0 && < 5,
@@ -54,13 +54,13 @@ library
       containers           >= 0.5.6 && < 0.7,
       deepseq              >= 1.4.1 && < 1.5,
       Diff                 >= 0.4.0 && < 0.5,
-      fgl                  >= 5.7.0 && < 5.9,
+      fgl                  (>= 5.7.0 && < 5.8.1.0) || (>= 5.8.1.1 && < 5.9),
       filepath             >= 1.4.0 && < 1.5,
-      mtl                  >= 2.2.2 && < 2.3,
+      mtl                  >= 2.2.2 && < 2.4,
       parsec               >= 3.1.14 && < 3.2,
       QuickCheck           >= 2.14.2 && < 2.15,
       regex-tdfa           >= 1.2.0 && < 1.4,
-      transformers         >= 0.4.2 && < 0.6,
+      transformers         >= 0.4.2 && < 0.7,
 
       -- getXdgDirectory from 1.2.3.0
       directory            >= 1.2.3 && < 1.4,
@@ -93,6 +93,7 @@ library
       ShellCheck.Formatter.Quiet
       ShellCheck.Interface
       ShellCheck.Parser
+      ShellCheck.PortageVariables
       ShellCheck.Prelude
       ShellCheck.Regex
     other-modules:

build/darwin.x86_64/Dockerfile

@@ -6,15 +6,18 @@ ENV TARGETNAME darwin.x86_64
 # Build dependencies
 USER root
 ENV DEBIAN_FRONTEND noninteractive
-RUN apt-get update && apt-get install -y ghc automake autoconf llvm curl
+RUN sed -e 's/focal/kinetic/g' -i /etc/apt/sources.list
+RUN apt-get update
+RUN apt-get dist-upgrade -y
+RUN apt-get install -y ghc automake autoconf llvm curl alex happy
 
 # Build GHC
 WORKDIR /ghc
-RUN curl -L "https://downloads.haskell.org/~ghc/8.10.4/ghc-8.10.4-src.tar.xz" | tar xJ --strip-components=1
+RUN curl -L "https://downloads.haskell.org/~ghc/9.2.5/ghc-9.2.5-src.tar.xz" | tar xJ --strip-components=1
-RUN ./boot && ./configure --host x86_64-linux-gnu --build x86_64-linux-gnu --target "$TARGET"
+RUN ./configure --host x86_64-linux-gnu --build x86_64-linux-gnu --target "$TARGET"
 RUN cp mk/flavours/quick-cross.mk mk/build.mk && make -j "$(nproc)"
 RUN make install
-RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.2.0.0/cabal-install-3.2.0.0-x86_64-unknown-linux.tar.xz" | tar xJv -C /usr/local/bin
+RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.9.0.0/cabal-install-3.9-x86_64-linux-alpine.tar.xz" | tar xJv -C /usr/local/bin
 
 # Due to an apparent cabal bug, we specify our options directly to cabal
 # It won't reuse caches if ghc-options are specified in ~/.cabal/config

build/linux.aarch64/Dockerfile

@@ -6,19 +6,25 @@ ENV TARGETNAME linux.aarch64
 # Build dependencies
 USER root
 ENV DEBIAN_FRONTEND noninteractive
-RUN apt-get update && apt-get install -y ghc automake autoconf build-essential llvm curl qemu-user-static gcc-$TARGET
+
+# These deps are from 20.04, because GHC's compiler/llvm support moves slowly
+RUN apt-get update && apt-get install -y llvm gcc-$TARGET
+
+# The rest are from 22.10
+RUN sed -e 's/focal/kinetic/g' -i /etc/apt/sources.list
+RUN apt-get update && apt-get install -y ghc alex happy automake autoconf build-essential curl qemu-user-static
 
 # Build GHC
 WORKDIR /ghc
-RUN curl -L "https://downloads.haskell.org/~ghc/8.10.4/ghc-8.10.4-src.tar.xz" | tar xJ --strip-components=1
+RUN curl -L "https://downloads.haskell.org/~ghc/9.2.5/ghc-9.2.5-src.tar.xz" | tar xJ --strip-components=1
 RUN ./boot && ./configure --host x86_64-linux-gnu --build x86_64-linux-gnu --target "$TARGET"
 RUN cp mk/flavours/quick-cross.mk mk/build.mk && make -j "$(nproc)"
 RUN make install
-RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.2.0.0/cabal-install-3.2.0.0-x86_64-unknown-linux.tar.xz" | tar xJv -C /usr/local/bin
+RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.9.0.0/cabal-install-3.9-x86_64-linux-alpine.tar.xz" | tar xJv -C /usr/local/bin
 
 # Due to an apparent cabal bug, we specify our options directly to cabal
 # It won't reuse caches if ghc-options are specified in ~/.cabal/config
-ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections;--with-ghc=$TARGET-ghc;--with-hc-pkg=$TARGET-ghc-pkg"
+ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections -optc-fPIC;--with-ghc=$TARGET-ghc;--with-hc-pkg=$TARGET-ghc-pkg"
 
 # Prebuild the dependencies
 RUN cabal update && IFS=';' && cabal install --dependencies-only $CABALOPTS ShellCheck

shellcheck.1.md

@@ -87,8 +87,9 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts.
 
 :   Specify Bourne shell dialect. Valid values are *sh*, *bash*, *dash* and *ksh*.
     The default is to deduce the shell from the file's `shell` directive,
-    shebang, or `.bash/.bats/.dash/.ksh` extension, in that order. *sh* refers to
+    shebang, or `.bash/.bats/.dash/.ksh/.ebuild/.eclass` extension, in that
-    POSIX `sh` (not the system's), and will warn of portability issues.
+    order. *sh* refers to POSIX `sh` (not the system's), and will warn of
+    portability issues.
 
 **-S**\ *SEVERITY*,\ **--severity=***severity*
 

shellcheck.hs

@@ -21,6 +21,7 @@ import qualified ShellCheck.Analyzer
 import           ShellCheck.Checker
 import           ShellCheck.Data
 import           ShellCheck.Interface
+import           ShellCheck.PortageVariables
 import           ShellCheck.Regex
 
 import qualified ShellCheck.Formatter.CheckStyle
@@ -396,10 +397,12 @@ ioInterface options files = do
     inputs <- mapM normalize files
     cache <- newIORef emptyCache
     configCache <- newIORef ("", Nothing)
-    return SystemInterface {
+    portageVars <- newIORef Nothing
+    return (newSystemInterface :: SystemInterface IO) {
         siReadFile = get cache inputs,
         siFindSource = findSourceFile inputs (sourcePaths options),
-        siGetConfig = getConfig configCache
+        siGetConfig = getConfig configCache,
+        siGetPortageVariables = getOrLoadPortage portageVars
     }
   where
     emptyCache :: Map.Map FilePath String
@@ -523,6 +526,21 @@ ioInterface options files = do
                 ("SCRIPTDIR":rest) -> joinPath (scriptdir:rest)
                 _ -> str
 
+    getOrLoadPortage cache = do
+        x <- readIORef cache
+        case x of
+            Just m -> do
+                return m
+            Nothing -> do
+                vars <- readPortageVariables `catch` handler
+                writeIORef cache $ Just vars
+                return vars
+      where
+        handler :: IOException -> IO (Map.Map String [String])
+        handler e = do
+            hPutStrLn stderr $ "Error finding portage repos, eclass definitions will be ignored: " ++ show e
+            return $ Map.empty
+
 inputFile file = do
     (handle, shouldCache) <-
             if file == "-"

src/ShellCheck/ASTLib.hs

@@ -36,8 +36,6 @@ import Numeric (showHex)
 
 import Test.QuickCheck
 
-arguments (T_SimpleCommand _ _ (cmd:args)) = args
-
 -- Is this a type of loop?
 isLoop t = case t of
         T_WhileExpression {} -> True
@@ -546,17 +544,16 @@ getCommandNameAndToken direct t = fromMaybe (Nothing, t) $ do
                     return t
                 _ -> fail ""
 
--- If a command substitution is a single command, get its name.
+-- If a command substitution is a single SimpleCommand, return it.
---  $(date +%s) = Just "date"
+getSimpleCommandFromExpansion :: Token -> Maybe Token
-getCommandNameFromExpansion :: Token -> Maybe String
+getSimpleCommandFromExpansion t =
-getCommandNameFromExpansion t =
     case t of
         T_DollarExpansion _ [c] -> extract c
         T_Backticked _ [c] -> extract c
         T_DollarBraceCommandExpansion _ [c] -> extract c
         _ -> Nothing
   where
-    extract (T_Pipeline _ _ [cmd]) = getCommandName cmd
+    extract (T_Pipeline _ _ [c]) = getCommand c
     extract _ = Nothing
 
 -- Get the basename of a token representing a command
@@ -564,6 +561,10 @@ getCommandBasename = fmap basename . getCommandName
 
 basename = reverse . takeWhile (/= '/') . reverse
 
+-- Get the arguments to a command
+arguments (T_SimpleCommand _ _ (cmd:args)) = args
+arguments t = maybe [] arguments (getCommand t)
+
 isAssignment t =
     case t of
         T_Redirecting _ _ w -> isAssignment w
@@ -886,6 +887,15 @@ isUnmodifiedParameterExpansion t =
             in getBracedReference str == str
         _ -> False
 
+-- Return the referenced variable if (and only if) it's an unmodified parameter expansion.
+getUnmodifiedParameterExpansion t =
+    case t of
+        T_DollarBraced _ _ list -> do
+            let str = concat $ oversimplify list
+            guard $ getBracedReference str == str
+            return str
+        _ -> Nothing
+
 --- A list of the element and all its parents up to the root node.
 getPath tree t = t :
     case Map.lookup (getId t) tree of
@@ -903,5 +913,20 @@ getEnableDirectives root =
         T_Annotation _ list _ -> [s | EnableComment s <- list]
         _ -> []
 
+
+commandExpansionShouldBeSplit t = do
+    cmd <- getSimpleCommandFromExpansion t
+    name <- getCommandName cmd
+    case () of
+        -- Should probably be split
+        _ | name `elem` ["seq", "pgrep"] -> return True
+        -- Portage macros that return a single word or nothing
+        _ | name `elem` ["usev", "use_with", "use_enable"] -> return True
+        -- Portage macros that are fine as long as the arguments have no spaces
+        _ | name `elem` ["usex", "meson_use", "meson_feature"] -> do
+            return . not $ any (' ' `elem`) $ map (getLiteralStringDef " ") $ arguments cmd
+        _ -> Nothing
+
+
 return []
 runTests = $quickCheckAll

src/ShellCheck/Analytics.hs

@@ -201,6 +201,7 @@ nodeChecks = [
     ,checkOverwrittenExitCode
     ,checkUnnecessaryArithmeticExpansionIndex
     ,checkUnnecessaryParens
+    ,checkPlusEqualsNumber
     ]
 
 optionalChecks = map fst optionalTreeChecks
@@ -313,7 +314,7 @@ runAndGetComments f s = do
         let pr = pScript s
         root <- prRoot pr
         let spec = defaultSpec pr
-        let params = makeParameters spec
+        let params = runIdentity $ makeParameters (mockedSystemInterface []) spec
         return $
             filterByAnnotation spec params $
                 f params root
@@ -562,7 +563,7 @@ checkPipePitfalls _ (T_Pipeline id _ commands) = do
                 hasParameter "print0",
                 hasParameter "printf"
               ]) $ warn (getId find) 2038
-                      "Use -print0/-0 or -exec + to allow for non-alphanumeric filenames."
+                      "Use 'find .. -print0 | xargs -0 ..' or 'find .. -exec .. +' to allow non-alphanumeric filenames."
 
     for ["ps", "grep"] $
         \(ps:grep:_) ->
@@ -791,6 +792,11 @@ prop_checkUnquotedExpansions8 = verifyNot checkUnquotedExpansions "set -- $(seq
 prop_checkUnquotedExpansions9 = verifyNot checkUnquotedExpansions "echo foo `# inline comment`"
 prop_checkUnquotedExpansions10 = verify checkUnquotedExpansions "#!/bin/sh\nexport var=$(val)"
 prop_checkUnquotedExpansions11 = verifyNot checkUnquotedExpansions "ps -p $(pgrep foo)"
+prop_checkUnquotedExpansions12 = verify checkUnquotedExpansions "#!/bin/sh\nexport var=$(val)"
+prop_checkUnquotedExpansions13 = verifyNot checkUnquotedExpansions "echo $(usev X)"
+prop_checkUnquotedExpansions14 = verifyNot checkUnquotedExpansions "echo $(usex X \"\" Y)"
+prop_checkUnquotedExpansions15 = verify checkUnquotedExpansions "echo $(usex X \"Y Z\" W)"
+
 checkUnquotedExpansions params =
     check
   where
@@ -800,12 +806,9 @@ checkUnquotedExpansions params =
     check _ = return ()
     tree = parentMap params
     examine t contents =
-        unless (null contents || shouldBeSplit t || isQuoteFree (shellType params) tree t || usedAsCommandName tree t) $
+        unless (null contents || commandExpansionShouldBeSplit t == Just True || isQuoteFree (shellType params) tree t || usedAsCommandName tree t) $
             warn (getId t) 2046 "Quote this to prevent word splitting."
 
-    shouldBeSplit t =
-        getCommandNameFromExpansion t `elem` [Just "seq", Just "pgrep"]
-
 
 prop_checkRedirectToSame = verify checkRedirectToSame "cat foo > foo"
 prop_checkRedirectToSame2 = verify checkRedirectToSame "cat lol | sed -e 's/a/b/g' > lol"
@@ -1036,9 +1039,6 @@ checkStderrRedirect params redir@(T_Redirecting _ [
 
 checkStderrRedirect _ _ = return ()
 
-lt x = trace ("Tracing " ++ show x) x -- STRIP
-ltt t = trace ("Tracing " ++ show t)  -- STRIP
-
 
 prop_checkSingleQuotedVariables  = verify checkSingleQuotedVariables "echo '$foo'"
 prop_checkSingleQuotedVariables2 = verify checkSingleQuotedVariables "echo 'lol$1.jpg'"
@@ -1069,6 +1069,10 @@ prop_checkSingleQuotedVariables22 = verifyNot checkSingleQuotedVariables "jq '$_
 prop_checkSingleQuotedVariables23 = verifyNot checkSingleQuotedVariables "command jq '$__loc__'"
 prop_checkSingleQuotedVariables24 = verifyNot checkSingleQuotedVariables "exec jq '$__loc__'"
 prop_checkSingleQuotedVariables25 = verifyNot checkSingleQuotedVariables "exec -c -a foo jq '$__loc__'"
+prop_checkSingleQuotedVariablesCros1 = verifyNot checkSingleQuotedVariables "python_gen_any_dep 'dev-python/pyyaml[${PYTHON_USEDEP}]'"
+prop_checkSingleQuotedVariablesCros2 = verifyNot checkSingleQuotedVariables "python_gen_cond_dep 'dev-python/unittest2[${PYTHON_USEDEP}]' python2_7 pypy"
+prop_checkSingleQuotedVariablesCros3 = verifyNot checkSingleQuotedVariables "version_format_string '${PN}_source_$1_$2-$3_$4'"
+
 
 
 checkSingleQuotedVariables params t@(T_SingleQuoted id s) =
@@ -1108,6 +1112,9 @@ checkSingleQuotedVariables params t@(T_SingleQuoted id s) =
                 ,"git filter-branch"
                 ,"mumps -run %XCMD"
                 ,"mumps -run LOOP%XCMD"
+                ,"python_gen_any_dep"
+                ,"python_gen_cond_dep"
+                ,"version_format_string"
                 ]
             || "awk" `isSuffixOf` commandName
             || "perl" `isPrefixOf` commandName
@@ -2134,7 +2141,8 @@ checkSpacefulnessCfg' dirtyPass params token@(T_DollarBraced id _ list) =
                     addDoubleQuotesAround params token
 
   where
-    name = getBracedReference $ concat $ oversimplify list
+    bracedString = concat $ oversimplify list
+    name = getBracedReference bracedString
     parents = parentMap params
     needsQuoting =
               not (isArrayExpansion token) -- There's another warning for this
@@ -2153,14 +2161,10 @@ checkSpacefulnessCfg' dirtyPass params token@(T_DollarBraced id _ list) =
         || CF.spaceStatus (CF.variableValue state) == CF.SpaceStatusClean
 
     isDefaultAssignment parents token =
-        let modifier = getBracedModifier $ bracedString token in
+        let modifier = getBracedModifier bracedString in
             any (`isPrefixOf` modifier) ["=", ":="]
             && isParamTo parents ":" token
 
-    -- Given a T_DollarBraced, return a simplified version of the string contents.
-    bracedString (T_DollarBraced _ _ l) = concat $ oversimplify l
-    bracedString _ = error $ pleaseReport "bracedString on non-variable"
-
 checkSpacefulnessCfg' _ _ _ = return ()
 
 
@@ -2394,7 +2398,7 @@ checkUnusedAssignments params t = execWriter (mapM_ warnFor unused)
                 name ++ " appears unused. Verify use (or export if used externally)."
 
     stripSuffix = takeWhile isVariableChar
-    defaultMap = Map.fromList $ zip internalVariables $ repeat ()
+    defaultMap = Map.fromList $ zip (internalVariables ++ additionalKnownVariables params) $ repeat ()
 
 prop_checkUnassignedReferences1 = verifyTree checkUnassignedReferences "echo $foo"
 prop_checkUnassignedReferences2 = verifyNotTree checkUnassignedReferences "foo=hello; echo $foo"
@@ -2453,7 +2457,7 @@ checkUnassignedReferences = checkUnassignedReferences' False
 checkUnassignedReferences' includeGlobals params t = warnings
   where
     (readMap, writeMap) = execState (mapM tally $ variableFlow params) (Map.empty, Map.empty)
-    defaultAssigned = Map.fromList $ map (\a -> (a, ())) $ filter (not . null) internalVariables
+    defaultAssigned = Map.fromList $ map (\a -> (a, ())) $ filter (not . null) (internalVariables ++ additionalKnownVariables params)
 
     tally (Assignment (_, _, name, _))  =
         modify (\(read, written) -> (read, Map.insert name () written))
@@ -3511,6 +3515,7 @@ prop_checkSplittingInArrays5 = verifyNot checkSplittingInArrays "a=( $! $$ $# )"
 prop_checkSplittingInArrays6 = verifyNot checkSplittingInArrays "a=( ${#arr[@]} )"
 prop_checkSplittingInArrays7 = verifyNot checkSplittingInArrays "a=( foo{1,2} )"
 prop_checkSplittingInArrays8 = verifyNot checkSplittingInArrays "a=( * )"
+prop_checkSplittingInArrays9 = verifyNot checkSplittingInArrays "a=( $(use_enable foo) )"
 checkSplittingInArrays params t =
     case t of
         T_Array _ elements -> mapM_ check elements
@@ -3520,6 +3525,7 @@ checkSplittingInArrays params t =
         T_NormalWord _ parts -> mapM_ checkPart parts
         _ -> return ()
     checkPart part = case part of
+        _ | commandExpansionShouldBeSplit part == Just True -> return ()
         T_DollarExpansion id _ -> forCommand id
         T_DollarBraceCommandExpansion id _ -> forCommand id
         T_Backticked id _ -> forCommand id
@@ -5010,5 +5016,42 @@ checkUnnecessaryParens params t =
         ]
 
 
+prop_checkPlusEqualsNumber1 = verify checkPlusEqualsNumber "x+=1"
+prop_checkPlusEqualsNumber2 = verify checkPlusEqualsNumber "x+=42"
+prop_checkPlusEqualsNumber3 = verifyNot checkPlusEqualsNumber "(( x += 1 ))"
+prop_checkPlusEqualsNumber4 = verifyNot checkPlusEqualsNumber "declare -i x=0; x+=1"
+prop_checkPlusEqualsNumber5 = verifyNot checkPlusEqualsNumber "x+='1'"
+prop_checkPlusEqualsNumber6 = verifyNot checkPlusEqualsNumber "n=foo; x+=n"
+prop_checkPlusEqualsNumber7 = verify checkPlusEqualsNumber "n=4; x+=n"
+prop_checkPlusEqualsNumber8 = verify checkPlusEqualsNumber "n=4; x+=$n"
+prop_checkPlusEqualsNumber9 = verifyNot checkPlusEqualsNumber "declare -ia var; var[x]+=1"
+checkPlusEqualsNumber params t =
+    case t of
+        T_Assignment id Append var _ word -> sequence_ $ do
+            state <- CF.getIncomingState (cfgAnalysis params) id
+            guard $ isNumber state word
+            guard . not $ fromMaybe False $ CF.variableMayBeDeclaredInteger state var
+            return $ warn id 2324 "var+=1 will append, not increment. Use (( var += 1 )), declare -i var, or quote number to silence."
+        _ -> return ()
+
+  where
+    isNumber state word =
+        let
+            unquotedLiteral = getUnquotedLiteral word
+            isEmpty = unquotedLiteral == Just ""
+            isUnquotedNumber = not isEmpty && fromMaybe False (all isDigit <$> unquotedLiteral)
+            isNumericalVariableName = fromMaybe False $ do
+                str <- unquotedLiteral
+                CF.variableMayBeAssignedInteger state str
+            isNumericalVariableExpansion =
+                case word of
+                    T_NormalWord _ [part] -> fromMaybe False $ do
+                        str <- getUnmodifiedParameterExpansion part
+                        CF.variableMayBeAssignedInteger state str
+                    _ -> False
+        in
+            isUnquotedNumber || isNumericalVariableName || isNumericalVariableExpansion
+
+
 return []
 runTests =  $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |])

src/ShellCheck/Analyzer.hs

@@ -31,14 +31,14 @@ import qualified ShellCheck.Checks.ShellSupport
 
 
 -- TODO: Clean up the cruft this is layered on
-analyzeScript :: AnalysisSpec -> AnalysisResult
+analyzeScript :: Monad m => SystemInterface m -> AnalysisSpec -> m AnalysisResult
-analyzeScript spec = newAnalysisResult {
+analyzeScript sys spec = do
+    params <- makeParameters sys spec
+    return $ newAnalysisResult {
             arComments =
                 filterByAnnotation spec params . nub $
                     runChecker params (checkers spec params)
-}
+        }
-  where
-    params = makeParameters spec
 
 checkers spec params = mconcat $ map ($ params) [
     ShellCheck.Analytics.checker spec,

src/ShellCheck/AnalyzerLib.hs

@@ -88,6 +88,8 @@ data Parameters = Parameters {
     hasSetE            :: Bool,
     -- Whether this script has 'set -o pipefail' anywhere.
     hasPipefail        :: Bool,
+    -- Whether this script is an Ebuild file.
+    isPortage          :: Bool,
     -- A linear (bad) analysis of data flow
     variableFlow       :: [StackData],
     -- A map from Id to Token
@@ -103,9 +105,12 @@ data Parameters = Parameters {
     -- map from token id to start and end position
     tokenPositions     :: Map.Map Id (Position, Position),
     -- Result from Control Flow Graph analysis (including data flow analysis)
-    cfgAnalysis :: CF.CFGAnalysis
+    cfgAnalysis :: CF.CFGAnalysis,
+    -- A set of additional variables known to be set (TODO: make this more general?)
+    additionalKnownVariables :: [String]
     } deriving (Show)
 
+
 -- TODO: Cache results of common AST ops here
 data Cache = Cache {}
 
@@ -152,7 +157,7 @@ producesComments c s = do
         let pr = pScript s
         prRoot pr
         let spec = defaultSpec pr
-        let params = makeParameters spec
+        let params = runIdentity $ makeParameters (mockedSystemInterface []) spec
         return . not . null $ filterByAnnotation spec params $ runChecker params c
 
 makeComment :: Severity -> Id -> Code -> String -> TokenComment
@@ -196,11 +201,25 @@ makeCommentWithFix severity id code str fix =
         }
     in force withFix
 
-makeParameters spec = params
+makeParameters :: Monad m => SystemInterface m -> AnalysisSpec -> m Parameters
+makeParameters sys spec = do
+    extraVars <-
+        if asIsPortage spec
+        then do
+            vars <- siGetPortageVariables sys
+            let classes = getInheritedEclasses root
+            return $ concatMap (\c -> Map.findWithDefault [] c vars) classes
+        else
+            return []
+
+    return $ makeParams extraVars
+  where
+    shell = fromMaybe (determineShell (asFallbackShell spec) root) $ asShellType spec
+    makeParams extraVars = params
       where
         params = Parameters {
             rootNode = root,
-        shellType = fromMaybe (determineShell (asFallbackShell spec) root) $ asShellType spec,
+            shellType = shell,
             hasSetE = containsSetE root,
             hasLastpipe =
                 case shellType params of
@@ -221,15 +240,18 @@ makeParameters spec = params
                     Sh   -> True
                     Ksh  -> isOptionSet "pipefail" root,
             shellTypeSpecified = isJust (asShellType spec) || isJust (asFallbackShell spec),
+            isPortage = asIsPortage spec,
             idMap = getTokenMap root,
             parentMap = getParentTree root,
             variableFlow = getVariableFlow params root,
             tokenPositions = asTokenPositions spec,
-        cfgAnalysis = CF.analyzeControlFlow cfParams root
+            cfgAnalysis = CF.analyzeControlFlow cfParams root,
+            additionalKnownVariables = extraVars
         }
         cfParams = CF.CFGParameters {
             CF.cfLastpipe = hasLastpipe params,
-        CF.cfPipefail = hasPipefail params
+            CF.cfPipefail = hasPipefail params,
+            CF.cfAdditionalInitialVariables = additionalKnownVariables params
         }
     root = asScript spec
 
@@ -582,6 +604,17 @@ getReferencedVariableCommand base@(T_SimpleCommand _ _ (T_NormalWord _ (T_Litera
                 head:_ -> map (\x -> (base, head, x)) $ getVariablesFromLiteralToken head
                 _ -> []
         "alias" -> [(base, token, name) | token <- rest, name <- getVariablesFromLiteralToken token]
+
+        -- tc-export makes a list of toolchain variables available, similar to export.
+        -- Usage tc-export CC CXX
+        "tc-export" -> concatMap getReference rest
+
+        -- tc-export_build_env exports the listed variables plus a bunch of BUILD_XX variables.
+        -- Usage tc-export_build_env BUILD_CC
+        "tc-export_build_env" ->
+            concatMap getReference rest
+                ++ [ (base, base, v) | v <- portageBuildEnvVariables ]
+
         _ -> []
   where
     forDeclare =
@@ -653,6 +686,16 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T
         "DEFINE_integer" -> maybeToList $ getFlagVariable rest
         "DEFINE_string" -> maybeToList $ getFlagVariable rest
 
+        "tc-export" -> concatMap getModifierParamString rest
+
+        -- tc-export_build_env exports the listed variables plus a bunch of BUILD_XX variables.
+        -- Usage tc-export_build_env BUILD_CC
+        "tc-export_build_env" ->
+            concatMap getModifierParamString rest
+                ++ [ (base, base, var, DataString $ SourceExternal) |
+                        var <- ["BUILD_" ++ x, x ++ "_FOR_BUILD" ],
+                        x <- portageBuildEnvVariables ]
+
         _ -> []
   where
     flags = map snd $ getAllFlags base
@@ -918,6 +961,13 @@ modifiesVariable params token name =
             Assignment (_, _, n, source) -> isTrueAssignmentSource source && n == name
             _ -> False
 
+-- Ebuild files inherit eclasses using 'inherit myclass1 myclass2'
+getInheritedEclasses :: Token -> [String]
+getInheritedEclasses root = execWriter $ doAnalysis findInheritedEclasses root
+  where
+    findInheritedEclasses cmd
+        | cmd `isCommand` "inherit" = tell $ catMaybes $ getLiteralString <$> (arguments cmd)
+    findInheritedEclasses _ = return ()
 
 return []
 runTests =  $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |])

src/ShellCheck/CFG.hs

@@ -167,7 +167,9 @@ data CFGParameters = CFGParameters {
     -- Whether the last element in a pipeline runs in the current shell
     cfLastpipe :: Bool,
     -- Whether all elements in a pipeline count towards the exit status
-    cfPipefail :: Bool
+    cfPipefail :: Bool,
+    -- Additional variables to consider defined
+    cfAdditionalInitialVariables :: [String]
 }
 
 data CFGResult = CFGResult {
@@ -192,7 +194,7 @@ buildGraph params root =
                     base
 
         idToRange = M.fromList mapping
-        isRealEdge (from, to, edge) = case edge of CFEFlow -> True; _ -> False
+        isRealEdge (from, to, edge) = case edge of CFEFlow -> True; CFEExit -> True; _ -> False
         onlyRealEdges = filter isRealEdge edges
         (_, mainExit) = fromJust $ M.lookup (getId root) idToRange
 
@@ -1301,7 +1303,10 @@ findPostDominators mainexit graph = asArray
     reversed = grev withExitEdges
     postDoms = dom reversed mainexit
     (_, maxNode) = nodeRange graph
-    asArray = array (0, maxNode) postDoms
+    -- Holes in the array cause "Exception: (Array.!): undefined array element" while
+    -- inspecting/debugging, so fill the array first and then update.
+    initializedArray = listArray (0, maxNode) $ repeat []
+    asArray = initializedArray // postDoms
 
 return []
 runTests =  $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |])

src/ShellCheck/CFGAnalysis.hs

@@ -59,6 +59,8 @@ module ShellCheck.CFGAnalysis (
     ,getIncomingState
     ,getOutgoingState
     ,doesPostDominate
+    ,variableMayBeDeclaredInteger
+    ,variableMayBeAssignedInteger
     ,ShellCheck.CFGAnalysis.runTests -- STRIP
     ) where
 
@@ -153,6 +155,20 @@ doesPostDominate analysis target base = fromMaybe False $ do
     (targetStart, _) <- M.lookup target $ tokenToRange analysis
     return $ targetStart `elem` (postDominators analysis ! baseEnd)
 
+-- See if any execution path results in the variable containing a state
+variableMayHaveState :: ProgramState -> String -> CFVariableProp -> Maybe Bool
+variableMayHaveState state var property = do
+    value <- M.lookup var $ variablesInScope state
+    return $ any (S.member property) $ variableProperties value
+
+-- See if any execution path declares the variable an integer (declare -i).
+variableMayBeDeclaredInteger state var = variableMayHaveState state var CFVPInteger
+
+-- See if any execution path suggests the variable may contain an integer value
+variableMayBeAssignedInteger state var = do
+    value <- M.lookup var $ variablesInScope state
+    return $ (numericalStatus $ variableValue value) >= NumericalStatusMaybe
+
 getDataForNode analysis node = M.lookup node $ nodeToData analysis
 
 -- The current state of data flow at a point in the program, potentially as a diff
@@ -181,12 +197,13 @@ unreachableState = modified newInternalState {
 }
 
 -- The default state we assume we get from the environment
-createEnvironmentState :: InternalState
+createEnvironmentState :: CFGParameters -> InternalState
-createEnvironmentState = do
+createEnvironmentState params = do
     foldl' (flip ($)) newInternalState $ concat [
         addVars Data.internalVariables unknownVariableState,
         addVars Data.variablesWithoutSpaces spacelessVariableState,
-        addVars Data.specialIntegerVariables integerVariableState
+        addVars Data.specialIntegerVariables integerVariableState,
+        addVars (cfAdditionalInitialVariables params) unknownVariableState
         ]
   where
     addVars names val = map (\name -> insertGlobal name val) names
@@ -1328,7 +1345,7 @@ analyzeControlFlow params t =
         runST $ f cfg entry exit
   where
     f cfg entry exit = do
-        let env = createEnvironmentState
+        let env = createEnvironmentState params
         ctx <- newCtx $ cfGraph cfg
         -- Do a dataflow analysis starting on the root node
         exitState <- runRoot ctx env entry exit

src/ShellCheck/Checker.hs

@@ -25,6 +25,7 @@ import ShellCheck.ASTLib
 import ShellCheck.Interface
 import ShellCheck.Parser
 
+import Data.Char
 import Data.Either
 import Data.Functor
 import Data.List
@@ -54,6 +55,8 @@ shellFromFilename filename = listToMaybe candidates
     shellExtensions = [(".ksh", Ksh)
                       ,(".bash", Bash)
                       ,(".bats", Bash)
+                      ,(".ebuild", Bash)
+                      ,(".eclass", Bash)
                       ,(".dash", Dash)]
                       -- The `.sh` is too generic to determine the shell:
                       -- We fallback to Bash in this case and emit SC2148 if there is no shebang
@@ -84,18 +87,24 @@ checkScript sys spec = do
                     asShellType = csShellTypeOverride spec,
                     asFallbackShell = shellFromFilename $ csFilename spec,
                     asCheckSourced = csCheckSourced spec,
+                    asIsPortage = isPortage $ csFilename spec,
                     asExecutionMode = Executed,
                     asTokenPositions = tokenPositions,
                     asOptionalChecks = getEnableDirectives root ++ csOptionalChecks spec
                 } where as = newAnalysisSpec root
-        let analysisMessages =
+        let getAnalysisMessages =
-                maybe []
+                case prRoot result of
-                    (arComments . analyzeScript . analysisSpec)
+                    Just root -> arComments <$> (analyzeScript sys $ analysisSpec root)
-                        $ prRoot result
+                    Nothing -> return []
         let translator = tokenToPosition tokenPositions
+        analysisMessages <- getAnalysisMessages
         return . nub . sortMessages . filter shouldInclude $
             (parseMessages ++ map translator analysisMessages)
 
+    isPortage filename =
+        let f = map toLower filename in
+            ".ebuild" `isSuffixOf` f || ".eclass" `isSuffixOf` f
+
     shouldInclude pc =
             severity <= csMinSeverity spec &&
             case csIncludedWarnings spec of
@@ -508,5 +517,14 @@ prop_rcCanSuppressEarlyProblems2 = null result
         csScript = "!/bin/bash\necho 'hello world'"
     }
 
+prop_sourceWithHereDocWorks = null result
+  where
+    result = checkWithIncludes [("bar", "true\n")] "source bar << eof\nlol\neof"
+
+prop_hereDocsAreParsedWithoutTrailingLinefeed = 1044 `elem` result
+  where
+    result = check "cat << eof"
+
+
 return []
 runTests = $quickCheckAll

src/ShellCheck/Checks/ShellSupport.hs

@@ -60,6 +60,8 @@ checks = [
     ,checkBraceExpansionVars
     ,checkMultiDimensionalArrays
     ,checkPS1Assignments
+    ,checkMultipleBangs
+    ,checkBangAfterPipe
     ]
 
 testChecker (ForShell _ t) =
@@ -184,6 +186,11 @@ prop_checkBashisms96 = verifyNot checkBashisms "#!/bin/dash\necho $_"
 prop_checkBashisms97 = verify checkBashisms "#!/bin/sh\necho ${var,}"
 prop_checkBashisms98 = verify checkBashisms "#!/bin/sh\necho ${var^^}"
 prop_checkBashisms99 = verify checkBashisms "#!/bin/dash\necho [^f]oo"
+prop_checkBashisms100 = verify checkBashisms "read -r"
+prop_checkBashisms101 = verify checkBashisms "read"
+prop_checkBashisms102 = verifyNot checkBashisms "read -r foo"
+prop_checkBashisms103 = verifyNot checkBashisms "read foo"
+prop_checkBashisms104 = verifyNot checkBashisms "read ''"
 checkBashisms = ForShell [Sh, Dash] $ \t -> do
     params <- ask
     kludge params t
@@ -379,6 +386,9 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do
                 let literal = onlyLiteralString format
                 guard $ "%q" `isInfixOf` literal
                 return $ warnMsg (getId format) 3050 "printf %q is"
+
+            when (name == "read" && all isFlag rest) $
+                warnMsg (getId cmd) 3061 "read without a variable is"
       where
         unsupportedCommands = [
             "let", "caller", "builtin", "complete", "compgen", "declare", "dirs", "disown",
@@ -558,5 +568,29 @@ checkPS1Assignments = ForShell [Bash] f
     escapeRegex = mkRegex "\\\\x1[Bb]|\\\\e|\x1B|\\\\033"
 
 
+prop_checkMultipleBangs1 = verify checkMultipleBangs "! ! true"
+prop_checkMultipleBangs2 = verifyNot checkMultipleBangs "! true"
+checkMultipleBangs = ForShell [Dash, Sh] f
+  where
+    f token = case token of
+        T_Banged id (T_Banged _ _) ->
+            err id 2325 "Multiple ! in front of pipelines are a bash/ksh extension. Use only 0 or 1."
+        _ -> return ()
+
+
+prop_checkBangAfterPipe1 = verify checkBangAfterPipe "true | ! true"
+prop_checkBangAfterPipe2 = verifyNot checkBangAfterPipe "true | ( ! true )"
+prop_checkBangAfterPipe3 = verifyNot checkBangAfterPipe "! ! true | true"
+checkBangAfterPipe = ForShell [Dash, Sh, Bash] f
+  where
+    f token = case token of
+        T_Pipeline _ _ cmds -> mapM_ check cmds
+        _ -> return ()
+
+    check token = case token of
+        T_Banged id _ ->
+            err id 2326 "! is not allowed in the middle of pipelines. Use command group as in cmd | { ! cmd; } if necessary."
+        _ -> return ()
+
 return []
 runTests =  $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |])

src/ShellCheck/Data.hs

@@ -62,8 +62,92 @@ internalVariables = [
     , "FLAGS_ARGC", "FLAGS_ARGV", "FLAGS_ERROR", "FLAGS_FALSE", "FLAGS_HELP",
     "FLAGS_PARENT", "FLAGS_RESERVED", "FLAGS_TRUE", "FLAGS_VERSION",
     "flags_error", "flags_return"
+  ] ++ portageManualInternalVariables
+
+
+portageManualInternalVariables = [
+    -- toolchain settings
+    "CFLAGS", "CXXFLAGS", "CPPFLAGS", "LDFLAGS", "FFLAGS", "FCFLAGS",
+    "CBUILD", "CHOST", "MAKEOPTS",
+    -- TODO: Delete these if we can handle `tc-export CC` implicit export.
+    "CC", "CPP", "CXX",
+
+    -- portage internals
+    "EBUILD_PHASE", "EBUILD_SH_ARGS", "EMERGE_FROM", "FILESDIR",
+    "MERGE_TYPE", "PM_EBUILD_HOOK_DIR", "PORTAGE_ACTUAL_DISTDIR",
+    "PORTAGE_ARCHLIST", "PORTAGE_BASHRC", "PORTAGE_BINPKG_FILE",
+    "PORTAGE_BINPKG_TAR_OPTS", "PORTAGE_BINPKG_TMPFILE", "PORTAGE_BIN_PATH",
+    "PORTAGE_BUILDDIR", "PORTAGE_BUILD_GROUP", "PORTAGE_BUILD_USER",
+    "PORTAGE_BUNZIP2_COMMAND", "PORTAGE_BZIP2_COMMAND", "PORTAGE_COLORMAP",
+    "PORTAGE_CONFIGROOT", "PORTAGE_DEBUG", "PORTAGE_DEPCACHEDIR",
+    "PORTAGE_EBUILD_EXIT_FILE", "PORTAGE_ECLASS_LOCATIONS", "PORTAGE_GID",
+    "PORTAGE_GRPNAME", "PORTAGE_INST_GID", "PORTAGE_INST_UID",
+    "PORTAGE_INTERNAL_CALLER", "PORTAGE_IPC_DAEMON", "PORTAGE_IUSE",
+    "PORTAGE_LOG_FILE", "PORTAGE_MUTABLE_FILTERED_VARS",
+    "PORTAGE_OVERRIDE_EPREFIX", "PORTAGE_PYM_PATH", "PORTAGE_PYTHON",
+    "PORTAGE_PYTHONPATH", "PORTAGE_READONLY_METADATA", "PORTAGE_READONLY_VARS",
+    "PORTAGE_REPO_NAME", "PORTAGE_REPOSITORIES", "PORTAGE_RESTRICT",
+    "PORTAGE_SAVED_READONLY_VARS", "PORTAGE_SIGPIPE_STATUS", "PORTAGE_TMPDIR",
+    "PORTAGE_UPDATE_ENV", "PORTAGE_USERNAME", "PORTAGE_VERBOSE",
+    "PORTAGE_WORKDIR_MODE", "PORTAGE_XATTR_EXCLUDE", "REPLACING_VERSIONS",
+    "REPLACED_BY_VERSION", "__PORTAGE_HELPER", "__PORTAGE_TEST_HARDLINK_LOCKS",
+
+    -- generic ebuilds
+    "A", "ARCH", "BDEPEND", "BOARD_USE", "BROOT", "CATEGORY", "D",
+    "DEFINED_PHASES", "DEPEND", "DESCRIPTION", "DISTDIR", "DOCS", "EAPI",
+    "ECLASS", "ED", "EPREFIX", "EROOT", "ESYSROOT", "EXTRA_ECONF",
+    "EXTRA_EINSTALL", "EXTRA_MAKE", "FEATURES", "FILESDIR", "HOME", "HOMEPAGE",
+    "HTML_DOCS", "INHERITED", "IUSE", "KEYWORDS", "LICENSE", "P", "PATCHES",
+    "PDEPEND", "PF", "PKG_INSTALL_MASK", "PKGUSE", "PN", "PR", "PROPERTIES",
+    "PROVIDES_EXCLUDE", "PV", "PVR", "QA_AM_MAINTAINER_MODE",
+    "QA_CONFIGURE_OPTIONS", "QA_DESKTOP_FILE", "QA_DT_NEEDED", "QA_EXECSTACK",
+    "QA_FLAGS_IGNORED", "QA_MULTILIB_PATHS", "QA_PREBUILT", "QA_PRESTRIPPED",
+    "QA_SONAME", "QA_SONAME_NO_SYMLINK", "QA_TEXTRELS", "QA_WX_LOAD", "RDEPEND",
+    "REPOSITORY", "REQUIRED_USE", "REQUIRES_EXCLUDE", "RESTRICT", "ROOT", "S",
+    "SLOT", "SRC_TEST", "SRC_URI", "STRIP_MASK", "SUBSLOT", "SYSROOT", "T",
+    "WORKDIR",
+
+    -- autotest.eclass declared incorrectly
+    "AUTOTEST_CLIENT_TESTS", "AUTOTEST_CLIENT_SITE_TESTS",
+    "AUTOTEST_SERVER_TESTS", "AUTOTEST_SERVER_SITE_TESTS", "AUTOTEST_CONFIG",
+    "AUTOTEST_DEPS", "AUTOTEST_PROFILERS", "AUTOTEST_CONFIG_LIST",
+    "AUTOTEST_DEPS_LIST", "AUTOTEST_PROFILERS_LIST",
+
+    -- cros-board.eclass declared incorrectly
+    "CROS_BOARDS",
+
+    -- Undeclared cros-kernel2 vars
+    "AFDO_PROFILE_VERSION",
+
+    -- haskell-cabal.eclass declared incorrectly
+    "CABAL_FEATURES",
+
+    -- Undeclared haskell-cabal.eclass vars
+    "CABAL_CORE_LIB_GHC_PV",
+
+    -- Undeclared readme.gentoo.eclass vars
+    "DOC_CONTENTS",
+
+    -- Backwards compatibility perl-module.eclass vars
+    "MODULE_AUTHOR", "MODULE_VERSION",
+
+    -- Undeclared perl-module.eclass vars
+    "mydoc",
+
+    -- python-utils-r1.eclass declared incorrectly
+    "RESTRICT_PYTHON_ABIS", "PYTHON_MODNAME",
+
+    -- ABI variables
+    "ABI", "DEFAULT_ABI",
+
+    -- AFDO variables
+    "AFDO_LOCATION",
+
+    -- Linguas
+    "LINGUAS"
   ]
 
+
 specialIntegerVariables = [
     "$", "?", "!", "#"
   ]
@@ -90,7 +174,9 @@ unbracedVariables = specialVariables ++ [
 arrayVariables = [
     "BASH_ALIASES", "BASH_ARGC", "BASH_ARGV", "BASH_CMDS", "BASH_LINENO",
     "BASH_REMATCH", "BASH_SOURCE", "BASH_VERSINFO", "COMP_WORDS", "COPROC",
-    "DIRSTACK", "FUNCNAME", "GROUPS", "MAPFILE", "PIPESTATUS", "COMPREPLY"
+    "DIRSTACK", "FUNCNAME", "GROUPS", "MAPFILE", "PIPESTATUS", "COMPREPLY",
+    -- For Portage
+    "PATCHES"
   ]
 
 commonCommands = [
@@ -150,6 +236,11 @@ unaryTestOps = [
     "-o", "-v", "-R"
   ]
 
+-- Variables inspected by Portage tc-export_build_env
+portageBuildEnvVariables = [
+    "CFLAGS", "CXXFLAGS", "CPPFLAGS", "LDFLAGS"
+  ]
+
 shellForExecutable :: String -> Maybe Shell
 shellForExecutable name =
     case name of

src/ShellCheck/Debug.hs

@@ -117,7 +117,8 @@ dummySystemInterface = mockedSystemInterface [
 cfgParams :: CFGParameters
 cfgParams = CFGParameters {
     cfLastpipe = False,
-    cfPipefail = False
+    cfPipefail = False,
+    cfAdditionalInitialVariables = []
 }
 
 -- An example script to play with

src/ShellCheck/Interface.hs

@@ -25,7 +25,7 @@ module ShellCheck.Interface
     , CheckResult(crFilename, crComments)
     , ParseSpec(psFilename, psScript, psCheckSourced, psIgnoreRC, psShellTypeOverride)
     , ParseResult(prComments, prTokenPositions, prRoot)
-    , AnalysisSpec(asScript, asShellType, asFallbackShell, asExecutionMode, asCheckSourced, asTokenPositions, asOptionalChecks)
+    , AnalysisSpec(..)
     , AnalysisResult(arComments)
     , FormatterOptions(foColorOption, foWikiLinkCount)
     , Shell(Ksh, Sh, Bash, Dash)
@@ -39,11 +39,12 @@ module ShellCheck.Interface
     , ColorOption(ColorAuto, ColorAlways, ColorNever)
     , TokenComment(tcId, tcComment, tcFix)
     , emptyCheckResult
-    , newParseResult
-    , newAnalysisSpec
     , newAnalysisResult
+    , newAnalysisSpec
     , newFormatterOptions
+    , newParseResult
     , newPosition
+    , newSystemInterface
     , newTokenComment
     , mockedSystemInterface
     , mockRcFile
@@ -86,7 +87,9 @@ data SystemInterface m = SystemInterface {
     --   find the sourced file
     siFindSource :: String -> Maybe Bool -> [String] -> String -> m FilePath,
     -- | Get the configuration file (name, contents) for a filename
-    siGetConfig :: String -> m (Maybe (FilePath, String))
+    siGetConfig :: String -> m (Maybe (FilePath, String)),
+    -- | Look up Portage Eclass variables
+    siGetPortageVariables :: m (Map.Map String [String])
 }
 
 -- ShellCheck input and output
@@ -135,6 +138,15 @@ newParseSpec = ParseSpec {
     psShellTypeOverride = Nothing
 }
 
+newSystemInterface :: Monad m => SystemInterface m
+newSystemInterface =
+    SystemInterface {
+        siReadFile = \_ _ -> return $ Left "Not implemented",
+        siFindSource = \_ _ _ name -> return name,
+        siGetConfig = \_ -> return Nothing,
+        siGetPortageVariables = return Map.empty
+    }
+
 -- Parser input and output
 data ParseSpec = ParseSpec {
     psFilename :: String,
@@ -164,6 +176,7 @@ data AnalysisSpec = AnalysisSpec {
     asFallbackShell :: Maybe Shell,
     asExecutionMode :: ExecutionMode,
     asCheckSourced :: Bool,
+    asIsPortage :: Bool,
     asOptionalChecks :: [String],
     asTokenPositions :: Map.Map Id (Position, Position)
 }
@@ -174,6 +187,7 @@ newAnalysisSpec token = AnalysisSpec {
     asFallbackShell = Nothing,
     asExecutionMode = Executed,
     asCheckSourced = False,
+    asIsPortage = False,
     asOptionalChecks = [],
     asTokenPositions = Map.empty
 }
@@ -311,7 +325,7 @@ data ColorOption =
 
 -- For testing
 mockedSystemInterface :: [(String, String)] -> SystemInterface Identity
-mockedSystemInterface files = SystemInterface {
+mockedSystemInterface files = (newSystemInterface :: SystemInterface Identity) {
     siReadFile = rf,
     siFindSource = fs,
     siGetConfig = const $ return Nothing

src/ShellCheck/Parser.hs

@@ -2283,22 +2283,31 @@ readSource t@(T_Redirecting _ _ (T_SimpleCommand cmdId _ (cmd:file':rest'))) = d
 
     subRead name script =
         withContext (ContextSource name) $
-            inSeparateContext $
+            inSeparateContext $ do
-                subParse (initialPos name) (readScriptFile True) script
+                oldState <- getState
+                setState $ oldState { pendingHereDocs = [] }
+                result <- subParse (initialPos name) (readScriptFile True) script
+                newState <- getState
+                setState $ newState { pendingHereDocs = pendingHereDocs oldState }
+                return result
 readSource t = return t
 
 
 prop_readPipeline = isOk readPipeline "! cat /etc/issue | grep -i ubuntu"
 prop_readPipeline2 = isWarning readPipeline "!cat /etc/issue | grep -i ubuntu"
 prop_readPipeline3 = isOk readPipeline "for f; do :; done|cat"
+prop_readPipeline4 = isOk readPipeline "! ! true"
+prop_readPipeline5 = isOk readPipeline "true | ! true"
 readPipeline = do
     unexpecting "keyword/token" readKeyword
-    do
+    readBanged readPipeSequence
+
+readBanged parser = do
+    pos <- getPosition
     (T_Bang id) <- g_Bang
-        pipe <- readPipeSequence
+    next <- readBanged parser
-        return $ T_Banged id pipe
+    return $ T_Banged id next
-      <|>
+ <|> parser
-        readPipeSequence
 
 prop_readAndOr = isOk readAndOr "grep -i lol foo || exit 1"
 prop_readAndOr1 = isOk readAndOr "# shellcheck disable=1\nfoo"
@@ -2354,7 +2363,7 @@ readTerm = do
 
 readPipeSequence = do
     start <- startSpan
-    (cmds, pipes) <- sepBy1WithSeparators readCommand
+    (cmds, pipes) <- sepBy1WithSeparators (readBanged readCommand)
                         (readPipe `thenSkip` (spacing >> readLineBreak))
     id <- endSpan start
     spacing
@@ -2384,6 +2393,10 @@ readCommand = choice [
     ]
 
 readCmdName = do
+    -- If the command name is `!` then
+    optional . lookAhead . try $ do
+        char '!'
+        whitespace
     -- Ignore alias suppression
     optional . try $ do
         char '\\'
@@ -3322,6 +3335,7 @@ readScriptFile sourced = do
               then do
                     commands <- readCompoundListOrEmpty
                     id <- endSpan start
+                    readPendingHereDocs
                     verifyEof
                     let script = T_Annotation annotationId annotations $
                                     T_Script id shebang commands
@@ -3360,6 +3374,7 @@ readScriptFile sourced = do
         "awk",
         "csh",
         "expect",
+        "fish",
         "perl",
         "python",
         "ruby",

src/ShellCheck/PortageVariables.hs

@@ -0,0 +1,138 @@
+{-# LANGUAGE ApplicativeDo #-}
+{-# LANGUAGE LambdaCase #-}
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE TupleSections #-}
+
+module ShellCheck.PortageVariables (
+    readPortageVariables
+  ) where
+
+import ShellCheck.Regex
+
+import Control.Exception
+import Control.Monad
+import Data.Maybe
+import System.Directory (listDirectory)
+import System.Exit (ExitCode(..))
+import System.FilePath
+import System.IO
+import System.Process
+
+import qualified Data.ByteString as B
+import qualified Data.Map as M
+
+type RepoName = String
+type RepoPath = String
+type EclassName = String
+type EclassVar = String
+
+-- | This is used for looking up what eclass variables are inherited,
+--   keyed by the name of the eclass.
+type EclassMap = M.Map EclassName [EclassVar]
+
+data Repository = Repository
+    { repositoryName :: RepoName
+    , repositoryLocation :: RepoPath
+    , repositoryEclasses :: [Eclass]
+    } deriving (Show, Eq, Ord)
+
+data Eclass = Eclass
+    { eclassName :: EclassName
+    , eclassVars :: [EclassVar]
+    } deriving (Show, Eq, Ord)
+
+readPortageVariables :: IO (M.Map String [String])
+readPortageVariables = portageVariables <$> scanRepos
+
+-- | Map from eclass names to a list of eclass variables
+portageVariables :: [Repository] -> EclassMap
+portageVariables = foldMap $ foldMap go . repositoryEclasses
+  where
+    go e = M.singleton (eclassName e) (eclassVars e)
+
+-- | Run @portageq@ to gather a list of repo names and paths, then scan each
+--   one for eclasses and ultimately eclass metadata.
+scanRepos :: IO [Repository]
+scanRepos = do
+    let cmd = "portageq"
+    let args = ["repos_config", "/"]
+    out <- runOrDie cmd args
+    forM (reposParser $ lines out) $ \(n,p) -> Repository n p <$> getEclasses p
+
+-- | Get the name of the repo and its path from blocks outputted by
+--   @portageq@. If the path doesn't exist, this will return @Nothing@.
+reposParser :: [String] -> [(RepoName, RepoPath)]
+reposParser = f ""
+  where
+    segmentRegex = mkRegex "^\\[(.*)\\].*"
+    locationRegex = mkRegex "^[[:space:]]*location[[:space:]]*=[[:space:]]*(.*)[[:space:]]*$"
+    f name [] = []
+    f name (line:rest) =
+        case (matchRegex segmentRegex line, matchRegex locationRegex line) of
+            (Just [next], _) -> f next rest
+            (_, Just [location]) -> (name, location) : f name rest
+            _ -> f name rest
+
+-- | Scan the repo path for @*.eclass@ files in @eclass/@, then run
+--   'eclassParser' on each of them to produce @[Eclass]@.
+--
+--   If the @eclass/@ directory doesn't exist, the scan is skipped for that
+--   repo.
+getEclasses :: RepoPath -> IO [Eclass]
+getEclasses repoLoc = do
+    let eclassDir = repoLoc </> "eclass"
+
+    files <- handle catcher $ listDirectory eclassDir
+    let names = filter (\(_, e) -> e == ".eclass") $ map splitExtension files
+
+    forM (names :: [(String, String)]) $ \(name, ext) -> do
+        contents <- withFile (eclassDir </> name <.> ext) ReadMode readFully
+        return $ Eclass name $ eclassParser (lines contents)
+
+  where
+    catcher :: IOException -> IO [String]
+    catcher e = do
+        hPutStrLn stderr $ "Unable to find .eclass files: " ++ show e
+        return []
+
+-- | Scan a @.eclass@ file for any @@@ECLASS_VARIABLE:@ comments, generating
+--   a list of eclass variables.
+eclassParser :: [String] -> [String]
+eclassParser lines = mapMaybe match lines
+  where
+    varRegex = mkRegex "^[[:space:]]*#[[:space:]]*@ECLASS_VARIABLE:[[:space:]]*([^[:space:]]*)[[:space:]]*$"
+    match str = head <$> matchRegex varRegex str
+
+-- | Run the command and return the full stdout string (stdin is ignored).
+--
+--   If the command exits with a non-zero exit code, this will throw an
+--   error including the captured contents of stdout and stderr.
+runOrDie :: FilePath -> [String] -> IO String
+runOrDie cmd args = bracket acquire release $ \(_,o,e,p) -> do
+    ot <- readFully (fromJust o)
+    et <- readFully (fromJust e)
+    ec <- waitForProcess p
+    case ec of
+        ExitSuccess -> pure ot
+        ExitFailure i -> fail $ unlines $ map unwords
+            $   [ [ show cmd ]
+                    ++ map show args
+                    ++ [ "failed with exit code", show i]
+                , [ "stdout:" ], [ ot ]
+                , [ "stderr:" ], [ et ]
+                ]
+  where
+    acquire = createProcess (proc cmd args)
+        { std_in = NoStream
+        , std_out = CreatePipe
+        , std_err = CreatePipe
+        }
+    release (i,o,e,p) = do
+        _ <- waitForProcess p
+        forM_ [i,o,e] $ mapM_ hClose
+
+readFully :: Handle -> IO String
+readFully handle = do
+    hSetBinaryMode handle True
+    str <- hGetContents handle
+    length str `seq` return str