Stable version 0.4.4

This release is dedicated to AlphaGo,
embodying the second golden age of AI.

.gitignore

@@ -12,4 +12,4 @@ cabal-dev
 .cabal-sandbox/
 cabal.sandbox.config
 cabal.config
-
+.stack-work

README.md

@@ -7,14 +7,14 @@ ShellCheck is a GPLv3 tool that gives warnings and suggestions for bash/sh shell
 The goals of ShellCheck are
 
   - To point out and clarify typical beginner's syntax issues
-    that causes a shell to give cryptic error messages.
+    that cause a shell to give cryptic error messages.
 
   - To point out and clarify typical intermediate level semantic problems
-    that causes a shell to behave strangely and counter-intuitively.
+    that cause a shell to behave strangely and counter-intuitively.
 
   - To point out subtle caveats, corner cases and pitfalls that may cause an
     advanced user's otherwise working script to fail under future circumstances.
-    
+
 See [the gallery of bad code](README.md#user-content-gallery-of-bad-code) for examples of what ShellCheck can help you identify!
 
 
@@ -48,13 +48,13 @@ You can see ShellCheck suggestions directly in a variety of editors.
 
 * Atom, through [Linter](https://github.com/AtomLinter/linter-shellcheck).
 
-* Most other editors, through [GCC error compatibility](blob/master/shellcheck.1.md#user-content-formats).
+* Most other editors, through [GCC error compatibility](shellcheck.1.md#user-content-formats).
 
 
 #### In your build or test suites
 While ShellCheck is mostly intended for interactive use, it can easily be added to builds or test suites.
 
-Use ShellCheck's exit code, or it's [CheckStyle compatible XML output](blob/master/shellcheck.1.md#user-content-formats). There's also a simple JSON output format for easy integration.
+Use ShellCheck's exit code, or its [CheckStyle compatible XML output](shellcheck.1.md#user-content-formats). There's also a simple JSON output format for easy integration.
 
 
 ## Installing
@@ -98,7 +98,7 @@ or use OneClickInstall - https://software.opensuse.org/package/ShellCheck
 
 ## Compiling from source
 
-This sections describes how to build ShellCheck from a source directory. ShellCheck is written in Haskell and requires 2GB of RAM to compile.
+This section describes how to build ShellCheck from a source directory. ShellCheck is written in Haskell and requires 2GB of RAM to compile.
 
 
 #### Installing Cabal
@@ -174,7 +174,7 @@ ShellCheck can recognize many types of incorrect test statements.
     [[ -e *.mpg ]]                    # Existence checks of globs
     [[ $foo==0 ]]                     # Always true due to missing spaces
     [[ -n "$foo " ]]                  # Always true due to literals
-    [[ $foo =~ "fo+" ]]               # Quoted regex in =~ 
+    [[ $foo =~ "fo+" ]]               # Quoted regex in =~
     [ foo =~ re ]                     # Unsupported [ ] operators
     [ $1 -eq "shellcheck" ]           # Numerical comparison of strings
     [ $n && $m ]                      # && in [ .. ]
@@ -219,7 +219,7 @@ ShellCheck can make suggestions to improve style:
     [[ -z $(find /tmp | grep mpg) ]]  # Use grep -q instead
     a >> log; b >> log; c >> log      # Use a redirection block instead
     echo "The time is `date`"         # Use $() instead
-    cd dir; process *; cd ..;         # Use subshells instead 
+    cd dir; process *; cd ..;         # Use subshells instead
     echo $[1+2]                       # Use standard $((..)) instead of old $[]
     echo $(($RANDOM % 6))             # Don't use $ on variables in $((..))
     echo "$(date)"                    # Useless use of echo
@@ -265,7 +265,7 @@ ShellCheck will warn when using features not supported by the shebang. For examp
     foo-bar() { ..; }                # Undefined/unsupported function name
     [ $UID = 0 ]                     # Variable undefined in dash/sh
     local var=value                  # local is undefined in sh
-    
+
 
 #### Miscellaneous
 
@@ -279,7 +279,7 @@ ShellCheck recognizes a menagerie of other issues:
     #!/bin/bash -x -e                 # Common shebang errors
     echo $((n/180*100))               # Unnecessary loss of precision
     ls *[:digit:].txt                 # Bad character class globs
-    sed 's/foo/bar/ file > file       # Redirecting to input
+    sed 's/foo/bar/' file > file       # Redirecting to input
 
 
 ## Testimonials
@@ -292,14 +292,14 @@ Alexander Tarasikov,
 
 ## Reporting bugs
 
-Please use the Github issue tracker for any bugs or feature suggestions:
+Please use the GitHub issue tracker for any bugs or feature suggestions:
 
 https://github.com/koalaman/shellcheck/issues
 
 
 ## Contributing
 
-Please submit patches to code or documentation as Github pull requests!
+Please submit patches to code or documentation as GitHub pull requests!
 
 Contributions must be licensed under the GNU GPLv3.
 The contributor retains the copyright.

ShellCheck.cabal

@@ -1,5 +1,5 @@
 Name:             ShellCheck
-Version:          0.4.3
+Version:          0.4.4
 Synopsis:         Shell script analysis tool
 License:          GPL-3
 License-file:     LICENSE
@@ -52,7 +52,9 @@ library
       ShellCheck.ASTLib
       ShellCheck.Analytics
       ShellCheck.Analyzer
+      ShellCheck.AnalyzerLib
       ShellCheck.Checker
+      ShellCheck.Checks.Commands
       ShellCheck.Data
       ShellCheck.Formatter.Format
       ShellCheck.Formatter.CheckStyle

ShellCheck/AST.hs

@@ -34,6 +34,7 @@ data CaseType = CaseBreak | CaseFallThrough | CaseContinue deriving (Show, Eq)
 
 data Token =
     TA_Binary Id String Token Token
+    | TA_Assignment Id String Token Token
     | TA_Expansion Id [Token]
     | TA_Index Id Token
     | TA_Sequence Id [Token]
@@ -129,7 +130,11 @@ data Token =
     | T_Include Id Token Token -- . & source: SimpleCommand T_Script
     deriving (Show)
 
-data Annotation = DisableComment Integer | SourceOverride String deriving (Show, Eq)
+data Annotation =
+    DisableComment Integer
+    | SourceOverride String
+    | ShellOverride String
+    deriving (Show, Eq)
 data ConditionType = DoubleBracket | SingleBracket deriving (Show, Eq)
 
 -- This is an abomination.
@@ -162,7 +167,7 @@ analyze f g i =
     dll l m v = do
         x <- roundAll l
         y <- roundAll m
-        return $ v x m
+        return $ v x y
     d1 t v = do
         x <- round t
         return $ v x
@@ -246,6 +251,7 @@ analyze f g i =
     delve (TC_Noary id typ token) = d1 token $ TC_Noary id typ
 
     delve (TA_Binary id op t1 t2) = d2 t1 t2 $ TA_Binary id op
+    delve (TA_Assignment id op t1 t2) = d2 t1 t2 $ TA_Assignment id op
     delve (TA_Unary id op t1) = d1 t1 $ TA_Unary id op
     delve (TA_Sequence id l) = dl l $ TA_Sequence id
     delve (TA_Trinary id t1 t2 t3) = do
@@ -340,6 +346,7 @@ getId t = case t of
         TC_Unary id _ _ _  -> id
         TC_Noary id _ _  -> id
         TA_Binary id _ _ _  -> id
+        TA_Assignment id _ _ _  -> id
         TA_Unary id _ _  -> id
         TA_Sequence id _  -> id
         TA_Trinary id _ _ _  -> id

ShellCheck/Analyzer.hs

@@ -19,9 +19,18 @@
 -}
 module ShellCheck.Analyzer (analyzeScript) where
 
-import ShellCheck.Interface
 import ShellCheck.Analytics
+import ShellCheck.AnalyzerLib
+import ShellCheck.Interface
+import Data.List
+import qualified ShellCheck.Checks.Commands
+
 
 -- TODO: Clean up the cruft this is layered on
 analyzeScript :: AnalysisSpec -> AnalysisResult
-analyzeScript = runAnalytics
+analyzeScript spec = AnalysisResult {
+    arComments =
+        filterByAnnotation (asScript spec) . nub $
+            runAnalytics spec
+            ++ ShellCheck.Checks.Commands.runChecks spec
+}

ShellCheck/AnalyzerLib.hs

@@ -0,0 +1,632 @@
+{-
+    Copyright 2012-2015 Vidar Holen
+
+    This file is part of ShellCheck.
+    http://www.vidarholen.net/contents/shellcheck
+
+    ShellCheck is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    ShellCheck is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+-}
+{-# LANGUAGE TemplateHaskell #-}
+{-# LANGUAGE FlexibleContexts #-}
+module ShellCheck.AnalyzerLib where
+import ShellCheck.AST
+import ShellCheck.ASTLib
+import ShellCheck.Data
+import ShellCheck.Interface
+import ShellCheck.Parser
+import ShellCheck.Regex
+
+import Control.Arrow (first)
+import Control.Monad.Identity
+import Control.Monad.Reader
+import Control.Monad.State
+import Control.Monad.Writer
+import Data.Char
+import Data.List
+import Data.Maybe
+import qualified Data.Map as Map
+
+import Test.QuickCheck.All (forAllProperties)
+import Test.QuickCheck.Test (quickCheckWithResult, stdArgs, maxSuccess)
+
+type Analysis = ReaderT Parameters (Writer [TokenComment]) ()
+
+
+data Parameters = Parameters {
+    variableFlow :: [StackData],
+    parentMap :: Map.Map Id Token,
+    shellType :: Shell,
+    shellTypeSpecified :: Bool
+    }
+
+data Scope = SubshellScope String | NoneScope deriving (Show, Eq)
+data StackData =
+    StackScope Scope
+    | StackScopeEnd
+    -- (Base expression, specific position, var name, assigned values)
+    | Assignment (Token, Token, String, DataType)
+    | Reference (Token, Token, String)
+  deriving (Show)
+
+data DataType = DataString DataSource | DataArray DataSource
+  deriving (Show)
+
+data DataSource = SourceFrom [Token] | SourceExternal | SourceDeclaration | SourceInteger
+  deriving (Show)
+
+data VariableState = Dead Token String | Alive deriving (Show)
+
+defaultSpec root = AnalysisSpec {
+    asScript = root,
+    asShellType = Nothing,
+    asExecutionMode = Executed
+}
+
+pScript s =
+  let
+    pSpec = ParseSpec {
+        psFilename = "script",
+        psScript = s
+    }
+  in prRoot . runIdentity $ parseScript (mockedSystemInterface []) pSpec
+
+makeComment :: Severity -> Id -> Code -> String -> TokenComment
+makeComment severity id code note =
+    TokenComment id $ Comment severity code note
+
+addComment note = tell [note]
+
+warn :: MonadWriter [TokenComment] m => Id -> Code -> String -> m ()
+warn  id code str = addComment $ makeComment WarningC id code str
+err   id code str = addComment $ makeComment ErrorC id code str
+info  id code str = addComment $ makeComment InfoC id code str
+style id code str = addComment $ makeComment StyleC id code str
+
+makeParameters spec =
+    let params = Parameters {
+        shellType = fromMaybe (determineShell root) $ asShellType spec,
+        shellTypeSpecified = isJust $ asShellType spec,
+        parentMap = getParentTree root,
+        variableFlow =
+            getVariableFlow (shellType params) (parentMap params) root
+    } in params
+  where root = asScript spec
+
+prop_determineShell0 = determineShell (fromJust $ pScript "#!/bin/sh") == Sh
+prop_determineShell1 = determineShell (fromJust $ pScript "#!/usr/bin/env ksh") == Ksh
+prop_determineShell2 = determineShell (fromJust $ pScript "") == Bash
+prop_determineShell3 = determineShell (fromJust $ pScript "#!/bin/sh -e") == Sh
+prop_determineShell4 = determineShell (fromJust $ pScript
+    "#!/bin/ksh\n#shellcheck shell=sh\nfoo") == Sh
+prop_determineShell5 = determineShell (fromJust $ pScript
+    "#shellcheck shell=sh\nfoo") == Sh
+prop_determineShell6 = determineShell (fromJust $ pScript "#! /bin/sh") == Sh
+determineShell t = fromMaybe Bash $ do
+    shellString <- foldl mplus Nothing $ getCandidates t
+    shellForExecutable shellString
+  where
+    forAnnotation t =
+        case t of
+            (ShellOverride s) -> return s
+            _ -> fail ""
+    getCandidates :: Token -> [Maybe String]
+    getCandidates t@(T_Script {}) = [Just $ fromShebang t]
+    getCandidates (T_Annotation _ annotations s) =
+        map forAnnotation annotations ++
+           [Just $ fromShebang s]
+    fromShebang (T_Script _ s t) = shellFor s
+
+    shellFor s | "/env " `isInfixOf` s = head (drop 1 (words s)++[""])
+    shellFor s | ' ' `elem` s = shellFor $ takeWhile (/= ' ') s
+    shellFor s = reverse . takeWhile (/= '/') . reverse $ s
+
+
+--- Context seeking
+
+getParentTree t =
+    snd . snd $ runState (doStackAnalysis pre post t) ([], Map.empty)
+  where
+    pre t = modify (first ((:) t))
+    post t = do
+        (_:rest, map) <- get
+        case rest of [] -> put (rest, map)
+                     (x:_) -> put (rest, Map.insert (getId t) x map)
+
+getTokenMap t =
+    execState (doAnalysis f t) Map.empty
+  where
+    f t = modify (Map.insert (getId t) t)
+
+
+-- Is this node self quoting for a regular element?
+isQuoteFree = isQuoteFreeNode False
+
+-- Is this node striclty self quoting, for array expansions
+isStrictlyQuoteFree = isQuoteFreeNode True
+
+
+isQuoteFreeNode strict tree t =
+    (isQuoteFreeElement t == Just True) ||
+        head (mapMaybe isQuoteFreeContext (drop 1 $ getPath tree t) ++ [False])
+  where
+    -- Is this node self-quoting in itself?
+    isQuoteFreeElement t =
+        case t of
+            T_Assignment {} -> return True
+            T_FdRedirect {} -> return True
+            _ -> Nothing
+
+    -- Are any subnodes inherently self-quoting?
+    isQuoteFreeContext t =
+        case t of
+            TC_Noary _ DoubleBracket _ -> return True
+            TC_Unary _ DoubleBracket _ _ -> return True
+            TC_Binary _ DoubleBracket _ _ _ -> return True
+            TA_Sequence {} -> return True
+            T_Arithmetic {} -> return True
+            T_Assignment {} -> return True
+            T_Redirecting {} -> return $
+                if strict then False else
+                    -- Not true, just a hack to prevent warning about non-expansion refs
+                    any (isCommand t) ["local", "declare", "typeset", "export", "trap", "readonly"]
+            T_DoubleQuoted _ _ -> return True
+            T_DollarDoubleQuoted _ _ -> return True
+            T_CaseExpression {} -> return True
+            T_HereDoc {} -> return True
+            T_DollarBraced {} -> return True
+            -- When non-strict, pragmatically assume it's desirable to split here
+            T_ForIn {} -> return (not strict)
+            T_SelectIn {} -> return (not strict)
+            _ -> Nothing
+
+isParamTo tree cmd =
+    go
+  where
+    go x = case Map.lookup (getId x) tree of
+                Nothing -> False
+                Just parent -> check parent
+    check t =
+        case t of
+            T_SingleQuoted _ _ -> go t
+            T_DoubleQuoted _ _ -> go t
+            T_NormalWord _ _ -> go t
+            T_SimpleCommand {} -> isCommand t cmd
+            T_Redirecting {} -> isCommand t cmd
+            _ -> False
+
+getClosestCommand tree t =
+    msum . map getCommand $ getPath tree t
+  where
+    getCommand t@(T_Redirecting {}) = return t
+    getCommand _ = Nothing
+
+usedAsCommandName tree token = go (getId token) (tail $ getPath tree token)
+  where
+    go currentId (T_NormalWord id [word]:rest)
+        | currentId == getId word = go id rest
+    go currentId (T_DoubleQuoted id [word]:rest)
+        | currentId == getId word = go id rest
+    go currentId (T_SimpleCommand _ _ (word:_):_)
+        | currentId == getId word = True
+    go _ _ = False
+
+-- A list of the element and all its parents
+getPath tree t = t :
+    case Map.lookup (getId t) tree of
+        Nothing -> []
+        Just parent -> getPath tree parent
+
+isParentOf tree parent child =
+    elem (getId parent) . map getId $ getPath tree child
+
+parents params = getPath (parentMap params)
+
+pathTo t = do
+    parents <- reader parentMap
+    return $ getPath parents t
+
+-- Check whether a word is entirely output from a single command
+tokenIsJustCommandOutput t = case t of
+    T_NormalWord id [T_DollarExpansion _ cmds] -> check cmds
+    T_NormalWord id [T_DoubleQuoted _ [T_DollarExpansion _ cmds]] -> check cmds
+    T_NormalWord id [T_Backticked _ cmds] -> check cmds
+    T_NormalWord id [T_DoubleQuoted _ [T_Backticked _ cmds]] -> check cmds
+    _ -> False
+  where
+    check [x] = not $ isOnlyRedirection x
+    check _ = False
+
+-- TODO: Replace this with a proper Control Flow Graph
+getVariableFlow shell parents t =
+    let (_, stack) = runState (doStackAnalysis startScope endScope t) []
+    in reverse stack
+  where
+    startScope t =
+        let scopeType = leadType shell parents t
+        in do
+            when (scopeType /= NoneScope) $ modify (StackScope scopeType:)
+            when (assignFirst t) $ setWritten t
+
+    endScope t =
+        let scopeType = leadType shell parents t
+        in do
+            setRead t
+            unless (assignFirst t) $ setWritten t
+            when (scopeType /= NoneScope) $ modify (StackScopeEnd:)
+
+    assignFirst (T_ForIn {}) = True
+    assignFirst (T_SelectIn {}) = True
+    assignFirst _ = False
+
+    setRead t =
+        let read    = getReferencedVariables parents t
+        in mapM_ (\v -> modify (Reference v:)) read
+
+    setWritten t =
+        let written = getModifiedVariables t
+        in mapM_ (\v -> modify (Assignment v:)) written
+
+
+leadType shell parents t =
+    case t of
+        T_DollarExpansion _ _  -> SubshellScope "$(..) expansion"
+        T_Backticked _ _  -> SubshellScope "`..` expansion"
+        T_Backgrounded _ _  -> SubshellScope "backgrounding &"
+        T_Subshell _ _  -> SubshellScope "(..) group"
+        T_CoProcBody _ _  -> SubshellScope "coproc"
+        T_Redirecting {}  ->
+            if fromMaybe False causesSubshell
+            then SubshellScope "pipeline"
+            else NoneScope
+        _ -> NoneScope
+  where
+    parentPipeline = do
+        parent <- Map.lookup (getId t) parents
+        case parent of
+            T_Pipeline {} -> return parent
+            _ -> Nothing
+
+    causesSubshell = do
+        (T_Pipeline _ _ list) <- parentPipeline
+        if length list <= 1
+            then return False
+            else if lastCreatesSubshell
+                then return True
+                else return . not $ (getId . head $ reverse list) == getId t
+
+    lastCreatesSubshell =
+        case shell of
+            Bash -> True
+            Dash -> True
+            Sh -> True
+            Ksh -> False
+
+getModifiedVariables t =
+    case t of
+        T_SimpleCommand _ vars [] ->
+            concatMap (\x -> case x of
+                                T_Assignment id _ name _ w  ->
+                                    [(x, x, name, dataTypeFrom DataString w)]
+                                _ -> []
+                      ) vars
+        c@(T_SimpleCommand {}) ->
+            getModifiedVariableCommand c
+
+        TA_Unary _ "++|" var -> maybeToList $ do
+            name <- getLiteralString var
+            return (t, t, name, DataString $ SourceFrom [t])
+        TA_Unary _ "|++" var -> maybeToList $ do
+            name <- getLiteralString var
+            return (t, t, name, DataString $ SourceFrom [t])
+        TA_Assignment _ op lhs rhs -> maybeToList $ do
+            guard $ op `elem` ["=", "*=", "/=", "%=", "+=", "-=", "<<=", ">>=", "&=", "^=", "|="]
+            name <- getLiteralString lhs
+            return (t, t, name, DataString $ SourceFrom [rhs])
+
+        t@(T_FdRedirect _ ('{':var) op) -> -- {foo}>&2 modifies foo
+            [(t, t, takeWhile (/= '}') var, DataString SourceInteger) | not $ isClosingFileOp op]
+
+        t@(T_CoProc _ name _) ->
+            [(t, t, fromMaybe "COPROC" name, DataArray SourceInteger)]
+
+        --Points to 'for' rather than variable
+        T_ForIn id str words _ -> [(t, t, str, DataString $ SourceFrom words)]
+        T_SelectIn id str words _ -> [(t, t, str, DataString $ SourceFrom words)]
+        _ -> []
+
+isClosingFileOp op =
+    case op of
+        T_IoFile _ (T_GREATAND _) (T_NormalWord _ [T_Literal _ "-"]) -> True
+        T_IoFile _ (T_LESSAND  _) (T_NormalWord _ [T_Literal _ "-"]) -> True
+        _ -> False
+
+
+-- Consider 'export/declare -x' a reference, since it makes the var available
+getReferencedVariableCommand base@(T_SimpleCommand _ _ (T_NormalWord _ (T_Literal _ x:_):rest)) =
+    case x of
+        "export" -> if "f" `elem` flags
+            then []
+            else concatMap getReference rest
+        "declare" -> if any (`elem` flags) ["x", "p"]
+            then concatMap getReference rest
+            else []
+        "readonly" -> concatMap getReference rest
+        "trap" ->
+            case rest of
+                head:_ -> map (\x -> (head, head, x)) $ getVariablesFromLiteralToken head
+                _ -> []
+        _ -> []
+  where
+    getReference t@(T_Assignment _ _ name _ value) = [(t, t, name)]
+    getReference t@(T_NormalWord _ [T_Literal _ name]) | not ("-" `isPrefixOf` name) = [(t, t, name)]
+    getReference _ = []
+    flags = map snd $ getAllFlags base
+
+getReferencedVariableCommand _ = []
+
+getModifiedVariableCommand base@(T_SimpleCommand _ _ (T_NormalWord _ (T_Literal _ x:_):rest)) =
+   filter (\(_,_,s,_) -> not ("-" `isPrefixOf` s)) $
+    case x of
+        "read" ->
+            let params = map getLiteral rest in
+                catMaybes . takeWhile isJust . reverse $ params
+        "getopts" ->
+            case rest of
+                opts:var:_ -> maybeToList $ getLiteral var
+                _ -> []
+
+        "let" -> concatMap letParamToLiteral rest
+
+        "export" ->
+            if "f" `elem` flags then [] else concatMap getModifierParamString rest
+
+        "declare" -> if any (`elem` flags) ["F", "f", "p"] then [] else declaredVars
+        "typeset" -> declaredVars
+
+        "local" -> concatMap getModifierParamString rest
+        "readonly" -> concatMap getModifierParamString rest
+        "set" -> maybeToList $ do
+            params <- getSetParams rest
+            return (base, base, "@", DataString $ SourceFrom params)
+
+        "printf" -> maybeToList $ getPrintfVariable rest
+
+        "mapfile" -> maybeToList $ getMapfileArray base rest
+        "readarray" -> maybeToList $ getMapfileArray base rest
+
+        _ -> []
+  where
+    flags = map snd $ getAllFlags base
+    stripEquals s = let rest = dropWhile (/= '=') s in
+        if rest == "" then "" else tail rest
+    stripEqualsFrom (T_NormalWord id1 (T_Literal id2 s:rs)) =
+        T_NormalWord id1 (T_Literal id2 (stripEquals s):rs)
+    stripEqualsFrom (T_NormalWord id1 [T_DoubleQuoted id2 [T_Literal id3 s]]) =
+        T_NormalWord id1 [T_DoubleQuoted id2 [T_Literal id3 (stripEquals s)]]
+    stripEqualsFrom t = t
+
+    declaredVars = concatMap (getModifierParam defaultType) rest
+      where
+        defaultType = if any (`elem` flags) ["a", "A"] then DataArray else DataString
+
+    getLiteral t = do
+        s <- getLiteralString t
+        when ("-" `isPrefixOf` s) $ fail "argument"
+        return (base, t, s, DataString SourceExternal)
+
+    getModifierParamString = getModifierParam DataString
+
+    getModifierParam def t@(T_Assignment _ _ name _ value) =
+        [(base, t, name, dataTypeFrom def value)]
+    getModifierParam def t@(T_NormalWord {}) = maybeToList $ do
+        name <- getLiteralString t
+        guard $ isVariableName name
+        return (base, t, name, def SourceDeclaration)
+    getModifierParam _ _ = []
+
+    letParamToLiteral token =
+          if var == ""
+            then []
+            else [(base, token, var, DataString $ SourceFrom [stripEqualsFrom token])]
+        where var = takeWhile isVariableChar $ dropWhile (`elem` "+-") $ concat $ oversimplify token
+
+    getSetParams (t:_:rest) | getLiteralString t == Just "-o" = getSetParams rest
+    getSetParams (t:rest) =
+        let s = getLiteralString t in
+            case s of
+                Just "--" -> return rest
+                Just ('-':_) -> getSetParams rest
+                _ -> return (t:fromMaybe [] (getSetParams rest))
+    getSetParams [] = Nothing
+
+    getPrintfVariable list = f $ map (\x -> (x, getLiteralString x)) list
+      where
+        f ((_, Just "-v") : (t, Just var) : _) = return (base, t, var, DataString $ SourceFrom list)
+        f (_:rest) = f rest
+        f [] = fail "not found"
+
+    -- mapfile has some curious syntax allowing flags plus 0..n variable names
+    -- where only the first non-option one is used if any. Here we cheat and
+    -- just get the last one, if it's a variable name.
+    getMapfileArray base arguments = do
+        lastArg <- listToMaybe (reverse arguments)
+        name <- getLiteralString lastArg
+        guard $ isVariableName name
+        return (base, lastArg, name, DataArray SourceExternal)
+
+getModifiedVariableCommand _ = []
+
+getIndexReferences s = fromMaybe [] $ do
+    match <- matchRegex re s
+    index <- match !!! 0
+    return $ matchAllStrings variableNameRegex index
+  where
+    re = mkRegex "(\\[.*\\])"
+
+getReferencedVariables parents t =
+    case t of
+        T_DollarBraced id l -> let str = bracedString t in
+            (t, t, getBracedReference str) :
+                map (\x -> (l, l, x)) (getIndexReferences str)
+        TA_Expansion id _ ->
+            if isArithmeticAssignment t
+            then []
+            else getIfReference t t
+        T_Assignment id mode str _ word ->
+            [(t, t, str) | mode == Append] ++ specialReferences str t word
+
+        TC_Unary id _ "-v" token -> getIfReference t token
+        TC_Unary id _ "-R" token -> getIfReference t token
+        TC_Binary id DoubleBracket op lhs rhs ->
+            if isDereferencing op
+            then concatMap (getIfReference t) [lhs, rhs]
+            else []
+
+        t@(T_FdRedirect _ ('{':var) op) -> -- {foo}>&- references and closes foo
+            [(t, t, takeWhile (/= '}') var) | isClosingFileOp op]
+        x -> getReferencedVariableCommand x
+  where
+    -- Try to reduce false positives for unused vars only referenced from evaluated vars
+    specialReferences name base word =
+        if name `elem` [
+            "PS1", "PS2", "PS3", "PS4",
+            "PROMPT_COMMAND"
+          ]
+        then
+            map (\x -> (base, base, x)) $
+                getVariablesFromLiteralToken word
+        else []
+
+    literalizer (TA_Index {}) = return ""  -- x[0] becomes a reference of x
+    literalizer _ = Nothing
+
+    getIfReference context token = maybeToList $ do
+            str <- getLiteralStringExt literalizer token
+            guard . not $ null str
+            when (isDigit $ head str) $ fail "is a number"
+            return (context, token, getBracedReference str)
+
+    isDereferencing = (`elem` ["-eq", "-ne", "-lt", "-le", "-gt", "-ge"])
+
+    isArithmeticAssignment t = case getPath parents t of
+        this: TA_Assignment _ "=" _ _ :_ -> True
+        _ -> False
+
+dataTypeFrom defaultType v = (case v of T_Array {} -> DataArray; _ -> defaultType) $ SourceFrom [v]
+
+
+--- Command specific checks
+
+isCommand token str = isCommandMatch token (\cmd -> cmd  == str || ('/' : str) `isSuffixOf` cmd)
+isUnqualifiedCommand token str = isCommandMatch token (== str)
+
+isCommandMatch token matcher = fromMaybe False $ do
+    cmd <- getCommandName token
+    return $ matcher cmd
+
+isConfusedGlobRegex ('*':_) = True
+isConfusedGlobRegex [x,'*'] | x /= '\\' = True
+isConfusedGlobRegex _ = False
+
+isVariableStartChar x = x == '_' || isAsciiLower x || isAsciiUpper x
+isVariableChar x = isVariableStartChar x || isDigit x
+variableNameRegex = mkRegex "[_a-zA-Z][_a-zA-Z0-9]*"
+
+prop_isVariableName1 = isVariableName "_fo123"
+prop_isVariableName2 = not $ isVariableName "4"
+prop_isVariableName3 = not $ isVariableName "test: "
+isVariableName (x:r) = isVariableStartChar x && all isVariableChar r
+isVariableName _ = False
+
+getVariablesFromLiteralToken token =
+    getVariablesFromLiteral (fromJust $ getLiteralStringExt (const $ return " ") token)
+
+-- Try to get referenced variables from a literal string like "$foo"
+-- Ignores tons of cases like arithmetic evaluation and array indices.
+prop_getVariablesFromLiteral1 =
+    getVariablesFromLiteral "$foo${bar//a/b}$BAZ" == ["foo", "bar", "BAZ"]
+getVariablesFromLiteral string =
+    map (!! 0) $ matchAllSubgroups variableRegex string
+  where
+    variableRegex = mkRegex "\\$\\{?([A-Za-z0-9_]+)"
+
+prop_getBracedReference1 = getBracedReference "foo" == "foo"
+prop_getBracedReference2 = getBracedReference "#foo" == "foo"
+prop_getBracedReference3 = getBracedReference "#" == "#"
+prop_getBracedReference4 = getBracedReference "##" == "#"
+prop_getBracedReference5 = getBracedReference "#!" == "!"
+prop_getBracedReference6 = getBracedReference "!#" == "#"
+prop_getBracedReference7 = getBracedReference "!foo#?" == "foo"
+prop_getBracedReference8 = getBracedReference "foo-bar" == "foo"
+prop_getBracedReference9 = getBracedReference "foo:-bar" == "foo"
+prop_getBracedReference10= getBracedReference "foo: -1" == "foo"
+prop_getBracedReference11= getBracedReference "!os*" == ""
+prop_getBracedReference12= getBracedReference "!os?bar**" == ""
+getBracedReference s = fromMaybe s $
+    nameExpansion s `mplus` takeName noPrefix `mplus` getSpecial noPrefix `mplus` getSpecial s
+  where
+    noPrefix = dropPrefix s
+    dropPrefix (c:rest) = if c `elem` "!#" then rest else c:rest
+    dropPrefix "" = ""
+    takeName s = do
+        let name = takeWhile isVariableChar s
+        guard . not $ null name
+        return name
+    getSpecial (c:_) =
+        if c `elem` "*@#?-$!" then return [c] else fail "not special"
+    getSpecial _ = fail "empty"
+
+    nameExpansion ('!':rest) = do -- e.g. ${!foo*bar*}
+        let suffix = dropWhile isVariableChar rest
+        guard $ suffix /= rest -- e.g. ${!@}
+        first <- suffix !!! 0
+        guard $ first `elem` "*?"
+        return ""
+    nameExpansion _ = Nothing
+
+
+-- Useful generic functions
+potentially :: Monad m => Maybe (m ()) -> m ()
+potentially = fromMaybe (return ())
+
+headOrDefault _ (a:_) = a
+headOrDefault def _ = def
+
+(!!!) list i =
+    case drop i list of
+        [] -> Nothing
+        (r:_) -> Just r
+
+
+
+filterByAnnotation token =
+    filter (not . shouldIgnore)
+  where
+    idFor (TokenComment id _) = id
+    shouldIgnore note =
+        any (shouldIgnoreFor (getCode note)) $
+            getPath parents (T_Bang $ idFor note)
+    shouldIgnoreFor num (T_Annotation _ anns _) =
+        any hasNum anns
+      where
+        hasNum (DisableComment ts) = num == ts
+        hasNum _ = False
+    shouldIgnoreFor _ (T_Include {}) = True -- Ignore included files
+    shouldIgnoreFor _ _ = False
+    parents = getParentTree token
+    getCode (TokenComment _ (Comment _ c _)) = c
+
+
+return [] 
+runTests =  $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |])

ShellCheck/Checks/Commands.hs

@@ -0,0 +1,560 @@
+{-
+    Copyright 2012-2015 Vidar Holen
+
+    This file is part of ShellCheck.
+    http://www.vidarholen.net/contents/shellcheck
+
+    ShellCheck is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    ShellCheck is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+-}
+{-# LANGUAGE TemplateHaskell #-}
+{-# LANGUAGE FlexibleContexts #-}
+
+-- This module contains checks that examine specific commands by name.
+module ShellCheck.Checks.Commands (runChecks
+    , ShellCheck.Checks.Commands.runTests
+) where
+
+import ShellCheck.AST
+import ShellCheck.ASTLib
+import ShellCheck.AnalyzerLib
+import ShellCheck.Data
+import ShellCheck.Interface
+import ShellCheck.Parser
+import ShellCheck.Regex
+
+import Control.Monad
+import Control.Monad.Reader
+import Control.Monad.Writer
+import Data.Char
+import Data.List
+import Data.Maybe
+import qualified Data.Map as Map
+import Test.QuickCheck.All (forAllProperties)
+import Test.QuickCheck.Test (quickCheckWithResult, stdArgs, maxSuccess)
+
+data CommandName = Exactly String | Basename String
+    deriving (Eq, Ord)
+
+data CommandCheck =
+    CommandCheck CommandName (Token -> Analysis)
+
+nullCheck :: Token -> Analysis
+nullCheck _ = return ()
+
+
+verify :: CommandCheck -> String -> Bool
+verify f s = producesComments f s == Just True
+verifyNot f s = producesComments f s == Just False
+
+producesComments :: CommandCheck -> String -> Maybe Bool
+producesComments f s = do
+        root <- pScript s
+        return . not . null $ runList (defaultSpec root) [f]
+
+composeChecks f g t = do
+    f t
+    g t
+
+arguments (T_SimpleCommand _ _ (cmd:args)) = args
+
+commandChecks :: [CommandCheck]
+commandChecks = [
+    checkTr
+    ,checkFindNameGlob
+    ,checkNeedlessExpr
+    ,checkGrepRe
+    ,checkTrapQuotes
+    ,checkReturn
+    ,checkFindExecWithSingleArgument
+    ,checkUnusedEchoEscapes
+    ,checkInjectableFindSh
+    ,checkFindActionPrecedence
+    ,checkMkdirDashPM
+    ,checkNonportableSignals
+    ,checkInteractiveSu
+    ,checkSshCommandString
+    ,checkPrintfVar
+    ,checkUuoeCmd
+    ,checkSetAssignment
+    ,checkExportedExpansions
+    ,checkAliasesUsesArgs
+    ,checkAliasesExpandEarly
+    ]
+
+buildCommandMap :: [CommandCheck] -> Map.Map CommandName (Token -> Analysis)
+buildCommandMap = foldl' addCheck Map.empty
+  where
+    addCheck map (CommandCheck name function) =
+        Map.insertWith' composeChecks name function map
+
+
+checkCommand :: Map.Map CommandName (Token -> Analysis) -> Token -> Analysis
+checkCommand map t@(T_SimpleCommand id _ (cmd:rest)) = fromMaybe (return ()) $ do
+    name <- getLiteralString cmd
+    return $
+        if '/' `elem` name
+        then
+            Map.findWithDefault nullCheck (Basename $ basename name) map t
+        else do
+            Map.findWithDefault nullCheck (Exactly name) map t
+            Map.findWithDefault nullCheck (Basename name) map t
+
+  where
+    basename = reverse . takeWhile (/= '/') . reverse
+checkCommand _ _ = return ()
+
+runList spec list = notes
+    where
+        root = asScript spec
+        params = makeParameters spec
+        notes = execWriter $ runReaderT (doAnalysis (checkCommand map) root) params
+        map = buildCommandMap list
+
+runChecks spec = runList spec commandChecks
+
+
+prop_checkTr1 = verify checkTr "tr [a-f] [A-F]"
+prop_checkTr2 = verify checkTr "tr 'a-z' 'A-Z'"
+prop_checkTr2a= verify checkTr "tr '[a-z]' '[A-Z]'"
+prop_checkTr3 = verifyNot checkTr "tr -d '[:lower:]'"
+prop_checkTr3a= verifyNot checkTr "tr -d '[:upper:]'"
+prop_checkTr3b= verifyNot checkTr "tr -d '|/_[:upper:]'"
+prop_checkTr4 = verifyNot checkTr "ls [a-z]"
+prop_checkTr5 = verify checkTr "tr foo bar"
+prop_checkTr6 = verify checkTr "tr 'hello' 'world'"
+prop_checkTr8 = verifyNot checkTr "tr aeiou _____"
+prop_checkTr9 = verifyNot checkTr "a-z n-za-m"
+prop_checkTr10= verifyNot checkTr "tr --squeeze-repeats rl lr"
+prop_checkTr11= verifyNot checkTr "tr abc '[d*]'"
+checkTr = CommandCheck (Basename "tr") (mapM_ f . arguments)
+  where
+    f w | isGlob w = -- The user will go [ab] -> '[ab]' -> 'ab'. Fixme?
+        warn (getId w) 2060 "Quote parameters to tr to prevent glob expansion."
+    f word =
+      case getLiteralString word of
+        Just "a-z" -> info (getId word) 2018 "Use '[:lower:]' to support accents and foreign alphabets."
+        Just "A-Z" -> info (getId word) 2019 "Use '[:upper:]' to support accents and foreign alphabets."
+        Just s -> do  -- Eliminate false positives by only looking for dupes in SET2?
+          when (not ("-" `isPrefixOf` s || "[:" `isInfixOf` s) && duplicated s) $
+            info (getId word) 2020 "tr replaces sets of chars, not words (mentioned due to duplicates)."
+          unless ("[:" `isPrefixOf` s) $
+            when ("[" `isPrefixOf` s && "]" `isSuffixOf` s && (length s > 2) && ('*' `notElem` s)) $
+              info (getId word) 2021 "Don't use [] around ranges in tr, it replaces literal square brackets."
+        Nothing -> return ()
+
+    duplicated s =
+        let relevant = filter isAlpha s
+        in relevant /= nub relevant
+
+prop_checkFindNameGlob1 = verify checkFindNameGlob "find / -name *.php"
+prop_checkFindNameGlob2 = verify checkFindNameGlob "find / -type f -ipath *(foo)"
+prop_checkFindNameGlob3 = verifyNot checkFindNameGlob "find * -name '*.php'"
+checkFindNameGlob = CommandCheck (Basename "find") (f . arguments)  where
+    acceptsGlob (Just s) = s `elem` [ "-ilname", "-iname", "-ipath", "-iregex", "-iwholename", "-lname", "-name", "-path", "-regex", "-wholename" ]
+    acceptsGlob _ = False
+    f [] = return ()
+    f [x] = return ()
+    f (a:b:r) = do
+        when (acceptsGlob (getLiteralString a) && isGlob b) $ do
+            let (Just s) = getLiteralString a
+            warn (getId b) 2061 $ "Quote the parameter to " ++ s ++ " so the shell won't interpret it."
+        f (b:r)
+
+
+prop_checkNeedlessExpr = verify checkNeedlessExpr "foo=$(expr 3 + 2)"
+prop_checkNeedlessExpr2 = verify checkNeedlessExpr "foo=`echo \\`expr 3 + 2\\``"
+prop_checkNeedlessExpr3 = verifyNot checkNeedlessExpr "foo=$(expr foo : regex)"
+prop_checkNeedlessExpr4 = verifyNot checkNeedlessExpr "foo=$(expr foo \\< regex)"
+checkNeedlessExpr = CommandCheck (Basename "expr") f where
+    f t =
+        when (all (`notElem` exceptions) (words $ arguments t)) $
+            style (getId t) 2003
+                "expr is antiquated. Consider rewriting this using $((..)), ${} or [[ ]]."
+    -- These operators are hard to replicate in POSIX
+    exceptions = [ ":", "<", ">", "<=", ">=" ]
+    words = mapMaybe getLiteralString
+
+
+prop_checkGrepRe1 = verify checkGrepRe "cat foo | grep *.mp3"
+prop_checkGrepRe2 = verify checkGrepRe "grep -Ev cow*test *.mp3"
+prop_checkGrepRe3 = verify checkGrepRe "grep --regex=*.mp3 file"
+prop_checkGrepRe4 = verifyNot checkGrepRe "grep foo *.mp3"
+prop_checkGrepRe5 = verifyNot checkGrepRe "grep-v  --regex=moo *"
+prop_checkGrepRe6 = verifyNot checkGrepRe "grep foo \\*.mp3"
+prop_checkGrepRe7 = verify checkGrepRe "grep *foo* file"
+prop_checkGrepRe8 = verify checkGrepRe "ls | grep foo*.jpg"
+prop_checkGrepRe9 = verifyNot checkGrepRe "grep '[0-9]*' file"
+prop_checkGrepRe10= verifyNot checkGrepRe "grep '^aa*' file"
+prop_checkGrepRe11= verifyNot checkGrepRe "grep --include=*.png foo"
+
+checkGrepRe = CommandCheck (Basename "grep") (f . arguments) where
+    -- --regex=*(extglob) doesn't work. Fixme?
+    skippable (Just s) = not ("--regex=" `isPrefixOf` s) && "-" `isPrefixOf` s
+    skippable _ = False
+    f [] = return ()
+    f (x:r) | skippable (getLiteralStringExt (const $ return "_") x) = f r
+    f (re:_) = do
+        when (isGlob re) $
+            warn (getId re) 2062 "Quote the grep pattern so the shell won't interpret it."
+        let string = concat $ oversimplify re
+        if isConfusedGlobRegex string then
+            warn (getId re) 2063 "Grep uses regex, but this looks like a glob."
+          else potentially $ do
+            char <- getSuspiciousRegexWildcard string
+            return $ info (getId re) 2022 $
+                "Note that unlike globs, " ++ [char] ++ "* here matches '" ++ [char, char, char] ++ "' but not '" ++ wordStartingWith char ++ "'."
+
+    wordStartingWith c =
+        head . filter ([c] `isPrefixOf`) $ candidates
+      where
+        candidates =
+            sampleWords ++ map (\(x:r) -> toUpper x : r) sampleWords ++ [c:"test"]
+
+    getSuspiciousRegexWildcard str =
+        if not $ str `matches` contra
+        then do
+            match <- matchRegex suspicious str
+            str <- match !!! 0
+            str !!! 0
+        else
+            fail "looks good"
+      where
+        suspicious = mkRegex "([A-Za-z1-9])\\*"
+        contra = mkRegex "[^a-zA-Z1-9]\\*|[][^$+\\\\]"
+
+
+prop_checkTrapQuotes1 = verify checkTrapQuotes "trap \"echo $num\" INT"
+prop_checkTrapQuotes1a= verify checkTrapQuotes "trap \"echo `ls`\" INT"
+prop_checkTrapQuotes2 = verifyNot checkTrapQuotes "trap 'echo $num' INT"
+prop_checkTrapQuotes3 = verify checkTrapQuotes "trap \"echo $((1+num))\" EXIT DEBUG"
+checkTrapQuotes = CommandCheck (Exactly "trap") (f . arguments) where
+    f (x:_) = checkTrap x
+    f _ = return ()
+    checkTrap (T_NormalWord _ [T_DoubleQuoted _ rs]) = mapM_ checkExpansions rs
+    checkTrap _ = return ()
+    warning id = warn id 2064 "Use single quotes, otherwise this expands now rather than when signalled."
+    checkExpansions (T_DollarExpansion id _) = warning id
+    checkExpansions (T_Backticked id _) = warning id
+    checkExpansions (T_DollarBraced id _) = warning id
+    checkExpansions (T_DollarArithmetic id _) = warning id
+    checkExpansions _ = return ()
+
+
+prop_checkReturn1 = verifyNot checkReturn "return"
+prop_checkReturn2 = verifyNot checkReturn "return 1"
+prop_checkReturn3 = verifyNot checkReturn "return $var"
+prop_checkReturn4 = verifyNot checkReturn "return $((a|b))"
+prop_checkReturn5 = verify checkReturn "return -1"
+prop_checkReturn6 = verify checkReturn "return 1000"
+prop_checkReturn7 = verify checkReturn "return 'hello world'"
+checkReturn = CommandCheck (Exactly "return") (f . arguments)
+  where
+    f (first:second:_) =
+        err (getId second) 2151
+            "Only one integer 0-255 can be returned. Use stdout for other data."
+    f [value] =
+        when (isInvalid $ literal value) $
+            err (getId value) 2152
+                "Can only return 0-255. Other data should be written to stdout."
+    f _ = return ()
+
+    isInvalid s = s == "" || any (not . isDigit) s || length s > 5
+        || let value = (read s :: Integer) in value > 255
+
+    literal token = fromJust $ getLiteralStringExt lit token
+    lit (T_DollarBraced {}) = return "0"
+    lit (T_DollarArithmetic {}) = return "0"
+    lit (T_DollarExpansion {}) = return "0"
+    lit (T_Backticked {}) = return "0"
+    lit _ = return "WTF"
+
+
+prop_checkFindExecWithSingleArgument1 = verify checkFindExecWithSingleArgument "find . -exec 'cat {} | wc -l' \\;"
+prop_checkFindExecWithSingleArgument2 = verify checkFindExecWithSingleArgument "find . -execdir 'cat {} | wc -l' +"
+prop_checkFindExecWithSingleArgument3 = verifyNot checkFindExecWithSingleArgument "find . -exec wc -l {} \\;"
+checkFindExecWithSingleArgument = CommandCheck (Basename "find") (f . arguments)
+  where
+    f = void . sequence . mapMaybe check . tails
+    check (exec:arg:term:_) = do
+        execS <- getLiteralString exec
+        termS <- getLiteralString term
+        cmdS <- getLiteralStringExt (const $ return " ") arg
+
+        guard $ execS `elem` ["-exec", "-execdir"] && termS `elem` [";", "+"]
+        guard $ cmdS `matches` commandRegex
+        return $ warn (getId exec) 2150 "-exec does not invoke a shell. Rewrite or use -exec sh -c .. ."
+    check _ = Nothing
+    commandRegex = mkRegex "[ |;]"
+
+
+prop_checkUnusedEchoEscapes1 = verify checkUnusedEchoEscapes "echo 'foo\\nbar\\n'"
+prop_checkUnusedEchoEscapes2 = verifyNot checkUnusedEchoEscapes "echo -e 'foi\\nbar'"
+prop_checkUnusedEchoEscapes3 = verify checkUnusedEchoEscapes "echo \"n:\\t42\""
+prop_checkUnusedEchoEscapes4 = verifyNot checkUnusedEchoEscapes "echo lol"
+prop_checkUnusedEchoEscapes5 = verifyNot checkUnusedEchoEscapes "echo -n -e '\n'"
+checkUnusedEchoEscapes = CommandCheck (Basename "echo") (f . arguments)
+  where
+    isDashE = mkRegex "^-.*e"
+    hasEscapes = mkRegex "\\\\[rnt]"
+    f args | concat (concatMap oversimplify allButLast) `matches` isDashE =
+        return ()
+      where allButLast = reverse . drop 1 . reverse $ args
+    f args = mapM_ checkEscapes args
+
+    checkEscapes (T_NormalWord _ args) =
+        mapM_ checkEscapes args
+    checkEscapes (T_DoubleQuoted id args) =
+        mapM_ checkEscapes args
+    checkEscapes (T_Literal id str) = examine id str
+    checkEscapes (T_SingleQuoted id str) = examine id str
+    checkEscapes _ = return ()
+
+    examine id str =
+        when (str `matches` hasEscapes) $
+            info id 2028 "echo won't expand escape sequences. Consider printf."
+
+
+prop_checkInjectableFindSh1 = verify checkInjectableFindSh "find . -exec sh -c 'echo {}' \\;"
+prop_checkInjectableFindSh2 = verify checkInjectableFindSh "find . -execdir bash -c 'rm \"{}\"' ';'"
+prop_checkInjectableFindSh3 = verifyNot checkInjectableFindSh "find . -exec sh -c 'rm \"$@\"' _ {} \\;"
+checkInjectableFindSh = CommandCheck (Basename "find") (check . arguments)
+  where
+    check args = do
+        let idStrings = map (\x -> (getId x, onlyLiteralString x)) args
+        match pattern idStrings
+
+    match _ [] = return ()
+    match [] (next:_) = action next
+    match (p:tests) ((id, arg):args) = do
+        when (p arg) $ match tests args
+        match (p:tests) args
+
+    pattern = [
+        (`elem` ["-exec", "-execdir"]),
+        (`elem` ["sh", "bash", "ksh"]),
+        (== "-c")
+        ]
+    action (id, arg) =
+        when ("{}" `isInfixOf` arg) $
+            warn id 2156 "Injecting filenames is fragile and insecure. Use parameters."
+
+
+prop_checkFindActionPrecedence1 = verify checkFindActionPrecedence "find . -name '*.wav' -o -name '*.au' -exec rm {} +"
+prop_checkFindActionPrecedence2 = verifyNot checkFindActionPrecedence "find . -name '*.wav' -o \\( -name '*.au' -exec rm {} + \\)"
+prop_checkFindActionPrecedence3 = verifyNot checkFindActionPrecedence "find . -name '*.wav' -o -name '*.au'"
+checkFindActionPrecedence = CommandCheck (Basename "find") (f . arguments)
+  where
+    pattern = [isMatch, const True, isParam ["-o", "-or"], isMatch, const True, isAction]
+    f list | length list < length pattern = return ()
+    f list@(_:rest) =
+        if and (zipWith ($) pattern list)
+        then warnFor (list !! (length pattern - 1))
+        else f rest
+    isMatch = isParam [ "-name", "-regex", "-iname", "-iregex", "-wholename", "-iwholename" ]
+    isAction = isParam [ "-exec", "-execdir", "-delete", "-print", "-print0" ]
+    isParam strs t = fromMaybe False $ do
+        param <- getLiteralString t
+        return $ param `elem` strs
+    warnFor t = warn (getId t) 2146 "This action ignores everything before the -o. Use \\( \\) to group."
+
+
+prop_checkMkdirDashPM0 = verify checkMkdirDashPM "mkdir -p -m 0755 a/b"
+prop_checkMkdirDashPM1 = verify checkMkdirDashPM "mkdir -pm 0755 $dir"
+prop_checkMkdirDashPM2 = verify checkMkdirDashPM "mkdir -vpm 0755 a/b"
+prop_checkMkdirDashPM3 = verify checkMkdirDashPM "mkdir -pm 0755 -v a/b"
+prop_checkMkdirDashPM4 = verify checkMkdirDashPM "mkdir --parents --mode=0755 a/b"
+prop_checkMkdirDashPM5 = verify checkMkdirDashPM "mkdir --parents --mode 0755 a/b"
+prop_checkMkdirDashPM6 = verify checkMkdirDashPM "mkdir -p --mode=0755 a/b"
+prop_checkMkdirDashPM7 = verify checkMkdirDashPM "mkdir --parents -m 0755 a/b"
+prop_checkMkdirDashPM8 = verifyNot checkMkdirDashPM "mkdir -p a/b"
+prop_checkMkdirDashPM9 = verifyNot checkMkdirDashPM "mkdir -m 0755 a/b"
+prop_checkMkdirDashPM10 = verifyNot checkMkdirDashPM "mkdir a/b"
+prop_checkMkdirDashPM11 = verifyNot checkMkdirDashPM "mkdir --parents a/b"
+prop_checkMkdirDashPM12 = verifyNot checkMkdirDashPM "mkdir --mode=0755 a/b"
+prop_checkMkdirDashPM13 = verifyNot checkMkdirDashPM "mkdir_func -pm 0755 a/b"
+prop_checkMkdirDashPM14 = verifyNot checkMkdirDashPM "mkdir -p -m 0755 singlelevel"
+checkMkdirDashPM = CommandCheck (Basename "mkdir") check
+  where
+    check t = potentially $ do
+        let flags = getAllFlags t
+        dashP <- find ((\f -> f == "p" || f == "parents") . snd) flags
+        dashM <- find ((\f -> f == "m" || f == "mode") . snd) flags
+        guard $ any couldHaveSubdirs (drop 1 $ arguments t) -- mkdir -pm 0700 dir  is fine, but dir/subdir is not.
+        return $ warn (getId $ fst dashM) 2174 "When used with -p, -m only applies to the deepest directory."
+    couldHaveSubdirs t = fromMaybe True $ do
+        name <- getLiteralString t
+        return $ '/' `elem` name
+
+
+prop_checkNonportableSignals1 = verify checkNonportableSignals "trap f 8"
+prop_checkNonportableSignals2 = verifyNot checkNonportableSignals "trap f 0"
+prop_checkNonportableSignals3 = verifyNot checkNonportableSignals "trap f 14"
+prop_checkNonportableSignals4 = verify checkNonportableSignals "trap f SIGKILL"
+prop_checkNonportableSignals5 = verify checkNonportableSignals "trap f 9"
+prop_checkNonportableSignals6 = verify checkNonportableSignals "trap f stop"
+checkNonportableSignals = CommandCheck (Exactly "trap") (f . arguments)
+  where
+    f = mapM_ check
+    check param = potentially $ do
+        str <- getLiteralString param
+        let id = getId param
+        return $ sequence_ $ mapMaybe (\f -> f id str) [
+            checkNumeric,
+            checkUntrappable
+            ]
+
+    checkNumeric id str = do
+        guard $ not (null str)
+        guard $ all isDigit str
+        guard $ str /= "0" -- POSIX exit trap
+        guard $ str `notElem` ["1", "2", "3", "6", "9", "14", "15" ] -- XSI
+        return $ warn id 2172
+            "Trapping signals by number is not well defined. Prefer signal names."
+
+    checkUntrappable id str = do
+        guard $ map toLower str `elem` ["kill", "9", "sigkill", "stop", "sigstop"]
+        return $ err id 2173
+            "SIGKILL/SIGSTOP can not be trapped."
+
+
+prop_checkInteractiveSu1 = verify checkInteractiveSu "su; rm file; su $USER"
+prop_checkInteractiveSu2 = verify checkInteractiveSu "su foo; something; exit"
+prop_checkInteractiveSu3 = verifyNot checkInteractiveSu "echo rm | su foo"
+prop_checkInteractiveSu4 = verifyNot checkInteractiveSu "su root < script"
+checkInteractiveSu = CommandCheck (Basename "su") f
+  where
+    f cmd = when (length (arguments cmd) <= 1) $ do
+        path <- pathTo cmd
+        when (all undirected path) $
+            info (getId cmd) 2117
+                "To run commands as another user, use su -c or sudo."
+
+    undirected (T_Pipeline _ _ l) = length l <= 1
+    -- This should really just be modifications to stdin, but meh
+    undirected (T_Redirecting _ list _) = null list
+    undirected _ = True
+
+
+-- This is hard to get right without properly parsing ssh args
+prop_checkSshCmdStr1 = verify checkSshCommandString "ssh host \"echo $PS1\""
+prop_checkSshCmdStr2 = verifyNot checkSshCommandString "ssh host \"ls foo\""
+prop_checkSshCmdStr3 = verifyNot checkSshCommandString "ssh \"$host\""
+checkSshCommandString = CommandCheck (Basename "ssh") (f . arguments)
+  where
+    nonOptions =
+        filter (\x -> not $ "-" `isPrefixOf` concat (oversimplify x))
+    f args =
+        case nonOptions args of
+            (hostport:r@(_:_)) -> checkArg $ last r
+            _ -> return ()
+    checkArg (T_NormalWord _ [T_DoubleQuoted id parts]) =
+        case filter (not . isConstant) parts of
+            [] -> return ()
+            (x:_) -> info (getId x) 2029
+                "Note that, unescaped, this expands on the client side."
+    checkArg _ = return ()
+
+
+prop_checkPrintfVar1 = verify checkPrintfVar "printf \"Lol: $s\""
+prop_checkPrintfVar2 = verifyNot checkPrintfVar "printf 'Lol: $s'"
+prop_checkPrintfVar3 = verify checkPrintfVar "printf -v cow $(cmd)"
+prop_checkPrintfVar4 = verifyNot checkPrintfVar "printf \"%${count}s\" var"
+checkPrintfVar = CommandCheck (Exactly "printf") (f . arguments) where
+    f (dashv:var:rest) | getLiteralString dashv == Just "-v" = f rest
+    f (format:params) = check format
+    f _ = return ()
+    check format =
+        unless ('%' `elem` concat (oversimplify format) || isLiteral format) $
+          warn (getId format) 2059
+              "Don't use variables in the printf format string. Use printf \"..%s..\" \"$foo\"."
+
+
+prop_checkUuoeCmd1 = verify checkUuoeCmd "echo $(date)"
+prop_checkUuoeCmd2 = verify checkUuoeCmd "echo `date`"
+prop_checkUuoeCmd3 = verify checkUuoeCmd "echo \"$(date)\""
+prop_checkUuoeCmd4 = verify checkUuoeCmd "echo \"`date`\""
+prop_checkUuoeCmd5 = verifyNot checkUuoeCmd "echo \"The time is $(date)\""
+prop_checkUuoeCmd6 = verifyNot checkUuoeCmd "echo \"$(<file)\""
+checkUuoeCmd = CommandCheck (Exactly "echo") (f . arguments) where
+    msg id = style id 2005 "Useless echo? Instead of 'echo $(cmd)', just use 'cmd'."
+    f [token] = when (tokenIsJustCommandOutput token) $ msg (getId token)
+    f _ = return ()
+
+
+prop_checkSetAssignment1 = verify checkSetAssignment "set foo 42"
+prop_checkSetAssignment2 = verify checkSetAssignment "set foo = 42"
+prop_checkSetAssignment3 = verify checkSetAssignment "set foo=42"
+prop_checkSetAssignment4 = verifyNot checkSetAssignment "set -- if=/dev/null"
+prop_checkSetAssignment5 = verifyNot checkSetAssignment "set 'a=5'"
+prop_checkSetAssignment6 = verifyNot checkSetAssignment "set"
+checkSetAssignment = CommandCheck (Exactly "set") (f . arguments)
+  where
+    f (var:value:rest) =
+        let str = literal var in
+            when (isVariableName str || isAssignment str) $
+                msg (getId var)
+    f (var:_) =
+        when (isAssignment $ literal var) $
+            msg (getId var)
+    f _ = return ()
+
+    msg id = warn id 2121 "To assign a variable, use just 'var=value', no 'set ..'."
+
+    isAssignment str = '=' `elem` str
+    literal (T_NormalWord _ l) = concatMap literal l
+    literal (T_Literal _ str) = str
+    literal _ = "*"
+
+
+prop_checkExportedExpansions1 = verify checkExportedExpansions "export $foo"
+prop_checkExportedExpansions2 = verify checkExportedExpansions "export \"$foo\""
+prop_checkExportedExpansions3 = verifyNot checkExportedExpansions "export foo"
+checkExportedExpansions = CommandCheck (Exactly "export") (check . arguments)
+  where
+    check = mapM_ checkForVariables
+    checkForVariables f =
+        case getWordParts f of
+            [t@(T_DollarBraced {})] ->
+                warn (getId t) 2163 "Exporting an expansion rather than a variable."
+            _ -> return ()
+
+
+prop_checkAliasesUsesArgs1 = verify checkAliasesUsesArgs "alias a='cp $1 /a'"
+prop_checkAliasesUsesArgs2 = verifyNot checkAliasesUsesArgs "alias $1='foo'"
+prop_checkAliasesUsesArgs3 = verify checkAliasesUsesArgs "alias a=\"echo \\${@}\""
+checkAliasesUsesArgs = CommandCheck (Exactly "alias") (f . arguments)
+  where
+    re = mkRegex "\\$\\{?[0-9*@]"
+    f = mapM_ checkArg
+    checkArg arg =
+        let string = fromJust $ getLiteralStringExt (const $ return "_") arg in
+            when ('=' `elem` string && string `matches` re) $
+                err (getId arg) 2142
+                    "Aliases can't use positional parameters. Use a function."
+
+
+prop_checkAliasesExpandEarly1 = verify checkAliasesExpandEarly "alias foo=\"echo $PWD\""
+prop_checkAliasesExpandEarly2 = verifyNot checkAliasesExpandEarly "alias -p"
+prop_checkAliasesExpandEarly3 = verifyNot checkAliasesExpandEarly "alias foo='echo {1..10}'"
+checkAliasesExpandEarly = CommandCheck (Exactly "alias") (f . arguments)
+  where
+    f = mapM_ checkArg
+    checkArg arg | '=' `elem` concat (oversimplify arg) =
+        forM_ (take 1 $ filter (not . isLiteral) $ getWordParts arg) $
+            \x -> warn (getId x) 2139 "This expands when defined, not when used. Consider escaping."
+    checkArg _ = return ()
+
+
+return []
+runTests =  $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |])

ShellCheck/Parser.hs

@@ -52,20 +52,24 @@ type SCParser m v = ParsecT String UserState (SCBase m) v
 
 backslash :: Monad m => SCParser m Char
 backslash = char '\\'
-linefeed = optional carriageReturn >> char '\n'
+linefeed :: Monad m => SCParser m Char
+linefeed = do
+    optional carriageReturn
+    c <- char '\n'
+    readPendingHereDocs
+    return c
 singleQuote = char '\'' <|> unicodeSingleQuote
 doubleQuote = char '"' <|> unicodeDoubleQuote
 variableStart = upper <|> lower <|> oneOf "_"
 variableChars = upper <|> lower <|> digit <|> oneOf "_"
 functionChars = variableChars <|> oneOf ":+-.?"
 specialVariable = oneOf "@*#?-$!"
-tokenDelimiter = oneOf "&|;<> \t\n\r" <|> almostSpace
 quotableChars = "|&;<>()\\ '\t\n\r\xA0" ++ doubleQuotableChars
 quotable = almostSpace <|> unicodeDoubleQuote <|> oneOf quotableChars
 bracedQuotable = oneOf "}\"$`'"
 doubleQuotableChars = "\"$`" ++ unicodeDoubleQuoteChars
 doubleQuotable = unicodeDoubleQuote <|> oneOf doubleQuotableChars
-whitespace = oneOf " \t\n" <|> carriageReturn <|> almostSpace
+whitespace = oneOf " \t" <|> carriageReturn <|> almostSpace <|> linefeed
 linewhitespace = oneOf " \t" <|> almostSpace
 
 suspectCharAfterQuotes = variableChars <|> char '%'
@@ -138,15 +142,24 @@ data Context =
         | ContextSource String
     deriving (Show)
 
+data HereDocContext =
+        HereDocPending Token -- on linefeed, read this T_HereDoc
+        | HereDocBoundary -- but don't consider heredocs before this
+    deriving (Show)
+
 data UserState = UserState {
     lastId :: Id,
     positionMap :: Map.Map Id SourcePos,
-    parseNotes :: [ParseNote]
+    parseNotes :: [ParseNote],
+    hereDocMap :: Map.Map Id [Token],
+    pendingHereDocs :: [HereDocContext]
 }
 initialUserState = UserState {
     lastId = Id $ -1,
     positionMap = Map.empty,
-    parseNotes = []
+    parseNotes = [],
+    hereDocMap = Map.empty,
+    pendingHereDocs = []
 }
 
 codeForParseNote (ParseNote _ _ code _) = code
@@ -155,7 +168,6 @@ noteToParseNote map (Note id severity code message) =
     where
         pos = fromJust $ Map.lookup id map
 
-
 getLastId = lastId <$> getState
 
 getNextIdAt sourcepos = do
@@ -173,6 +185,58 @@ getNextId = do
     pos <- getPosition
     getNextIdAt pos
 
+addToHereDocMap id list = do
+    state <- getState
+    let map = hereDocMap state
+    putState $ state {
+        hereDocMap = Map.insert id list map
+    }
+
+withHereDocBoundary p = do
+    pushBoundary
+    do
+        v <- p
+        popBoundary
+        return v
+     <|> do
+        popBoundary
+        fail ""
+  where
+    pushBoundary = do
+        state <- getState
+        let docs = pendingHereDocs state
+        putState $ state {
+            pendingHereDocs = HereDocBoundary : docs
+        }
+    popBoundary = do
+        state <- getState
+        let docs = tail $ dropWhile (not . isHereDocBoundary) $
+                    pendingHereDocs state
+        putState $ state {
+            pendingHereDocs = docs
+        }
+
+addPendingHereDoc t = do
+    state <- getState
+    let docs = pendingHereDocs state
+    putState $ state {
+        pendingHereDocs = HereDocPending t : docs
+    }
+
+popPendingHereDocs = do
+    state <- getState
+    let (pending, boundary) = break isHereDocBoundary $ pendingHereDocs state
+    putState $ state {
+        pendingHereDocs = boundary
+    }
+    return . map extract . reverse $ pendingHereDocs state
+  where
+    extract (HereDocPending t) = t
+
+isHereDocBoundary x = case x of
+    HereDocBoundary -> True
+    otherwise -> False
+
 getMap = positionMap <$> getState
 getParseNotes = parseNotes <$> getState
 
@@ -522,8 +586,8 @@ readConditionContents single =
       where
         readGlobLiteral = do
             id <- getNextId
-            s <- many1 (extglobStart <|> oneOf "{}[]$")
-            return $ T_Literal id s
+            s <- extglobStart <|> oneOf "{}[]$"
+            return $ T_Literal id [s]
         readGroup = called "regex grouping" $ do
             id <- getNextId
             char '('
@@ -644,7 +708,9 @@ readArithmeticContents =
         l <- readAssignment `sepBy` (char ',' >> spacing)
         return $ TA_Sequence id l
 
-    readAssignment = readTrinary `splitBy` ["=", "*=", "/=", "%=", "+=", "-=", "<<=", ">>=", "&=", "^=", "|="]
+    readAssignment = chainr1 readTrinary readAssignmentOp
+    readAssignmentOp = readComboOp ["=", "*=", "/=", "%=", "+=", "-=", "<<=", ">>=", "&=", "^=", "|="] TA_Assignment
+
     readTrinary = do
         x <- readLogicalOr
         do
@@ -772,7 +838,7 @@ prop_readAnnotation3 = isOk readAnnotation "# shellcheck disable=SC1234 source=/
 readAnnotation = called "shellcheck annotation" $ do
     try readAnnotationPrefix
     many1 linewhitespace
-    values <- many1 (readDisable <|> readSourceOverride)
+    values <- many1 (readDisable <|> readSourceOverride <|> readShellOverride)
     linefeed
     many linewhitespace
     return $ concat values
@@ -789,6 +855,14 @@ readAnnotation = called "shellcheck annotation" $ do
         filename <- many1 $ noneOf " \n"
         return [SourceOverride filename]
 
+    readShellOverride = forKey "shell" $ do
+        pos <- getPosition
+        shell <- many1 $ noneOf " \n"
+        when (isNothing $ shellForExecutable shell) $
+            parseNoteAt pos ErrorC 1103
+                "This shell type is unknown. Use e.g. sh or bash."
+        return [ShellOverride shell]
+
     forKey s p = do
         try $ string s
         char '='
@@ -811,6 +885,9 @@ prop_readNormalWord3 = isOk readNormalWord "foo#"
 prop_readNormalWord4 = isOk readNormalWord "$\"foo\"$'foo\nbar'"
 prop_readNormalWord5 = isWarning readNormalWord "${foo}}"
 prop_readNormalWord6 = isOk readNormalWord "foo/{}"
+prop_readNormalWord7 = isOk readNormalWord "foo\\\nbar"
+prop_readNormalWord8 = isWarning readSubshell "(foo\\ \nbar)"
+prop_readNormalWord9 = isOk readSubshell "(foo\\ ;\nbar)"
 readNormalWord = readNormalishWord ""
 
 readNormalishWord end = do
@@ -1090,6 +1167,7 @@ readNormalEscaped = called "escaped char" $ do
     backslash
     do
         next <- quotable <|> oneOf "?*@!+[]{}.,~#"
+        when (next == ' ') $ checkTrailingSpaces pos <|> return ()
         return $ if next == '\n' then "" else [next]
       <|>
         do
@@ -1106,6 +1184,11 @@ readNormalEscaped = called "escaped char" $ do
     escapedChar 'r' = Just "carriage return"
     escapedChar _ = Nothing
 
+    checkTrailingSpaces pos = lookAhead . try $ do
+        many linewhitespace
+        void linefeed <|> eof
+        parseProblemAt pos ErrorC 1101 "Delete trailing spaces after \\ to break line (or use quotes for literal space)."
+
 
 prop_readExtglob1 = isOk readExtglob "!(*.mp3)"
 prop_readExtglob2 = isOk readExtglob "!(*.mp3|*.wmv)"
@@ -1217,7 +1300,10 @@ readBraced = try braceExpansion
 
 readNormalDollar = readDollarExpression <|> readDollarDoubleQuote <|> readDollarSingleQuote <|> readDollarLonely
 readDoubleQuotedDollar = readDollarExpression <|> readDollarLonely
-readDollarExpression = readDollarArithmetic <|> readDollarBracket <|> readDollarBraceCommandExpansion <|> readDollarBraced <|> readDollarExpansion <|> readDollarVariable
+
+prop_readDollarExpression1 = isOk readDollarExpression "$(((1) && 3))"
+prop_readDollarExpression2 = isWarning readDollarExpression "$(((1)) && 3)"
+readDollarExpression = readTripleParenthesis "$" readDollarArithmetic readDollarExpansion <|> readDollarArithmetic <|> readDollarBracket <|> readDollarBraceCommandExpansion <|> readDollarBraced <|> readDollarExpansion <|> readDollarVariable
 
 prop_readDollarSingleQuote = isOk readDollarSingleQuote "$'foo\\\'lol'"
 readDollarSingleQuote = called "$'..' expression" $ do
@@ -1243,7 +1329,9 @@ readDollarArithmetic = called "$((..)) expression" $ do
     id <- getNextId
     try (string "$((")
     c <- readArithmeticContents
-    string "))"
+    pos <- getPosition
+    char ')'
+    char ')' <|> fail "Expected a double )) to end the $((..))"
     return (T_DollarArithmetic id c)
 
 readDollarBracket = called "$[..] expression" $ do
@@ -1261,6 +1349,25 @@ readArithmeticExpression = called "((..)) command" $ do
     string "))"
     return (T_Arithmetic id c)
 
+-- Check if maybe ((( was intended as ( (( rather than (( (
+readTripleParenthesis prefix expected alternative = do
+    pos <- try . lookAhead $ do
+        string prefix
+        p <- getPosition
+        string "(((" -- should optimally be "((" but it's noisy and rarely helpful
+        return p
+
+    -- If the expected parser fails, try the alt.
+    -- If the alt fails, run the expected one again for the errors.
+    try expected <|> tryAlt pos <|> expected
+  where
+    tryAlt pos = do
+        t <- try alternative
+        parseNoteAt pos WarningC 1102 $
+            "Shells differ in parsing ambiguous " ++ prefix ++ "(((. Use spaces: " ++ prefix ++ "( (( ."
+        return t
+
+
 prop_readDollarBraceCommandExpansion1 = isOk readDollarBraceCommandExpansion "${ ls; }"
 prop_readDollarBraceCommandExpansion2 = isOk readDollarBraceCommandExpansion "${\nls\n}"
 readDollarBraceCommandExpansion = called "ksh ${ ..; } command expansion" $ do
@@ -1343,14 +1450,17 @@ readDollarLonely = do
     n <- lookAhead (anyChar <|> (eof >> return '_'))
     return $ T_Literal id "$"
 
-prop_readHereDoc = isOk readHereDoc "<< foo\nlol\ncow\nfoo"
-prop_readHereDoc2 = isWarning readHereDoc "<<- EOF\n  cow\n  EOF"
-prop_readHereDoc3 = isOk readHereDoc "<< foo\n$\"\nfoo"
-prop_readHereDoc4 = isOk readHereDoc "<< foo\n`\nfoo"
-prop_readHereDoc5 = isOk readHereDoc "<<- !foo\nbar\n!foo"
-prop_readHereDoc6 = isOk readHereDoc "<< foo\\ bar\ncow\nfoo bar"
-prop_readHereDoc7 = isOk readHereDoc "<< foo\n\\$(f ())\nfoo"
-prop_readHereDoc8 = isOk readHereDoc "<<foo>>bar\netc\nfoo"
+prop_readHereDoc = isOk readScript "cat << foo\nlol\ncow\nfoo"
+prop_readHereDoc2 = isWarning readScript "cat <<- EOF\n  cow\n  EOF"
+prop_readHereDoc3 = isOk readScript "cat << foo\n$\"\nfoo"
+prop_readHereDoc4 = isOk readScript "cat << foo\n`\nfoo"
+prop_readHereDoc5 = isOk readScript "cat <<- !foo\nbar\n!foo"
+prop_readHereDoc6 = isOk readScript "cat << foo\\ bar\ncow\nfoo bar"
+prop_readHereDoc7 = isOk readScript "cat << foo\n\\$(f ())\nfoo"
+prop_readHereDoc8 = isOk readScript "cat <<foo>>bar\netc\nfoo"
+prop_readHereDoc9 = isOk readScript "if true; then cat << foo; fi\nbar\nfoo\n"
+prop_readHereDoc10= isOk readScript "if true; then cat << foo << bar; fi\nfoo\nbar\n"
+prop_readHereDoc11= isOk readScript "cat << foo $(\nfoo\n)lol\nfoo\n"
 readHereDoc = called "here document" $ do
     fid <- getNextId
     pos <- getPosition
@@ -1367,24 +1477,11 @@ readHereDoc = called "here document" $ do
             liftM (\ x -> (Quoted, stripLiteral x)) readDoubleQuotedLiteral
             <|> liftM (\ x -> (Quoted, x)) readSingleQuotedLiteral
             <|> (readToken >>= (\x -> return (Unquoted, x)))
-    spacing
-
-    startPos <- getPosition
-    hereData <- anyChar `reluctantlyTill` do
-                    linefeed
-                    spacing
-                    string endToken
-                    disregard linefeed  <|> eof
-
-    do
-        linefeed
-        spaces <- spacing
-        verifyHereDoc dashed quoted spaces hereData
-        string endToken
-        parsedData <- parseHereData quoted startPos hereData
-        return $ T_FdRedirect fid "" $ T_HereDoc hid dashed quoted endToken parsedData
-     `attempting` (eof >> debugHereDoc tokenPosition endToken hereData)
 
+    -- add empty tokens for now, read the rest in readPendingHereDocs
+    let doc = T_HereDoc hid dashed quoted endToken []
+    addPendingHereDoc doc
+    return $ T_FdRedirect fid "" doc
   where
     stripLiteral (T_Literal _ x) = x
     stripLiteral (T_SingleQuoted _ x) = x
@@ -1399,6 +1496,27 @@ readHereDoc = called "here document" $ do
             c <- anyChar
             return [c]
 
+
+readPendingHereDocs = do
+    docs <- popPendingHereDocs
+    mapM_ readDoc docs
+  where
+    readDoc (T_HereDoc id dashed quoted endToken _) = do
+        pos <- getPosition
+        hereData <- anyChar `reluctantlyTill` do
+                        spacing
+                        string endToken
+                        disregard (char '\n') <|> eof
+        do
+            spaces <- spacing
+            verifyHereDoc dashed quoted spaces hereData
+            string endToken
+            parsedData <- parseHereData quoted pos hereData
+            list <- parseHereData quoted pos hereData
+            addToHereDocMap id list
+
+         `attempting` (eof >> debugHereDoc pos endToken hereData)
+
     parseHereData Quoted startPos hereData = do
         id <- getNextIdAt startPos
         return [T_Literal id hereData]
@@ -1483,7 +1601,7 @@ readHereString = called "here string" $ do
     word <- readNormalWord
     return $ T_FdRedirect id "" $ T_HereString id2 word
 
-readNewlineList = many1 ((newline <|> carriageReturn) `thenSkip` spacing)
+readNewlineList = many1 ((linefeed <|> carriageReturn) `thenSkip` spacing)
 readLineBreak = optional readNewlineList
 
 prop_readSeparator1 = isWarning readScript "a &; b"
@@ -1657,6 +1775,7 @@ readTermOrNone = do
         eof
         return []
 
+prop_readTerm = isOk readTerm "time ( foo; bar; )"
 readTerm = do
     allspacing
     m <- readAndOr
@@ -2010,6 +2129,7 @@ readFunctionDefinition = called "function" $ do
         readWithoutFunction = try $ do
             id <- getNextId
             name <- readFunctionName
+            guard $ name /= "time"  -- Interfers with time ( foo )
             spacing
             readParens
             return $ T_Function id (FunctionKeyword False) (FunctionParentheses True) name
@@ -2054,7 +2174,20 @@ readPattern = (readNormalWord `thenSkip` spacing) `sepBy1` (char '|' `thenSkip`
 prop_readCompoundCommand = isOk readCompoundCommand "{ echo foo; }>/dev/null"
 readCompoundCommand = do
     id <- getNextId
-    cmd <- choice [ readBraceGroup, readArithmeticExpression, readSubshell, readCondition, readWhileClause, readUntilClause, readIfClause, readForClause, readSelectClause, readCaseClause, readFunctionDefinition]
+    cmd <- choice [
+        readBraceGroup,
+        readTripleParenthesis "" readArithmeticExpression readSubshell,
+        readArithmeticExpression,
+        readSubshell,
+        readCondition,
+        readWhileClause,
+        readUntilClause,
+        readIfClause,
+        readForClause,
+        readSelectClause,
+        readCaseClause,
+        readFunctionDefinition
+        ]
     spacing
     redirs <- many readIoRedirect
     unless (null redirs) $ optional $ do
@@ -2083,13 +2216,29 @@ readTimeSuffix = do
         lookAhead $ char '-'
         readCmdWord
 
--- Fixme: this is a hack that doesn't handle let '++c' or let a\>b
+-- Fixme: this is a hack that doesn't handle let c='4'"5" or let a\>b
+readLetSuffix :: Monad m => SCParser m [Token]
 readLetSuffix = many1 (readIoRedirect <|> try readLetExpression <|> readCmdWord)
   where
+    readLetExpression :: Monad m => SCParser m Token
     readLetExpression = do
         startPos <- getPosition
         expression <- readStringForParser readCmdWord
-        subParse startPos readArithmeticContents expression
+        let (unQuoted, newPos) = kludgeAwayQuotes expression startPos
+        subParse newPos readArithmeticContents unQuoted
+
+    kludgeAwayQuotes :: String -> SourcePos -> (String, SourcePos)
+    kludgeAwayQuotes s p =
+        case s of
+            first:rest@(_:_) ->
+                let (last:backwards) = reverse rest
+                    middle = reverse backwards
+                in
+                    if first `elem` "'\"" && first == last
+                    then (middle, updatePosChar p first)
+                    else (s, p)
+            x -> (s, p)
+
 
 -- bash allows a=(b), ksh allows $a=(b). dash allows neither. Let's warn.
 readEvalSuffix = many1 (readIoRedirect <|> readCmdWord <|> evalFallback)
@@ -2288,8 +2437,11 @@ ifParse p t f =
 
 prop_readShebang1 = isOk readShebang "#!/bin/sh\n"
 prop_readShebang2 = isWarning readShebang "!# /bin/sh\n"
+prop_readShebang3 = isNotOk readShebang "#shellcheck shell=/bin/sh\n"
+prop_readShebang4 = isWarning readShebang "! /bin/sh"
 readShebang = do
-    try readCorrect <|> try readSwapped
+    try readCorrect <|> try readSwapped <|> try readMissingHash
+    many linewhitespace
     str <- many $ noneOf "\r\n"
     optional carriageReturn
     optional linefeed
@@ -2302,6 +2454,15 @@ readShebang = do
         parseProblemAt pos ErrorC 1084
             "Use #!, not !#, for the shebang."
 
+    readMissingHash = do
+        pos <- getPosition
+        char '!'
+        lookAhead $ do
+            many linewhitespace
+            char '/'
+        parseProblemAt pos ErrorC 1104
+            "Use #!, not just !, for the shebang."
+
 verifyEof = eof <|> choice [
         ifParsable g_Lparen $
             parseProblem ErrorC 1088 "Parsing stopped here. Invalid use of parentheses?",
@@ -2332,9 +2493,11 @@ readScript = do
     verifyShell pos (getShell sb)
     if isValidShell (getShell sb) /= Just False
       then do
-            commands <- readCompoundListOrEmpty
+            annotationId <- getNextId
+            annotations <- readAnnotations
+            commands <- withAnnotations annotations readCompoundListOrEmpty
             verifyEof
-            return $ T_Script id sb commands
+            return $ T_Annotation annotationId annotations $  T_Script id sb commands
         else do
             many anyChar
             return $ T_Script id sb []
@@ -2405,9 +2568,8 @@ parsesCleanly parser string = runIdentity $ do
 
 parseWithNotes parser = do
     item <- parser
-    map <- getMap
-    parseNotes <- getParseNotes
-    return (item, map, nub . sortNotes $ parseNotes)
+    state <- getState
+    return (item, state)
 
 compareNotes (ParseNote pos1 level1 _ s1) (ParseNote pos2 level2 _ s2) = compare (pos1, level1) (pos2, level2)
 sortNotes = sortBy compareNotes
@@ -2447,11 +2609,12 @@ system = lift . lift . lift
 parseShell sys name contents = do
     (result, state) <- runParser sys (parseWithNotes readScript) name contents
     case result of
-        Right (script, tokenMap, notes) ->
+        Right (script, userstate) ->
             return ParseResult {
-                prComments = map toPositionedComment $ nub $ notes ++ parseProblems state,
-                prTokenPositions = Map.map posToPos tokenMap,
-                prRoot = Just script
+                prComments = map toPositionedComment $ nub $ parseNotes userstate ++ parseProblems state,
+                prTokenPositions = Map.map posToPos (positionMap userstate),
+                prRoot = Just $
+                    reattachHereDocs script (hereDocMap userstate)
             }
         Left err ->
             return ParseResult {
@@ -2472,6 +2635,13 @@ parseShell sys name contents = do
     second (ContextName pos str) = ParseNote pos InfoC 1009 $
         "The mentioned parser error was in this " ++ str ++ "."
 
+reattachHereDocs root map =
+    doTransform f root
+  where
+    f t@(T_HereDoc id dash quote string []) = fromMaybe t $ do
+        list <- Map.lookup id map
+        return $ T_HereDoc id dash quote string list
+    f t = t
 
 toPositionedComment :: ParseNote -> PositionedComment
 toPositionedComment (ParseNote pos severity code message) =

quicktest

@@ -4,12 +4,18 @@
 # 'cabal test' remains the source of truth.
 
 (
- var=$(echo 'liftM and $ sequence [ShellCheck.Analytics.runTests, ShellCheck.Parser.runTests, ShellCheck.Checker.runTests]' | cabal repl | tee /dev/stderr)
+ var=$(echo 'liftM and $ sequence [
+  ShellCheck.Analytics.runTests
+  ,ShellCheck.Parser.runTests
+  ,ShellCheck.Checker.runTests
+  ,ShellCheck.Checks.Commands.runTests
+  ,ShellCheck.AnalyzerLib.runTests
+    ]' | tr -d '\n' | cabal repl 2>&1 | tee /dev/stderr)
 if [[ $var == *$'\nTrue'* ]]
 then
   exit 0
 else
-  grep -C 3 "Fail" <<< "$var"
+  grep -C 3 -e "Fail" -e "Tracing" <<< "$var"
   exit 1
 fi
 ) 2>&1

shellcheck.1.md

@@ -34,7 +34,7 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts.
 
 **-C**[*WHEN*],\ **--color**[=*WHEN*]
 
-:   For TTY outut, enable colors *always*, *never* or *auto*. The default
+:   For TTY output, enable colors *always*, *never* or *auto*. The default
     is *auto*. **--color** without an argument is equivalent to
     **--color=always**.
 

stack.yaml

@@ -0,0 +1,35 @@
+# This file was automatically generated by stack init
+# For more information, see: http://docs.haskellstack.org/en/stable/yaml_configuration/
+
+# Specifies the GHC version and set of packages available (e.g., lts-3.5, nightly-2015-09-21, ghc-7.10.2)
+resolver: lts-5.5
+
+# Local packages, usually specified by relative directory name
+packages:
+- '.'
+# Packages to be pulled from upstream that are not in the resolver (e.g., acme-missiles-0.3)
+extra-deps: []
+
+# Override default flag values for local packages and extra-deps
+flags: {}
+
+# Extra package databases containing global packages
+extra-package-dbs: []
+
+# Control whether we use the GHC we find on the path
+# system-ghc: true
+
+# Require a specific version of stack, using version ranges
+# require-stack-version: -any # Default
+# require-stack-version: >= 1.0.0
+
+# Override the architecture used by stack, especially useful on Windows
+# arch: i386
+# arch: x86_64
+
+# Extra directories used by stack for building
+# extra-include-dirs: [/path/to/dir]
+# extra-lib-dirs: [/path/to/dir]
+
+# Allow a newer minor version of GHC than the snapshot specifies
+# compiler-check: newer-minor

test/shellcheck.hs

@@ -4,13 +4,17 @@ import Control.Monad
 import System.Exit
 import qualified ShellCheck.Checker
 import qualified ShellCheck.Analytics
+import qualified ShellCheck.AnalyzerLib
 import qualified ShellCheck.Parser
+import qualified ShellCheck.Checks.Commands
 
 main = do
     putStrLn "Running ShellCheck tests..."
     results <- sequence [
         ShellCheck.Checker.runTests,
+        ShellCheck.Checks.Commands.runTests,
         ShellCheck.Analytics.runTests,
+        ShellCheck.AnalyzerLib.runTests,
         ShellCheck.Parser.runTests
       ]
     if and results