Stable version 0.4.0

This release is dedicated to the Debian project,
for basically inventing the modern Linux distro.

.ghci

@@ -0,0 +1 @@
+:set -idist/build/autogen

README.md

@@ -101,4 +101,10 @@ To run the unit test suite:
     cabal build
     cabal test
 
+## Reporting bugs
+
+Please use the Github issue tracker for any bugs or feature suggestions:
+
+https://github.com/koalaman/shellcheck/issues
+
 Happy ShellChecking!

ShellCheck.cabal

@@ -1,5 +1,5 @@
 Name:             ShellCheck
-Version:          0.3.8
+Version:          0.4.0
 Synopsis:         Shell script analysis tool
 License:          GPL-3
 License-file:     LICENSE
@@ -41,18 +41,21 @@ library
       containers,
       directory,
       json,
-      mtl,
+      mtl >= 2.2.1,
       parsec,
       regex-tdfa,
       QuickCheck >= 2.7.4
     exposed-modules:
-      ShellCheck.Analytics
       ShellCheck.AST
+      ShellCheck.ASTLib
+      ShellCheck.Analytics
+      ShellCheck.Analyzer
+      ShellCheck.Checker
       ShellCheck.Data
-      ShellCheck.Options
+      ShellCheck.Formatter.Format
+      ShellCheck.Interface
       ShellCheck.Parser
       ShellCheck.Regex
-      ShellCheck.Simple
     other-modules:
       Paths_ShellCheck
 
@@ -63,10 +66,9 @@ executable shellcheck
       containers,
       directory,
       json,
-      mtl,
+      mtl >= 2.2.1,
       parsec,
       regex-tdfa,
-      transformers,
       QuickCheck >= 2.7.4
     main-is: shellcheck.hs
 
@@ -78,10 +80,9 @@ test-suite test-shellcheck
       containers,
       directory,
       json,
-      mtl,
+      mtl >= 2.2.1,
       parsec,
       regex-tdfa,
-      transformers,
       QuickCheck >= 2.7.4
     main-is: test/shellcheck.hs
 

ShellCheck/AST.hs

@@ -1,4 +1,6 @@
 {-
+    Copyright 2012-2015 Vidar Holen
+
     This file is part of ShellCheck.
     http://www.vidarholen.net/contents/shellcheck
 
@@ -70,6 +72,7 @@ data Token =
     | T_DollarDoubleQuoted Id [Token]
     | T_DollarExpansion Id [Token]
     | T_DollarSingleQuoted Id String
+    | T_DollarBraceCommandExpansion Id [Token]
     | T_Done Id
     | T_DoubleQuoted Id [Token]
     | T_EOF Id
@@ -123,9 +126,10 @@ data Token =
     | T_Pipe Id String
     | T_CoProc Id (Maybe String) Token
     | T_CoProcBody Id Token
+    | T_Include Id Token Token -- . & source: SimpleCommand T_Script
     deriving (Show)
 
-data Annotation = DisableComment Integer deriving (Show, Eq)
+data Annotation = DisableComment Integer | SourceOverride String deriving (Show, Eq)
 data ConditionType = DoubleBracket | SingleBracket deriving (Show, Eq)
 
 -- This is an abomination.
@@ -171,6 +175,7 @@ analyze f g i =
     delve (T_DoubleQuoted id list) = dl list $ T_DoubleQuoted id
     delve (T_DollarDoubleQuoted id list) = dl list $ T_DollarDoubleQuoted id
     delve (T_DollarExpansion id list) = dl list $ T_DollarExpansion id
+    delve (T_DollarBraceCommandExpansion id list) = dl list $ T_DollarBraceCommandExpansion id
     delve (T_BraceExpansion id list) = dl list $ T_BraceExpansion id
     delve (T_Backticked id list) = dl list $ T_Backticked id
     delve (T_DollarArithmetic id c) = d1 c $ T_DollarArithmetic id
@@ -253,6 +258,7 @@ analyze f g i =
     delve (T_Annotation id anns t) = d1 t $ T_Annotation id anns
     delve (T_CoProc id var body) = d1 body $ T_CoProc id var
     delve (T_CoProcBody id t) = d1 t $ T_CoProcBody id
+    delve (T_Include id includer script) = d2 includer script $ T_Include id
     delve t = return t
 
 getId t = case t of
@@ -298,6 +304,7 @@ getId t = case t of
         T_DollarBraced id _  -> id
         T_DollarArithmetic id _  -> id
         T_BraceExpansion id _  -> id
+        T_DollarBraceCommandExpansion id _  -> id
         T_IoFile id _ _  -> id
         T_HereDoc id _ _ _ _ -> id
         T_HereString id _  -> id
@@ -348,6 +355,7 @@ getId t = case t of
         T_Pipe id _ -> id
         T_CoProc id _ _ -> id
         T_CoProcBody id _ -> id
+        T_Include id _ _ -> id
 
 blank :: Monad m => Token -> m ()
 blank = const $ return ()
@@ -355,10 +363,3 @@ doAnalysis f = analyze f blank id
 doStackAnalysis startToken endToken = analyze startToken endToken id
 doTransform i = runIdentity . analyze blank blank i
 
-isLoop t = case t of
-        T_WhileExpression {} -> True
-        T_UntilExpression {} -> True
-        T_ForIn {} -> True
-        T_ForArithmetic {} -> True
-        T_SelectIn {}  -> True
-        _ -> False

ShellCheck/ASTLib.hs

@@ -0,0 +1,240 @@
+{-
+    Copyright 2012-2015 Vidar Holen
+
+    This file is part of ShellCheck.
+    http://www.vidarholen.net/contents/shellcheck
+
+    ShellCheck is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    ShellCheck is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+-}
+module ShellCheck.ASTLib where
+
+import ShellCheck.AST
+
+import Control.Monad
+import Data.List
+import Data.Maybe
+
+-- Is this a type of loop?
+isLoop t = case t of
+        T_WhileExpression {} -> True
+        T_UntilExpression {} -> True
+        T_ForIn {} -> True
+        T_ForArithmetic {} -> True
+        T_SelectIn {}  -> True
+        _ -> False
+
+-- Will this split into multiple words when used as an argument?
+willSplit x =
+  case x of
+    T_DollarBraced {} -> True
+    T_DollarExpansion {} -> True
+    T_Backticked {} -> True
+    T_BraceExpansion {} -> True
+    T_Glob {} -> True
+    T_Extglob {} -> True
+    T_NormalWord _ l -> any willSplit l
+    _ -> False
+
+isGlob (T_Extglob {}) = True
+isGlob (T_Glob {}) = True
+isGlob (T_NormalWord _ l) = any isGlob l
+isGlob _ = False
+
+-- Is this shell word a constant?
+isConstant token =
+    case token of
+        T_NormalWord _ l   -> all isConstant l
+        T_DoubleQuoted _ l -> all isConstant l
+        T_SingleQuoted _ _ -> True
+        T_Literal _ _ -> True
+        _ -> False
+
+-- Is this an empty literal?
+isEmpty token =
+    case token of
+        T_NormalWord _ l   -> all isEmpty l
+        T_DoubleQuoted _ l -> all isEmpty l
+        T_SingleQuoted _ "" -> True
+        T_Literal _ "" -> True
+        _ -> False
+
+-- Quick&lazy oversimplification of commands, throwing away details
+-- and returning a list like  ["find", ".", "-name", "${VAR}*" ].
+oversimplify token =
+    case token of
+        (T_NormalWord _ l) -> [concat (concatMap oversimplify l)]
+        (T_DoubleQuoted _ l) -> [concat (concatMap oversimplify l)]
+        (T_SingleQuoted _ s) -> [s]
+        (T_DollarBraced _ _) -> ["${VAR}"]
+        (T_DollarArithmetic _ _) -> ["${VAR}"]
+        (T_DollarExpansion _ _) -> ["${VAR}"]
+        (T_Backticked _ _) -> ["${VAR}"]
+        (T_Glob _ s) -> [s]
+        (T_Pipeline _ _ [x]) -> oversimplify x
+        (T_Literal _ x) -> [x]
+        (T_SimpleCommand _ vars words) -> concatMap oversimplify words
+        (T_Redirecting _ _ foo) -> oversimplify foo
+        (T_DollarSingleQuoted _ s) -> [s]
+        (T_Annotation _ _ s) -> oversimplify s
+        -- Workaround for let "foo = bar" parsing
+        (TA_Sequence _ [TA_Expansion _ v]) -> concatMap oversimplify v
+        otherwise -> []
+
+
+-- Turn a SimpleCommand foo -avz --bar=baz into args "a", "v", "z", "bar",
+-- each in a tuple of (token, stringFlag).
+getFlagsUntil stopCondition (T_SimpleCommand _ _ (_:args)) =
+    let textArgs = takeWhile (not . stopCondition . snd) $ map (\x -> (x, concat $ oversimplify x)) args in
+        concatMap flag textArgs
+  where
+    flag (x, '-':'-':arg) = [ (x, takeWhile (/= '=') arg) ]
+    flag (x, '-':args) = map (\v -> (x, [v])) args
+    flag _ = []
+getFlagsUntil _ _ = error "Internal shellcheck error, please report! (getFlags on non-command)"
+
+-- Get all flags in a GNU way, up until --
+getAllFlags = getFlagsUntil (== "--")
+-- Get all flags in a BSD way, up until first non-flag argument
+getLeadingFlags = getFlagsUntil (not . ("-" `isPrefixOf`))
+
+
+-- Given a T_DollarBraced, return a simplified version of the string contents.
+bracedString (T_DollarBraced _ l) = concat $ oversimplify l
+bracedString _ = error "Internal shellcheck error, please report! (bracedString on non-variable)"
+
+-- Is this an expansion of multiple items of an array?
+isArrayExpansion t@(T_DollarBraced _ _) =
+    let string = bracedString t in
+        "@" `isPrefixOf` string ||
+            not ("#" `isPrefixOf` string) && "[@]" `isInfixOf` string
+isArrayExpansion _ = False
+
+-- Is it possible that this arg becomes multiple args?
+mayBecomeMultipleArgs t = willBecomeMultipleArgs t || f t
+  where
+    f t@(T_DollarBraced _ _) =
+        let string = bracedString t in
+            "!" `isPrefixOf` string
+    f (T_DoubleQuoted _ parts) = any f parts
+    f (T_NormalWord _ parts) = any f parts
+    f _ = False
+
+-- Is it certain that this word will becomes multiple words?
+willBecomeMultipleArgs t = willConcatInAssignment t || f t
+  where
+    f (T_Extglob {}) = True
+    f (T_Glob {}) = True
+    f (T_BraceExpansion {}) = True
+    f (T_DoubleQuoted _ parts) = any f parts
+    f (T_NormalWord _ parts) = any f parts
+    f _ = False
+
+-- This does token cause implicit concatenation in assignments?
+willConcatInAssignment token =
+    case token of
+        t@(T_DollarBraced {}) -> isArrayExpansion t
+        (T_DoubleQuoted _ parts) -> any willConcatInAssignment parts
+        (T_NormalWord _ parts) -> any willConcatInAssignment parts
+        _ -> False
+
+-- Maybe get the literal string corresponding to this token
+getLiteralString :: Token -> Maybe String
+getLiteralString = getLiteralStringExt (const Nothing)
+
+-- Definitely get a literal string, skipping over all non-literals
+onlyLiteralString :: Token -> String
+onlyLiteralString = fromJust . getLiteralStringExt (const $ return "")
+
+-- Maybe get a literal string, but only if it's an unquoted argument.
+getUnquotedLiteral (T_NormalWord _ list) =
+    liftM concat $ mapM str list
+  where
+    str (T_Literal _ s) = return s
+    str _ = Nothing
+getUnquotedLiteral _ = Nothing
+
+-- Maybe get the literal string of this token and any globs in it.
+getGlobOrLiteralString = getLiteralStringExt f
+  where
+    f (T_Glob _ str) = return str
+    f _ = Nothing
+
+-- Maybe get the literal value of a token, using a custom function
+-- to map unrecognized Tokens into strings.
+getLiteralStringExt :: (Token -> Maybe String) -> Token -> Maybe String
+getLiteralStringExt more = g
+  where
+    allInList = liftM concat . mapM g
+    g (T_DoubleQuoted _ l) = allInList l
+    g (T_DollarDoubleQuoted _ l) = allInList l
+    g (T_NormalWord _ l) = allInList l
+    g (TA_Expansion _ l) = allInList l
+    g (T_SingleQuoted _ s) = return s
+    g (T_Literal _ s) = return s
+    g x = more x
+
+-- Is this token a string literal?
+isLiteral t = isJust $ getLiteralString t
+
+
+-- Turn a NormalWord like foo="bar $baz" into a series of constituent elements like [foo=,bar ,$baz]
+getWordParts (T_NormalWord _ l)   = concatMap getWordParts l
+getWordParts (T_DoubleQuoted _ l) = l
+getWordParts other                = [other]
+
+-- Return a list of NormalWords that would result from brace expansion
+braceExpand (T_NormalWord id list) = take 1000 $ do
+    items <- mapM part list
+    return $ T_NormalWord id items
+  where
+    part (T_BraceExpansion id items) = do
+        item <- items
+        braceExpand item
+    part x = return x
+
+-- Maybe get the command name of a token representing a command
+getCommandName t =
+    case t of
+        T_Redirecting _ _ w -> getCommandName w
+        T_SimpleCommand _ _ (w:_) -> getLiteralString w
+        T_Annotation _ _ t -> getCommandName t
+        otherwise -> Nothing
+
+-- Get the basename of a token representing a command
+getCommandBasename = liftM basename . getCommandName
+  where
+    basename = reverse . takeWhile (/= '/') . reverse
+
+isAssignment t =
+    case t of
+        T_Redirecting _ _ w -> isAssignment w
+        T_SimpleCommand _ (w:_) [] -> True
+        T_Assignment {} -> True
+        T_Annotation _ _ w -> isAssignment w
+        otherwise -> False
+
+-- Get the list of commands from tokens that contain them, such as
+-- the body of while loops and if statements.
+getCommandSequences t =
+    case t of
+        T_Script _ _ cmds -> [cmds]
+        T_BraceGroup _ cmds -> [cmds]
+        T_Subshell _ cmds -> [cmds]
+        T_WhileExpression _ _ cmds -> [cmds]
+        T_UntilExpression _ _ cmds -> [cmds]
+        T_ForIn _ _ _ cmds -> [cmds]
+        T_ForArithmetic _ _ _ _ cmds -> [cmds]
+        T_IfExpression _ thens elses -> map snd thens ++ [elses]
+        otherwise -> []
+

ShellCheck/Analyzer.hs

@@ -0,0 +1,27 @@
+{-
+    Copyright 2012-2015 Vidar Holen
+
+    This file is part of ShellCheck.
+    http://www.vidarholen.net/contents/shellcheck
+
+    ShellCheck is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    ShellCheck is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+-}
+module ShellCheck.Analyzer (analyzeScript) where
+
+import ShellCheck.Interface
+import ShellCheck.Analytics
+
+-- TODO: Clean up the cruft this is layered on
+analyzeScript :: AnalysisSpec -> AnalysisResult
+analyzeScript = runAnalytics

ShellCheck/Checker.hs

@@ -0,0 +1,159 @@
+{-
+    Copyright 2012-2015 Vidar Holen
+
+    This file is part of ShellCheck.
+    http://www.vidarholen.net/contents/shellcheck
+
+    ShellCheck is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    ShellCheck is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+-}
+{-# LANGUAGE TemplateHaskell #-}
+module ShellCheck.Checker (checkScript, ShellCheck.Checker.runTests) where
+
+import ShellCheck.Interface
+import ShellCheck.Parser
+import ShellCheck.Analyzer
+
+import Data.Either
+import Data.Functor
+import Data.List
+import Data.Maybe
+import Data.Ord
+import Control.Monad.Identity
+import qualified Data.Map as Map
+import qualified System.IO
+import Prelude hiding (readFile)
+import Control.Monad
+
+import Test.QuickCheck.All
+
+tokenToPosition map (TokenComment id c) = fromMaybe fail $ do
+    position <- Map.lookup id map
+    return $ PositionedComment position c
+  where
+    fail = error "Internal shellcheck error: id doesn't exist. Please report!"
+
+checkScript :: Monad m => SystemInterface m -> CheckSpec -> m CheckResult
+checkScript sys spec = do
+    results <- checkScript (csScript spec)
+    return CheckResult {
+        crFilename = csFilename spec,
+        crComments = results
+    }
+  where
+    checkScript contents = do
+        result <- parseScript sys ParseSpec {
+            psFilename = csFilename spec,
+            psScript = contents
+        }
+        let parseMessages = prComments result
+        let analysisMessages =
+                fromMaybe [] $
+                    (arComments . analyzeScript . analysisSpec)
+                        <$> prRoot result
+        let translator = tokenToPosition (prTokenPositions result)
+        return . nub . sortMessages . filter shouldInclude $
+            (parseMessages ++ map translator analysisMessages)
+
+    shouldInclude (PositionedComment _ (Comment _ code _)) =
+        code `notElem` csExcludedWarnings spec
+
+    sortMessages = sortBy (comparing order)
+    order (PositionedComment pos (Comment severity code message)) =
+        (posFile pos, posLine pos, posColumn pos, severity, code, message)
+    getPosition (PositionedComment pos _) = pos
+
+    analysisSpec root =
+        AnalysisSpec {
+            asScript = root,
+            asShellType = csShellTypeOverride spec,
+            asExecutionMode = Executed
+         }
+
+getErrors sys spec =
+    sort . map getCode . crComments $
+        runIdentity (checkScript sys spec)
+  where
+    getCode (PositionedComment _ (Comment _ code _)) = code
+
+check = checkWithIncludes []
+
+checkWithIncludes includes src =
+    getErrors
+        (mockedSystemInterface includes)
+        emptyCheckSpec {
+            csScript = src,
+            csExcludedWarnings = [2148]
+        }
+
+prop_findsParseIssue = check "echo \"$12\"" == [1037]
+
+prop_commentDisablesParseIssue1 =
+    null $ check "#shellcheck disable=SC1037\necho \"$12\""
+prop_commentDisablesParseIssue2 =
+    null $ check "#shellcheck disable=SC1037\n#lol\necho \"$12\""
+
+prop_findsAnalysisIssue =
+    check "echo $1" == [2086]
+prop_commentDisablesAnalysisIssue1 =
+    null $ check "#shellcheck disable=SC2086\necho $1"
+prop_commentDisablesAnalysisIssue2 =
+    null $ check "#shellcheck disable=SC2086\n#lol\necho $1"
+
+prop_optionDisablesIssue1 =
+    null $ getErrors
+                (mockedSystemInterface [])
+                emptyCheckSpec {
+                    csScript = "echo $1",
+                    csExcludedWarnings = [2148, 2086]
+                }
+
+prop_optionDisablesIssue2 =
+    null $ getErrors
+                (mockedSystemInterface [])
+                emptyCheckSpec {
+                    csScript = "echo \"$10\"",
+                    csExcludedWarnings = [2148, 1037]
+                }
+
+prop_canParseDevNull =
+    [] == check "source /dev/null"
+
+prop_failsWhenNotSourcing =
+    [1091, 2154] == check "source lol; echo \"$bar\""
+
+prop_worksWhenSourcing =
+    null $ checkWithIncludes [("lib", "bar=1")] "source lib; echo \"$bar\""
+
+prop_worksWhenDotting =
+    null $ checkWithIncludes [("lib", "bar=1")] ". lib; echo \"$bar\""
+
+prop_noInfiniteSourcing =
+    [] == checkWithIncludes  [("lib", "source lib")] "source lib"
+
+prop_canSourceBadSyntax =
+    [1094, 2086] == checkWithIncludes [("lib", "for f; do")] "source lib; echo $1"
+
+prop_cantSourceDynamic =
+    [1090] == checkWithIncludes [("lib", "")] ". \"$1\""
+
+prop_canSourceDynamicWhenRedirected =
+    null $ checkWithIncludes [("lib", "")] "#shellcheck source=lib\n. \"$1\""
+
+prop_sourceDirectiveDoesntFollowFile =
+    null $ checkWithIncludes
+                [("foo", "source bar"), ("bar", "baz=3")]
+                "#shellcheck source=foo\n. \"$1\"; echo \"$baz\""
+
+return []
+runTests = $quickCheckAll

ShellCheck/Data.hs

@@ -1,5 +1,6 @@
 module ShellCheck.Data where
 
+import ShellCheck.Interface
 import Data.Version (showVersion)
 import Paths_ShellCheck (version)
 
@@ -73,3 +74,15 @@ sampleWords = [
     "tango", "uniform", "victor", "whiskey", "xray", "yankee",
     "zulu"
   ]
+
+shellForExecutable :: String -> Maybe Shell
+shellForExecutable "sh" = return Sh
+shellForExecutable "ash" = return Sh
+shellForExecutable "dash" = return Sh
+
+shellForExecutable "ksh" = return Ksh
+shellForExecutable "ksh88" = return Ksh
+shellForExecutable "ksh93" = return Ksh
+
+shellForExecutable "bash" = return Bash
+shellForExecutable _ = Nothing

ShellCheck/Formatter/CheckStyle.hs

@@ -0,0 +1,82 @@
+{-
+    Copyright 2012-2015 Vidar Holen
+
+    This file is part of ShellCheck.
+    http://www.vidarholen.net/contents/shellcheck
+
+    ShellCheck is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    ShellCheck is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+-}
+module ShellCheck.Formatter.CheckStyle (format) where
+
+import ShellCheck.Interface
+import ShellCheck.Formatter.Format
+
+import Data.Char
+import Data.List
+import GHC.Exts
+import System.IO
+
+format :: IO Formatter
+format = return Formatter {
+    header = do
+        putStrLn "<?xml version='1.0' encoding='UTF-8'?>"
+        putStrLn "<checkstyle version='4.3'>",
+
+    onFailure = outputError,
+    onResult = outputResult,
+
+    footer = putStrLn "</checkstyle>"
+}
+
+outputResult result contents = do
+    let comments = makeNonVirtual (crComments result) contents
+    putStrLn . formatFile (crFilename result) $ comments
+
+formatFile name comments = concat [
+    "<file ", attr "name" name, ">\n",
+        concatMap formatComment comments,
+    "</file>"
+    ]
+
+formatComment c = concat [
+    "<error ",
+    attr "line" $ show . lineNo $ c,
+    attr "column" $ show . colNo $ c,
+    attr "severity" . severity $ severityText c,
+    attr "message" $ messageText c,
+    attr "source" $ "ShellCheck.SC" ++ show (codeNo c),
+    "/>\n"
+    ]
+
+outputError file error = putStrLn $ concat [
+    "<file ", attr "name" file, ">\n",
+    "<error ",
+        attr "line" "1",
+        attr "column" "1",
+        attr "severity" "error",
+        attr "message" error,
+        attr "source" "ShellCheck",
+    "/>\n",
+    "</file>"
+    ]
+
+
+attr s v = concat [ s, "='", escape v, "' " ]
+escape = concatMap escape'
+escape' c = if isOk c then [c] else "&#" ++ show (ord c) ++ ";"
+isOk x = any ($x) [isAsciiUpper, isAsciiLower, isDigit, (`elem` " ./")]
+
+severity "error" = "error"
+severity "warning" = "warning"
+severity _ = "info"

ShellCheck/Formatter/Format.hs

@@ -0,0 +1,61 @@
+{-
+    Copyright 2012-2015 Vidar Holen
+
+    This file is part of ShellCheck.
+    http://www.vidarholen.net/contents/shellcheck
+
+    ShellCheck is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    ShellCheck is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+-}
+module ShellCheck.Formatter.Format where
+
+import ShellCheck.Data
+import ShellCheck.Interface
+
+-- A formatter that carries along an arbitrary piece of data
+data Formatter = Formatter {
+    header ::  IO (),
+    onResult :: CheckResult -> String -> IO (),
+    onFailure :: FilePath -> ErrorMessage -> IO (),
+    footer :: IO ()
+}
+
+lineNo (PositionedComment pos _) = posLine pos
+colNo  (PositionedComment pos _) = posColumn pos
+codeNo (PositionedComment _ (Comment _ code _)) = code
+messageText (PositionedComment _ (Comment _ _ t)) = t
+
+severityText :: PositionedComment -> String
+severityText (PositionedComment _ (Comment c _ _)) =
+    case c of
+        ErrorC   -> "error"
+        WarningC -> "warning"
+        InfoC    -> "info"
+        StyleC   -> "style"
+
+-- Realign comments from a tabstop of 8 to 1
+makeNonVirtual comments contents =
+    map fix comments
+  where
+    ls = lines contents
+    fix c@(PositionedComment pos comment) = PositionedComment pos {
+        posColumn =
+            if lineNo c > 0 && lineNo c <= fromIntegral (length ls)
+            then real (ls !! fromIntegral (lineNo c - 1)) 0 0 (colNo c)
+            else colNo c
+    } comment
+    real _ r v target | target <= v = r
+    real [] r v _ = r -- should never happen
+    real ('\t':rest) r v target =
+        real rest (r+1) (v + 8 - (v `mod` 8)) target
+    real (_:rest) r v target = real rest (r+1) (v+1) target

ShellCheck/Formatter/GCC.hs

@@ -0,0 +1,54 @@
+{-
+    Copyright 2012-2015 Vidar Holen
+
+    This file is part of ShellCheck.
+    http://www.vidarholen.net/contents/shellcheck
+
+    ShellCheck is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    ShellCheck is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+-}
+module ShellCheck.Formatter.GCC (format) where
+
+import ShellCheck.Interface
+import ShellCheck.Formatter.Format
+
+import Data.List
+import GHC.Exts
+import System.IO
+
+format :: IO Formatter
+format = return Formatter {
+    header = return (),
+    footer = return (),
+    onFailure = outputError,
+    onResult = outputResult
+}
+
+outputError file error = hPutStrLn stderr $ file ++ ": " ++ error
+
+outputResult result contents = do
+    let comments = makeNonVirtual (crComments result) contents
+    mapM_ (putStrLn . formatComment (crFilename result)) comments
+
+formatComment filename c = concat [
+    filename, ":",
+    show $ lineNo c, ":",
+    show $ colNo c, ": ",
+    case severityText c of
+        "error" -> "error"
+        "warning" -> "warning"
+        _ -> "note",
+    ": ",
+    concat . lines $ messageText c,
+    " [SC", show $ codeNo c, "]"
+  ]

ShellCheck/Formatter/JSON.hs

@@ -0,0 +1,58 @@
+{-
+    Copyright 2012-2015 Vidar Holen
+
+    This file is part of ShellCheck.
+    http://www.vidarholen.net/contents/shellcheck
+
+    ShellCheck is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    ShellCheck is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+-}
+module ShellCheck.Formatter.JSON (format) where
+
+import ShellCheck.Interface
+import ShellCheck.Formatter.Format
+
+import Data.IORef
+import GHC.Exts
+import System.IO
+import Text.JSON
+
+format = do
+    ref <- newIORef []
+    return Formatter {
+        header = return (),
+        onResult = collectResult ref,
+        onFailure = outputError,
+        footer = finish ref
+    }
+
+instance JSON (PositionedComment) where
+  showJSON comment@(PositionedComment pos (Comment level code string)) = makeObj [
+      ("file", showJSON $ posFile pos),
+      ("line", showJSON $ posLine pos),
+      ("column", showJSON $ posColumn pos),
+      ("level", showJSON $ severityText comment),
+      ("code", showJSON code),
+      ("message", showJSON string)
+      ]
+
+  readJSON = undefined
+
+outputError file msg = hPutStrLn stderr $ file ++ ": " ++ msg
+collectResult ref result _ =
+    modifyIORef ref (\x -> crComments result ++ x)
+
+finish ref = do
+    list <- readIORef ref
+    putStrLn $ encodeStrict list
+

ShellCheck/Formatter/TTY.hs

@@ -0,0 +1,86 @@
+{-
+    Copyright 2012-2015 Vidar Holen
+
+    This file is part of ShellCheck.
+    http://www.vidarholen.net/contents/shellcheck
+
+    ShellCheck is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    ShellCheck is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+-}
+module ShellCheck.Formatter.TTY (format) where
+
+import ShellCheck.Interface
+import ShellCheck.Formatter.Format
+
+import Data.List
+import GHC.Exts
+import System.Info
+import System.IO
+
+format :: IO Formatter
+format = return Formatter {
+    header = return (),
+    footer = return (),
+    onFailure = outputError,
+    onResult = outputResult
+}
+
+colorForLevel level = 
+    case level of
+        "error"   -> 31 -- red
+        "warning" -> 33 -- yellow
+        "info"    -> 32 -- green
+        "style"   -> 32 -- green
+        "message" -> 1 -- bold
+        "source"  -> 0 -- none
+        otherwise -> 0 -- none
+    
+outputError file error = do
+    color <- getColorFunc
+    hPutStrLn stderr $ color "error" $ file ++ ": " ++ error
+
+outputResult result contents = do
+    color <- getColorFunc
+    let comments = crComments result
+    let fileLines = lines contents
+    let lineCount = fromIntegral $ length fileLines
+    let groups = groupWith lineNo comments
+    mapM_ (\x -> do
+        let lineNum = lineNo (head x)
+        let line = if lineNum < 1 || lineNum > lineCount
+                        then ""
+                        else fileLines !! fromIntegral (lineNum - 1)
+        putStrLn ""
+        putStrLn $ color "message" $
+           "In " ++ crFilename result ++" line " ++ show lineNum ++ ":"
+        putStrLn (color "source" line)
+        mapM_ (\c -> putStrLn (color (severityText c) $ cuteIndent c)) x
+        putStrLn ""
+      ) groups
+
+cuteIndent :: PositionedComment -> String
+cuteIndent comment =
+    replicate (fromIntegral $ colNo comment - 1) ' ' ++
+        "^-- " ++ code (codeNo comment) ++ ": " ++ messageText comment
+
+code code = "SC" ++ show code
+
+getColorFunc = do
+    term <- hIsTerminalDevice stdout
+    let windows = "mingw" `isPrefixOf` os
+    return $ if term && not windows then colorComment else const id
+  where
+    colorComment level comment =
+        ansi (colorForLevel level) ++ comment ++ clear
+    clear = ansi 0
+    ansi n = "\x1B[" ++ show n ++ "m"

ShellCheck/Interface.hs

@@ -0,0 +1,103 @@
+{-
+    Copyright 2012-2015 Vidar Holen
+
+    This file is part of ShellCheck.
+    http://www.vidarholen.net/contents/shellcheck
+
+    ShellCheck is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    ShellCheck is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+-}
+module ShellCheck.Interface where
+
+import ShellCheck.AST
+import Control.Monad.Identity
+import qualified Data.Map as Map
+
+
+data SystemInterface m = SystemInterface {
+    -- Read a file by filename, or return an error
+    siReadFile :: String -> m (Either ErrorMessage String)
+}
+
+-- ShellCheck input and output
+data CheckSpec = CheckSpec {
+    csFilename :: String,
+    csScript :: String,
+    csExcludedWarnings :: [Integer],
+    csShellTypeOverride :: Maybe Shell
+} deriving (Show, Eq)
+
+data CheckResult = CheckResult {
+    crFilename :: String,
+    crComments :: [PositionedComment]
+} deriving (Show, Eq)
+
+emptyCheckSpec = CheckSpec {
+    csFilename = "",
+    csScript = "",
+    csExcludedWarnings = [],
+    csShellTypeOverride = Nothing
+}
+
+-- Parser input and output
+data ParseSpec = ParseSpec {
+    psFilename :: String,
+    psScript :: String
+} deriving (Show, Eq)
+
+data ParseResult = ParseResult {
+    prComments :: [PositionedComment],
+    prTokenPositions :: Map.Map Id Position,
+    prRoot :: Maybe Token
+} deriving (Show, Eq)
+
+-- Analyzer input and output
+data AnalysisSpec = AnalysisSpec {
+    asScript :: Token,
+    asShellType :: Maybe Shell,
+    asExecutionMode :: ExecutionMode
+}
+
+data AnalysisResult = AnalysisResult {
+    arComments :: [TokenComment]
+}
+
+-- Supporting data types
+data Shell = Ksh | Sh | Bash deriving (Show, Eq)
+data ExecutionMode = Executed | Sourced deriving (Show, Eq)
+
+type ErrorMessage = String
+type Code = Integer
+
+data Severity = ErrorC | WarningC | InfoC | StyleC deriving (Show, Eq, Ord)
+data Position = Position {
+    posFile :: String,    -- Filename
+    posLine :: Integer,   -- 1 based source line
+    posColumn :: Integer  -- 1 based source column, where tabs are 8
+} deriving (Show, Eq)
+
+data Comment = Comment Severity Code String deriving (Show, Eq)
+data PositionedComment = PositionedComment Position Comment deriving (Show, Eq)
+data TokenComment = TokenComment Id Comment deriving (Show, Eq)
+
+-- For testing
+mockedSystemInterface :: [(String, String)] -> SystemInterface Identity
+mockedSystemInterface files = SystemInterface {
+    siReadFile = rf
+}
+  where
+    rf file =
+        case filter ((== file) . fst) files of
+            [] -> return $ Left "File not included in mock."
+            [(_, contents)] -> return $ Right contents
+

ShellCheck/Options.hs

@@ -1,14 +0,0 @@
-module ShellCheck.Options where
-
-data Shell = Ksh | Sh | Bash
-    deriving (Show, Eq)
-
-data AnalysisOptions = AnalysisOptions {
-    optionShellType :: Maybe Shell,
-    optionExcludes :: [Integer]
-}
-
-defaultAnalysisOptions = AnalysisOptions {
-    optionShellType = Nothing,
-    optionExcludes = []
-}

ShellCheck/Parser.hs

@@ -1,4 +1,6 @@
 {-
+    Copyright 2012-2015 Vidar Holen
+
     This file is part of ShellCheck.
     http://www.vidarholen.net/contents/shellcheck
 
@@ -16,26 +18,39 @@
     along with this program.  If not, see <http://www.gnu.org/licenses/>.
 -}
 {-# LANGUAGE NoMonomorphismRestriction, TemplateHaskell, FlexibleContexts #-}
-module ShellCheck.Parser (Note(..), Severity(..), parseShell, ParseResult(..), ParseNote(..), sortNotes, noteToParseNote, runTests, readScript) where
+module ShellCheck.Parser (parseScript, runTests) where
 
 import ShellCheck.AST
+import ShellCheck.ASTLib
 import ShellCheck.Data
-import ShellCheck.Options
+import ShellCheck.Interface
-import Text.Parsec
+
-import Debug.Trace
+import Control.Applicative ((<*))
 import Control.Monad
-import Control.Arrow (first)
+import Control.Monad.Identity
+import Control.Monad.Trans
 import Data.Char
+import Data.Functor
 import Data.List (isPrefixOf, isInfixOf, isSuffixOf, partition, sortBy, intercalate, nub)
-import qualified Data.Map as Map
-import qualified Control.Monad.State as Ms
 import Data.Maybe
+import Data.Monoid
+import Debug.Trace
+import GHC.Exts (sortWith)
 import Prelude hiding (readList)
 import System.IO
+import Text.Parsec hiding (runParser, (<?>))
 import Text.Parsec.Error
-import GHC.Exts (sortWith)
+import Text.Parsec.Pos
+import qualified Control.Monad.Reader as Mr
+import qualified Control.Monad.State as Ms
+import qualified Data.Map as Map
+
 import Test.QuickCheck.All (quickCheckAll)
 
+type SCBase m = Mr.ReaderT (SystemInterface m) (Ms.StateT SystemState m)
+type SCParser m v = ParsecT String UserState (SCBase m) v
+
+backslash :: Monad m => SCParser m Char
 backslash = char '\\'
 linefeed = optional carriageReturn >> char '\n'
 singleQuote = char '\'' <|> unicodeSingleQuote
@@ -44,14 +59,14 @@ variableStart = upper <|> lower <|> oneOf "_"
 variableChars = upper <|> lower <|> digit <|> oneOf "_"
 functionChars = variableChars <|> oneOf ":+-.?"
 specialVariable = oneOf "@*#?-$!"
-tokenDelimiter = oneOf "&|;<> \t\n\r" <|> nbsp
+tokenDelimiter = oneOf "&|;<> \t\n\r" <|> almostSpace
 quotableChars = "|&;<>()\\ '\t\n\r\xA0" ++ doubleQuotableChars
-quotable = nbsp <|> unicodeDoubleQuote <|> oneOf quotableChars
+quotable = almostSpace <|> unicodeDoubleQuote <|> oneOf quotableChars
 bracedQuotable = oneOf "}\"$`'"
 doubleQuotableChars = "\"$`" ++ unicodeDoubleQuoteChars
 doubleQuotable = unicodeDoubleQuote <|> oneOf doubleQuotableChars
-whitespace = oneOf " \t\n" <|> carriageReturn <|> nbsp
+whitespace = oneOf " \t\n" <|> carriageReturn <|> almostSpace
-linewhitespace = oneOf " \t" <|> nbsp
+linewhitespace = oneOf " \t" <|> almostSpace
 
 suspectCharAfterQuotes = variableChars <|> char '%'
 
@@ -103,17 +118,36 @@ carriageReturn = do
     parseNote ErrorC 1017 "Literal carriage return. Run script through tr -d '\\r' ."
     char '\r'
 
-nbsp = do
+almostSpace =
-    parseNote ErrorC 1018 "This is a &nbsp;. Delete it and retype as space."
+    choice [
-    char '\xA0'
+        check '\xA0' "unicode non-breaking space",
+        check '\x200B' "unicode zerowidth space"
+    ]
+  where
+    check c name = do
+        parseNote ErrorC 1018 $ "This is a " ++ name ++ ". Delete and retype it."
+        char c
         return ' '
 
 --------- Message/position annotation on top of user state
 data Note = Note Id Severity Code String deriving (Show, Eq)
 data ParseNote = ParseNote SourcePos Severity Code String deriving (Show, Eq)
-data Severity = ErrorC | WarningC | InfoC | StyleC deriving (Show, Eq, Ord)
+data Context =
-data Context = ContextName SourcePos String | ContextAnnotation [Annotation] deriving (Show)
+        ContextName SourcePos String
-type Code = Integer
+        | ContextAnnotation [Annotation]
+        | ContextSource String
+    deriving (Show)
+
+data UserState = UserState {
+    lastId :: Id,
+    positionMap :: Map.Map Id SourcePos,
+    parseNotes :: [ParseNote]
+}
+initialUserState = UserState {
+    lastId = Id $ -1,
+    positionMap = Map.empty,
+    parseNotes = []
+}
 
 codeForParseNote (ParseNote _ _ code _) = code
 noteToParseNote map (Note id severity code message) =
@@ -121,17 +155,17 @@ noteToParseNote map (Note id severity code message) =
     where
         pos = fromJust $ Map.lookup id map
 
-initialState = (Id $ -1, Map.empty, [])
 
-getLastId = do
+getLastId = lastId <$> getState
-    (id, _, _) <- getState
-    return id
 
 getNextIdAt sourcepos = do
-    (id, map, notes) <- getState
+    state <- getState
-    let newId = incId id
+    let newId = incId (lastId state)
-    let newMap = Map.insert newId sourcepos map
+    let newMap = Map.insert newId sourcepos (positionMap state)
-    putState (newId, newMap, notes)
+    putState $ state {
+        lastId = newId,
+        positionMap = newMap
+    }
     return newId
   where incId (Id n) = Id $ n+1
 
@@ -139,23 +173,16 @@ getNextId = do
     pos <- getPosition
     getNextIdAt pos
 
-modifyMap f = do
+getMap = positionMap <$> getState
-    (id, map, parsenotes) <- getState
+getParseNotes = parseNotes <$> getState
-    putState (id, f map, parsenotes)
-
-getMap = do
-    (_, map, _) <- getState
-    return map
-
-getParseNotes = do
-    (_, _, notes) <- getState
-    return notes
 
 addParseNote n = do
     irrelevant <- shouldIgnoreCode (codeForParseNote n)
     unless irrelevant $ do
-        (a, b, notes) <- getState
+        state <- getState
-        putState (a, b, n:notes)
+        putState $ state {
+            parseNotes = n : parseNotes state
+        }
 
 shouldIgnoreCode code = do
     context <- getCurrentContexts
@@ -163,20 +190,57 @@ shouldIgnoreCode code = do
   where
     disabling (ContextAnnotation list) =
         any disabling' list
+    disabling (ContextSource _) = True -- Don't add messages for sourced files
     disabling _ = False
     disabling' (DisableComment n) = code == n
+    disabling' _ = False
+
+shouldFollow file = do
+    context <- getCurrentContexts
+    if any isThisFile context
+      then return False
+      else
+        if length (filter isSource context) >= 100
+          then do
+            parseProblem ErrorC 1092 "Stopping at 100 'source' frames :O"
+            return False
+          else
+            return True
+  where
+    isSource (ContextSource _) = True
+    isSource _ = False
+    isThisFile (ContextSource name) | name == file = True
+    isThisFile _= False
+
+getSourceOverride = do
+    context <- getCurrentContexts
+    return . msum . map findFile $ takeWhile isSameFile context
+  where
+    isSameFile (ContextSource _) = False
+    isSameFile _ = True
+
+    findFile (ContextAnnotation list) = msum $ map getFile list
+    findFile _ = Nothing
+    getFile (SourceOverride str) = Just str
+    getFile _ = Nothing
 
 -- Store potential parse problems outside of parsec
+
+data SystemState = SystemState {
+    contextStack :: [Context],
+    parseProblems :: [ParseNote]
+}
+initialSystemState = SystemState {
+    contextStack = [],
+    parseProblems = []
+}
+
 parseProblem level code msg = do
     pos <- getPosition
     parseProblemAt pos level code msg
 
-setCurrentContexts c =
+setCurrentContexts c = Ms.modify (\state -> state { contextStack = c })
-    Ms.modify (\(list, _) -> (list, c))
+getCurrentContexts = contextStack <$> Ms.get
-
-getCurrentContexts = do
-    (_, context) <- Ms.get
-    return context
 
 popContext = do
     v <- getCurrentContexts
@@ -195,7 +259,11 @@ pushContext c = do
 parseProblemAt pos level code msg = do
     irrelevant <- shouldIgnoreCode code
     unless irrelevant $
-        Ms.modify (first ((:) (ParseNote pos level code msg)))
+        Ms.modify (\state -> state {
+            parseProblems = note:parseProblems state
+        })
+  where
+    note = ParseNote pos level code msg
 
 -- Store non-parse problems inside
 
@@ -212,9 +280,9 @@ thenSkip main follow = do
     return r
 
 unexpecting s p = try $
-    (try p >> unexpected s) <|> return ()
+    (try p >> fail ("Unexpected " ++ s)) <|> return ()
 
-notFollowedBy2 = unexpecting "keyword/token"
+notFollowedBy2 = unexpecting ""
 
 disregard = void
 
@@ -315,7 +383,7 @@ readConditionContents single =
     readCondUnaryExp = do
       op <- readCondUnaryOp
       pos <- getPosition
-      (readCondWord >>= return . op) `orFail` do
+      liftM op readCondWord `orFail` do
           parseProblemAt pos ErrorC 1019 "Expected this to be an argument to the unary condition."
           return "Expected an argument for the unary operator"
 
@@ -484,10 +552,16 @@ prop_a15= isOk readArithmeticContents "foo[`echo foo | sed s/foo/4/g` * 3] + 4"
 prop_a16= isOk readArithmeticContents "$foo$bar"
 prop_a17= isOk readArithmeticContents "i<(0+(1+1))"
 prop_a18= isOk readArithmeticContents "a?b:c"
+prop_a19= isOk readArithmeticContents "\\\n3 +\\\n  2"
+prop_a20= isOk readArithmeticContents "a ? b ? c : d : e"
+prop_a21= isOk readArithmeticContents "a ? b : c ? d : e"
+prop_a22= isOk readArithmeticContents "!!a"
 readArithmeticContents =
     readSequence
   where
-    spacing = many whitespace
+    spacing =
+        let lf = try (string "\\\n") >> return '\n'
+        in many (whitespace <|> lf)
 
     splitBy x ops = chainl1 x (readBinary ops)
     readBinary ops = readComboOp ops TA_Binary
@@ -547,16 +621,15 @@ readArithmeticContents =
 
     readAssignment = readTrinary `splitBy` ["=", "*=", "/=", "%=", "+=", "-=", "<<=", ">>=", "&=", "^=", "|="]
     readTrinary = do
-        let part = readLogicalOr
+        x <- readLogicalOr
-        x <- part
         do
             id <- getNextId
             string "?"
             spacing
-            y <- part
+            y <- readTrinary
             string ":"
             spacing
-            z <- part
+            z <- readTrinary
             return $ TA_Trinary id x y z
          <|>
           return x
@@ -578,7 +651,7 @@ readArithmeticContents =
         id <- getNextId
         op <- oneOf "!~"
         spacing
-        x <- readAnySigned
+        x <- readAnyNegated
         return $ TA_Unary id [op] x
 
     readAnySigned = readSigned <|> readAnycremented
@@ -652,7 +725,7 @@ readCondition = called "test expression" $ do
     condition <- readConditionContents single
 
     cpos <- getPosition
-    close <- try (string "]]") <|> string "]"
+    close <- try (string "]]") <|> string "]" <|> fail "Expected test to end here"
     when (open == "[[" && close /= "]]") $ parseProblemAt cpos ErrorC 1033 "Did you mean ]] ?"
     when (open == "[" && close /= "]" ) $ parseProblemAt opos ErrorC 1034 "Did you mean [[ ?"
     spacing
@@ -666,10 +739,11 @@ readAnnotationPrefix = do
 
 prop_readAnnotation1 = isOk readAnnotation "# shellcheck disable=1234,5678\n"
 prop_readAnnotation2 = isOk readAnnotation "# shellcheck disable=SC1234 disable=SC5678\n"
+prop_readAnnotation3 = isOk readAnnotation "# shellcheck disable=SC1234 source=/dev/null disable=SC5678\n"
 readAnnotation = called "shellcheck annotation" $ do
     try readAnnotationPrefix
     many1 linewhitespace
-    values <- many1 readDisable
+    values <- many1 (readDisable <|> readSourceOverride)
     linefeed
     many linewhitespace
     return $ concat values
@@ -681,6 +755,11 @@ readAnnotation = called "shellcheck annotation" $ do
             optional $ string "SC"
             int <- many1 digit
             return $ DisableComment (read int)
+
+    readSourceOverride = forKey "source" $ do
+        filename <- many1 $ noneOf " \n"
+        return [SourceOverride filename]
+
     forKey s p = do
         try $ string s
         char '='
@@ -796,7 +875,7 @@ readSingleQuoted = called "single quoted string" $ do
     s <- readSingleQuotedPart `reluctantlyTill` singleQuote
     let string = concat s
     endPos <- getPosition
-    singleQuote <?> "end of single quoted string"
+    singleQuote <|> fail "Expected end of single quoted string"
 
     optional $ do
         c <- try . lookAhead $ suspectCharAfterQuotes <|> oneOf "'"
@@ -871,18 +950,32 @@ subParse pos parser input = do
     setPosition lastPosition
     return result
 
+inSeparateContext parser = do
+    context <- Ms.get
+    success context <|> failure context
+  where
+    success c = do
+        res <- try parser
+        Ms.put c
+        return res
+    failure c = do
+        Ms.put c
+        fail ""
+
 prop_readDoubleQuoted = isOk readDoubleQuoted "\"Hello $FOO\""
 prop_readDoubleQuoted2 = isOk readDoubleQuoted "\"$'\""
 prop_readDoubleQuoted3 = isWarning readDoubleQuoted "\x201Chello\x201D"
 prop_readDoubleQuoted4 = isWarning readSimpleCommand "\"foo\nbar\"foo"
 prop_readDoubleQuoted5 = isOk readSimpleCommand "lol \"foo\nbar\" etc"
+prop_readDoubleQuoted6 = isOk readSimpleCommand "echo \"${ ls; }\""
+prop_readDoubleQuoted7 = isOk readSimpleCommand "echo \"${ ls;}bar\""
 readDoubleQuoted = called "double quoted string" $ do
     id <- getNextId
     startPos <- getPosition
     doubleQuote
     x <- many doubleQuotedPart
     endPos <- getPosition
-    doubleQuote <?> "end of double quoted string"
+    doubleQuote <|> fail "Expected end of double quoted string"
     optional $ do
         try . lookAhead $ suspectCharAfterQuotes <|> oneOf "$\""
         when (any hasLineFeed x && not (startsWithLineFeed x)) $
@@ -1081,7 +1174,7 @@ readBraced = try braceExpansion
 
 readNormalDollar = readDollarExpression <|> readDollarDoubleQuote <|> readDollarSingleQuote <|> readDollarLonely
 readDoubleQuotedDollar = readDollarExpression <|> readDollarLonely
-readDollarExpression = readDollarArithmetic <|> readDollarBracket <|> readDollarBraced <|> readDollarExpansion <|> readDollarVariable
+readDollarExpression = readDollarArithmetic <|> readDollarBracket <|> readDollarBraceCommandExpansion <|> readDollarBraced <|> readDollarExpansion <|> readDollarVariable
 
 prop_readDollarSingleQuote = isOk readDollarSingleQuote "$'foo\\\'lol'"
 readDollarSingleQuote = called "$'..' expression" $ do
@@ -1098,7 +1191,7 @@ readDollarDoubleQuote = do
     char '$'
     doubleQuote
     x <- many doubleQuotedPart
-    doubleQuote <?> "end of translated double quoted string"
+    doubleQuote <|> fail "Expected end of translated double quoted string"
     return $ T_DollarDoubleQuoted id x
 
 prop_readDollarArithmetic = isOk readDollarArithmetic "$(( 3 * 4 +5))"
@@ -1125,6 +1218,18 @@ readArithmeticExpression = called "((..)) command" $ do
     string "))"
     return (T_Arithmetic id c)
 
+prop_readDollarBraceCommandExpansion1 = isOk readDollarBraceCommandExpansion "${ ls; }"
+prop_readDollarBraceCommandExpansion2 = isOk readDollarBraceCommandExpansion "${\nls\n}"
+readDollarBraceCommandExpansion = called "ksh ${ ..; } command expansion" $ do
+    id <- getNextId
+    try $ do
+        string "${"
+        whitespace
+    allspacing
+    term <- readTerm
+    char '}' <|> fail "Expected } to end the ksh ${ ..; } command expansion"
+    return $ T_DollarBraceCommandExpansion id term
+
 prop_readDollarBraced1 = isOk readDollarBraced "${foo//bar/baz}"
 prop_readDollarBraced2 = isOk readDollarBraced "${foo/'{cow}'}"
 prop_readDollarBraced3 = isOk readDollarBraced "${foo%%$(echo cow\\})}"
@@ -1143,7 +1248,7 @@ readDollarExpansion = called "command expansion" $ do
     id <- getNextId
     try (string "$(")
     cmds <- readCompoundListOrEmpty
-    char ')' <?> "end of $(..) expression"
+    char ')' <|> fail "Expected end of $(..) expression"
     return $ T_DollarExpansion id cmds
 
 prop_readDollarVariable = isOk readDollarVariable "$@"
@@ -1202,6 +1307,7 @@ prop_readHereDoc4 = isOk readHereDoc "<< foo\n`\nfoo"
 prop_readHereDoc5 = isOk readHereDoc "<<- !foo\nbar\n!foo"
 prop_readHereDoc6 = isOk readHereDoc "<< foo\\ bar\ncow\nfoo bar"
 prop_readHereDoc7 = isOk readHereDoc "<< foo\n\\$(f ())\nfoo"
+prop_readHereDoc8 = isOk readHereDoc "<<foo>>bar\netc\nfoo"
 readHereDoc = called "here document" $ do
     fid <- getNextId
     pos <- getPosition
@@ -1244,7 +1350,7 @@ readHereDoc = called "here document" $ do
         liftM concat $ many1 (escaped <|> quoted <|> normal)
       where
         quoted = liftM stripLiteral readDoubleQuotedLiteral <|> readSingleQuotedLiteral
-        normal = anyChar `reluctantlyTill1` (whitespace <|> oneOf ";&)'\"\\")
+        normal = anyChar `reluctantlyTill1` (whitespace <|> oneOf "<>;&)'\"\\")
         escaped = do -- surely the user must be doing something wrong at this point
             char '\\'
             c <- anyChar
@@ -1295,6 +1401,13 @@ readIoFile = called "redirection" $ do
     file <- readFilename
     return $ T_FdRedirect id "" $ T_IoFile id op file
 
+readIoVariable = try $ do
+    char '{'
+    x <- readVariableName
+    char '}'
+    lookAhead readIoFileOp
+    return $ "{" ++ x ++ "}"
+
 readIoNumber = try $ do
     x <- many1 digit <|> string "&"
     lookAhead readIoFileOp
@@ -1304,9 +1417,11 @@ prop_readIoNumberRedirect = isOk readIoNumberRedirect "3>&2"
 prop_readIoNumberRedirect2 = isOk readIoNumberRedirect "2> lol"
 prop_readIoNumberRedirect3 = isOk readIoNumberRedirect "4>&-"
 prop_readIoNumberRedirect4 = isOk readIoNumberRedirect "&> lol"
+prop_readIoNumberRedirect5 = isOk readIoNumberRedirect "{foo}>&2"
+prop_readIoNumberRedirect6 = isOk readIoNumberRedirect "{foo}<&-"
 readIoNumberRedirect = do
     id <- getNextId
-    n <- readIoNumber
+    n <- readIoVariable <|> readIoNumber
     op <- readHereString <|> readHereDoc <|> readIoFile
     let actualOp = case op of T_FdRedirect _ "" x -> x
     spacing
@@ -1374,7 +1489,6 @@ makeSimpleCommand id1 id2 prefix cmd suffix =
     redirection (T_FdRedirect {}) = True
     redirection _ = False
 
-
 prop_readSimpleCommand = isOk readSimpleCommand "echo test > file"
 prop_readSimpleCommand2 = isOk readSimpleCommand "cmd &> file"
 prop_readSimpleCommand3 = isOk readSimpleCommand "export foo=(bar baz)"
@@ -1382,20 +1496,26 @@ prop_readSimpleCommand4 = isOk readSimpleCommand "typeset -a foo=(lol)"
 prop_readSimpleCommand5 = isOk readSimpleCommand "time if true; then echo foo; fi"
 prop_readSimpleCommand6 = isOk readSimpleCommand "time -p ( ls -l; )"
 readSimpleCommand = called "simple command" $ do
+    pos <- getPosition
     id1 <- getNextId
     id2 <- getNextId
     prefix <- option [] readCmdPrefix
     cmd <- option Nothing $ do { f <- readCmdName; return $ Just f; }
-    when (null prefix && isNothing cmd) $ fail "No command"
+    when (null prefix && isNothing cmd) $ fail "Expected a command"
     case cmd of
       Nothing -> return $ makeSimpleCommand id1 id2 prefix [] []
       Just cmd -> do
             suffix <- option [] $ getParser readCmdSuffix cmd [
                         (["declare", "export", "local", "readonly", "typeset"], readModifierSuffix),
                         (["time"], readTimeSuffix),
-                        (["let"], readLetSuffix)
+                        (["let"], readLetSuffix),
+                        (["eval"], readEvalSuffix)
                     ]
-            return $ makeSimpleCommand id1 id2 prefix [cmd] suffix
+
+            let result = makeSimpleCommand id1 id2 prefix [cmd] suffix
+            if isCommand ["source", "."] cmd
+                then readSource pos result
+                else return result
   where
     isCommand strings (T_NormalWord _ [T_Literal _ s]) = s `elem` strings
     isCommand _ _ = False
@@ -1405,6 +1525,55 @@ readSimpleCommand = called "simple command" $ do
         then action
         else getParser def cmd rest
 
+
+readSource :: Monad m => SourcePos -> Token -> SCParser m Token
+readSource pos t@(T_Redirecting _ _ (T_SimpleCommand _ _ (cmd:file:_))) = do
+    override <- getSourceOverride
+    let literalFile = override `mplus` getLiteralString file
+    case literalFile of
+        Nothing -> do
+            parseNoteAt pos WarningC 1090
+                "Can't follow non-constant source. Use a directive to specify location."
+            return t
+        Just filename -> do
+            proceed <- shouldFollow filename
+            if not proceed
+              then do
+                parseNoteAt pos InfoC 1093
+                    "This file appears to be recursively sourced. Ignoring."
+                return t
+              else do
+                sys <- Mr.ask
+                input <-
+                    if filename == "/dev/null" -- always allow /dev/null
+                    then return (Right "")
+                    else system $ siReadFile sys filename
+                case input of
+                    Left err -> do
+                        parseNoteAt pos InfoC 1091 $
+                            "Not following: " ++ err
+                        return t
+                    Right script -> do
+                        id <- getNextIdAt pos
+
+                        let included = do
+                            src <- subRead filename script
+                            return $ T_Include id t src
+
+                        let failed = do
+                            parseNoteAt pos WarningC 1094
+                                "Parsing of sourced file failed. Ignoring it."
+                            return t
+
+                        included <|> failed
+  where
+    subRead name script =
+        withContext (ContextSource name) $
+            inSeparateContext $
+                subParse (initialPos name) readScript script
+readSource _ t = return t
+
+
 prop_readPipeline = isOk readPipeline "! cat /etc/issue | grep -i ubuntu"
 prop_readPipeline2 = isWarning readPipeline "!cat /etc/issue | grep -i ubuntu"
 prop_readPipeline3 = isOk readPipeline "for f; do :; done|cat"
@@ -1490,15 +1659,8 @@ readCommand = choice [
     readSimpleCommand
     ]
 
-readCmdName = do
+readCmdName = readCmdWord
-    f <- readNormalWord
+readCmdWord = readNormalWord <* spacing
-    spacing
-    return f
-
-readCmdWord = do
-    f <- readNormalWord
-    spacing
-    return f
 
 prop_readIfClause = isOk readIfClause "if false; then foo; elif true; then stuff; more stuff; else cows; fi"
 prop_readIfClause2 = isWarning readIfClause "if false; then; echo oo; fi"
@@ -1515,7 +1677,7 @@ readIfClause = called "if expression" $ do
     g_Fi `orFail` do
         parseProblemAt pos ErrorC 1046 "Couldn't find 'fi' for this 'if'."
         parseProblem ErrorC 1047 "Expected 'fi' matching previously mentioned 'if'."
-        return "Expected 'fi'."
+        return "Expected 'fi'"
 
     return $ T_IfExpression id ((condition, action):elifs) elses
 
@@ -1531,13 +1693,13 @@ readIfPart = do
     allspacing
     condition <- readTerm
 
-    ifNextToken (g_Fi <|> g_Elif) $
+    ifNextToken (g_Fi <|> g_Elif <|> g_Else) $
         parseProblemAt pos ErrorC 1049 "Did you forget the 'then' for this 'if'?"
 
     called "then clause" $ do
         g_Then `orFail` do
             parseProblem ErrorC 1050 "Expected 'then'."
-            return "Expected 'then'."
+            return "Expected 'then'"
 
         acceptButWarn g_Semi ErrorC 1051 "No semicolons directly after 'then'."
         allspacing
@@ -1554,7 +1716,7 @@ readElifPart = called "elif clause" $ do
     allspacing
     condition <- readTerm
 
-    ifNextToken (g_Fi <|> g_Elif) $
+    ifNextToken (g_Fi <|> g_Elif <|> g_Else) $
         parseProblemAt pos ErrorC 1049 "Did you forget the 'then' for this 'elif'?"
 
     g_Then
@@ -1587,7 +1749,7 @@ readSubshell = called "explicit subshell" $ do
     allspacing
     list <- readCompoundList
     allspacing
-    char ')'
+    char ')' <|> fail ") closing the subshell"
     return $ T_Subshell id list
 
 prop_readBraceGroup = isOk readBraceGroup "{ a; b | c | d; e; }"
@@ -1630,7 +1792,7 @@ readDoGroup loopPos = do
 
     g_Do `orFail` do
         parseProblem ErrorC 1058 "Expected 'do'."
-        return "Expected 'do'."
+        return "Expected 'do'"
 
     acceptButWarn g_Semi ErrorC 1059 "No semicolons directly after 'do'."
     allspacing
@@ -1643,7 +1805,7 @@ readDoGroup loopPos = do
     g_Done `orFail` do
             parseProblemAt pos ErrorC 1061 "Couldn't find 'done' for this 'do'."
             parseProblem ErrorC 1062 "Expected 'done' matching previously mentioned 'do'."
-            return "Expected 'done'."
+            return "Expected 'done'"
     return commands
 
 
@@ -1730,10 +1892,10 @@ readCaseClause = called "case expression" $ do
     g_Case
     word <- readNormalWord
     allspacing
-    g_In
+    g_In <|> fail "Expected 'in'"
     readLineBreak
     list <- readCaseList
-    g_Esac
+    g_Esac <|> fail "Expected 'esac' to close the case statement"
     return $ T_CaseExpression id word list
 
 readCaseList = many readCaseItem
@@ -1788,20 +1950,20 @@ readFunctionDefinition = called "function" $ do
                 whitespace
             spacing
             name <- readFunctionName
-            optional spacing
+            spacing
             hasParens <- wasIncluded readParens
             return $ T_Function id (FunctionKeyword True) (FunctionParentheses hasParens) name
 
         readWithoutFunction = try $ do
             id <- getNextId
             name <- readFunctionName
-            optional spacing
+            spacing
             readParens
             return $ T_Function id (FunctionKeyword False) (FunctionParentheses True) name
 
         readParens = do
             g_Lparen
-            optional spacing
+            spacing
             g_Rparen <|> do
                 parseProblem ErrorC 1065 "Trying to declare parameters? Don't. Use () and refer to params as $1, $2.."
                 many $ noneOf "\n){"
@@ -1840,7 +2002,7 @@ prop_readCompoundCommand = isOk readCompoundCommand "{ echo foo; }>/dev/null"
 readCompoundCommand = do
     id <- getNextId
     cmd <- choice [ readBraceGroup, readArithmeticExpression, readSubshell, readCondition, readWhileClause, readUntilClause, readIfClause, readForClause, readSelectClause, readCaseClause, readFunctionDefinition]
-    optional spacing
+    spacing
     redirs <- many readIoRedirect
     unless (null redirs) $ optional $ do
         lookAhead $ try (spacing >> needsSeparator)
@@ -1876,6 +2038,15 @@ readLetSuffix = many1 (readIoRedirect <|> try readLetExpression <|> readCmdWord)
         expression <- readStringForParser readCmdWord
         subParse startPos readArithmeticContents expression
 
+-- bash allows a=(b), ksh allows $a=(b). dash allows neither. Let's warn.
+readEvalSuffix = many1 (readIoRedirect <|> readCmdWord <|> evalFallback)
+  where
+    evalFallback = do
+        pos <- getPosition
+        lookAhead $ char '('
+        parseProblemAt pos WarningC 1098 "Quote/escape special characters when using eval, e.g. eval \"a=(b)\"."
+        fail "Unexpected parentheses. Make sure to quote when eval'ing as shell parsers differ."
+
 -- Get whatever a parser would parse as a string
 readStringForParser parser = do
     pos <- lookAhead (parser >> getPosition)
@@ -1923,8 +2094,19 @@ readAssignmentWord = try $ do
         spacing
         return $ T_Assignment id op variable index value
   where
-    readAssignmentOp =
+    readAssignmentOp = do
-        (string "+=" >> return Append) <|> (string "=" >> return Assign)
+        pos <- getPosition
+        unexpecting "" $ string "==="
+        choice [
+            string "+=" >> return Append,
+            do
+                try (string "==")
+                parseProblemAt pos ErrorC 1097
+                    "Unexpected ==. For assignment, use =. For comparison, use [/[[."
+                return Assign,
+
+            string "=" >> return Assign
+            ]
     readEmptyLiteral = do
         id <- getNextId
         return $ T_Literal id ""
@@ -1941,7 +2123,7 @@ readArray = called "array assignment" $ do
     char '('
     allspacing
     words <- readElement `reluctantlyTill` char ')'
-    char ')'
+    char ')' <|> fail "Expected ) to close array assignment"
     return $ T_Array id words
   where
     readElement = (readIndexed <|> readRegular) `thenSkip` allspacing
@@ -1975,17 +2157,22 @@ tryWordToken s t = tryParseWordToken s t `thenSkip` spacing
 tryParseWordToken keyword t = try $ do
     id <- getNextId
     str <- anycaseString keyword
-    optional (do
+
+    optional $ do
         try . lookAhead $ char '['
-        parseProblem ErrorC 1069 "You need a space before the [.")
+        parseProblem ErrorC 1069 "You need a space before the [."
+    optional $ do
+        try . lookAhead $ char '#'
+        parseProblem ErrorC 1099 "You need a space before the #."
+
     try $ lookAhead keywordSeparator
     when (str /= keyword) $
         parseProblem ErrorC 1081 $
             "Scripts are case sensitive. Use '" ++ keyword ++ "', not '" ++ str ++ "'."
     return $ t id
 
-anycaseString str =
+anycaseString =
-    mapM anycaseChar str <?> str
+    mapM anycaseChar
   where
     anycaseChar c = char (toLower c) <|> char (toUpper c)
 
@@ -2016,7 +2203,10 @@ g_For = tryWordToken "for" T_For
 g_Select = tryWordToken "select" T_Select
 g_In = tryWordToken "in" T_In
 g_Lbrace = tryWordToken "{" T_Lbrace
-g_Rbrace = tryWordToken "}" T_Rbrace
+g_Rbrace = do -- handled specially due to ksh echo "${ foo; }bar"
+    id <- getNextId
+    char '}'
+    return $ T_Rbrace id
 
 g_Lparen = tryToken "(" T_Lparen
 g_Rparen = tryToken ")" T_Rparen
@@ -2141,15 +2331,21 @@ readScript = do
 
     readUtf8Bom = called "Byte Order Mark" $ string "\xFEFF"
 
-rp p filename contents = Ms.runState (runParserT p initialState filename contents) ([], [])
 
-isWarning p s = fst cs && (not . null . snd $ cs) where cs = checkString p s
+isWarning p s = parsesCleanly p s == Just False
-isOk p s = fst cs && (null . snd $ cs) where cs = checkString p s
+isOk p s =      parsesCleanly p s == Just True
 
-checkString parser string =
+testParse string = runIdentity $ do
-    case rp (parser >> eof >> getState) "-" string of
+    (res, _) <- runParser (mockedSystemInterface []) readScript "-" string
-        (Right (tree, map, notes), (problems, _)) -> (True, notes ++ problems)
+    return res
-        (Left _, (n, _)) -> (False, n)
+
+parsesCleanly parser string = runIdentity $ do
+    (res, sys) <- runParser (mockedSystemInterface [])
+                    (parser >> eof >> getState) "-" string
+    case (res, sys) of
+        (Right userState, systemState) ->
+            return $ Just . null $ parseNotes userState ++ parseProblems systemState
+        (Left _, _) -> return Nothing
 
 parseWithNotes parser = do
     item <- parser
@@ -2161,8 +2357,6 @@ compareNotes (ParseNote pos1 level1 _ s1) (ParseNote pos2 level2 _ s2) = compare
 sortNotes = sortBy compareNotes
 
 
-data ParseResult = ParseResult { parseResult :: Maybe (Token, Map.Map Id SourcePos), parseNotes :: [ParseNote] } deriving (Show)
-
 makeErrorFor parsecError =
     ParseNote (errorPos parsecError) ErrorC 1072 $
         getStringFromParsec $ errorMessages parsecError
@@ -2174,19 +2368,45 @@ getStringFromParsec errors =
     where
         f err =
             case err of
-                UnExpect s    ->  return $ unexpected s
+                UnExpect s    ->  Nothing -- Due to not knowing Parsec, none of these
-                SysUnExpect s ->  return $ unexpected s
+                SysUnExpect s ->  Nothing -- are actually helpful. <?> has been hidden
-                Expect s      ->  return $ "Expected " ++ s ++ "."
+                Expect s      ->  Nothing -- and we only show explicit fail statements.
                 Message s     ->  if null s then Nothing else return $ s ++ "."
-        unexpected s = "Unexpected " ++ (if null s then "eof" else s) ++ "."
 
-parseShell options filename contents =
+runParser :: Monad m =>
-    case rp (parseWithNotes readScript) filename contents of
+    SystemInterface m ->
-        (Right (script, map, notes), (parsenotes, _)) ->
+    SCParser m v ->
-            ParseResult (Just (script, map)) (nub . sortNotes . excludeNotes $ notes ++ parsenotes)
+    String ->
-        (Left err, (p, context)) ->
+    String ->
-            ParseResult Nothing
+    m (Either ParseError v, SystemState)
-                (nub . sortNotes . excludeNotes $ p ++ notesForContext context ++ [makeErrorFor err])
+
+runParser sys p filename contents =
+    Ms.runStateT
+        (Mr.runReaderT
+            (runParserT p initialUserState filename contents)
+            sys)
+        initialSystemState
+system = lift . lift . lift
+
+parseShell sys name contents = do
+    (result, state) <- runParser sys (parseWithNotes readScript) name contents
+    case result of
+        Right (script, tokenMap, notes) ->
+            return ParseResult {
+                prComments = map toPositionedComment $ nub $ notes ++ parseProblems state,
+                prTokenPositions = Map.map posToPos tokenMap,
+                prRoot = Just script
+            }
+        Left err ->
+            return ParseResult {
+                prComments =
+                    map toPositionedComment $
+                        notesForContext (contextStack state)
+                        ++ [makeErrorFor err]
+                        ++ parseProblems state,
+                prTokenPositions = Map.empty,
+                prRoot = Nothing
+            }
   where
     isName (ContextName _ _) = True
     isName _ = False
@@ -2195,7 +2415,25 @@ parseShell options filename contents =
         "Couldn't parse this " ++ str ++ "."
     second (ContextName pos str) = ParseNote pos InfoC 1009 $
         "The mentioned parser error was in this " ++ str ++ "."
-    excludeNotes = filter (\c -> codeForParseNote c `notElem` optionExcludes options)
+
+
+toPositionedComment :: ParseNote -> PositionedComment
+toPositionedComment (ParseNote pos severity code message) =
+    PositionedComment (posToPos pos) $ Comment severity code message
+
+posToPos :: SourcePos -> Position
+posToPos sp = Position {
+    posFile = sourceName sp,
+    posLine = fromIntegral $ sourceLine sp,
+    posColumn = fromIntegral $ sourceColumn sp
+}
+
+-- TODO: Clean up crusty old code that this is layered on top of
+parseScript :: Monad m =>
+        SystemInterface m -> ParseSpec -> m ParseResult
+parseScript sys spec =
+    parseShell sys (psFilename spec) (psScript spec)
+
 
 lt x = trace (show x) x
 ltt t = trace (show t)

ShellCheck/Regex.hs

@@ -1,4 +1,6 @@
 {-
+    Copyright 2012-2015 Vidar Holen
+
     This file is part of ShellCheck.
     http://www.vidarholen.net/contents/shellcheck
 
@@ -69,3 +71,10 @@ subRegex re input replacement = f input
         (before, match, after) <- matchM re str :: Maybe (String, String, String)
         when (null match) $ error ("Internal error: substituted empty in " ++ str)
         return $ before ++ replacement ++ f after
+
+-- Split a string based on a regex.
+splitOn :: String -> Regex -> [String]
+splitOn input re =
+    case matchM re input :: Maybe (String, String, String) of
+        Just (before, match, after) -> before : after `splitOn` re
+        Nothing -> [input]

ShellCheck/Simple.hs

@@ -1,80 +0,0 @@
-{-
-    This file is part of ShellCheck.
-    http://www.vidarholen.net/contents/shellcheck
-
-    ShellCheck is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    ShellCheck is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
--}
-{-# LANGUAGE TemplateHaskell #-}
-module ShellCheck.Simple (shellCheck, ShellCheckComment, scLine, scColumn, scSeverity, scCode, scMessage, runTests) where
-
-import Data.List
-import Data.Maybe
-import ShellCheck.Analytics hiding (runTests)
-import ShellCheck.Options
-import ShellCheck.Parser hiding (runTests)
-import Test.QuickCheck.All (quickCheckAll)
-import Text.Parsec.Pos
-
-shellCheck :: AnalysisOptions -> String -> [ShellCheckComment]
-shellCheck options script =
-    let (ParseResult result notes) = parseShell options "-" script in
-        let allNotes = notes ++ concat (maybeToList $ do
-            (tree, posMap) <- result
-            let list = runAnalytics options tree
-            return $ map (noteToParseNote posMap) $ filterByAnnotation tree list
-            )
-        in
-            map formatNote $ nub $ sortNotes allNotes
-
-data ShellCheckComment = ShellCheckComment { scLine :: Int, scColumn :: Int, scSeverity :: String, scCode :: Int, scMessage :: String }
-
-instance Show ShellCheckComment where
-    show c = concat ["(", show $ scLine c, ",", show $ scColumn c, ") ", scSeverity c, ": ", show (scCode c), " ", scMessage c]
-
-severityToString s =
-    case s of
-        ErrorC -> "error"
-        WarningC -> "warning"
-        InfoC -> "info"
-        StyleC -> "style"
-
-formatNote (ParseNote pos severity code text) =
-    ShellCheckComment (sourceLine pos) (sourceColumn pos) (severityToString severity) (fromIntegral code) text
-
-testCheck = shellCheck defaultAnalysisOptions { optionExcludes = [2148] } -- Ignore #! warnings
-prop_findsParseIssue =
-    let comments = testCheck "echo \"$12\"" in
-        length comments == 1 && scCode (head comments) == 1037
-prop_commentDisablesParseIssue1 =
-    null $ testCheck "#shellcheck disable=SC1037\necho \"$12\""
-prop_commentDisablesParseIssue2 =
-    null $ testCheck "#shellcheck disable=SC1037\n#lol\necho \"$12\""
-
-prop_findsAnalysisIssue =
-    let comments = testCheck "echo $1" in
-        length comments == 1 && scCode (head comments) == 2086
-prop_commentDisablesAnalysisIssue1 =
-    null $ testCheck "#shellcheck disable=SC2086\necho $1"
-prop_commentDisablesAnalysisIssue2 =
-    null $ testCheck "#shellcheck disable=SC2086\n#lol\necho $1"
-
-prop_optionDisablesIssue1 =
-    null $ shellCheck (defaultAnalysisOptions { optionExcludes = [2086, 2148] }) "echo $1"
-
-prop_optionDisablesIssue2 =
-    null $ shellCheck (defaultAnalysisOptions { optionExcludes = [2148, 1037] }) "echo \"$10\""
-
-return []
-runTests = $quickCheckAll
-

shellcheck.1.md

@@ -50,9 +50,16 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts.
     The default is to use the file's shebang, or *bash* if the target shell
     can't be determined.
 
-**-V**\ *version*,\ **--version**
+**-V**,\ **--version**
 
-:   Print version and exit.
+:   Print version information and exit.
+
+**-x**,\ **-external-sources**
+
+:   Follow 'source' statements even when the file is not specified as input.
+    By default, `shellcheck` will only follow files specified on the command
+    line (plus `/dev/null`). This option allows following any file the script
+    may `source`.
 
 # FORMATS
 
@@ -119,7 +126,12 @@ For example, to suppress SC2035 about using `./*.jpg`:
     # shellcheck disable=SC2035
     echo "Files: " *.jpg
 
-Here a shell brace group is used to suppress on multiple lines:
+To tell ShellCheck where to look for an otherwise dynamically determined file:
+
+    # shellcheck source=./lib.sh
+    source "$(find_install_dir)/lib.sh"
+
+Here a shell brace group is used to suppress a warning on multiple lines:
 
     # shellcheck disable=SC2016
     {
@@ -134,6 +146,18 @@ Valid keys are:
     The command can be a simple command like `echo foo`, or a compound command
     like a function definition, subshell block or loop.
 
+**source**
+:   Overrides the filename included by a `source`/`.` statement. This can be
+    used to tell shellcheck where to look for a file whose name is determined
+    at runtime, or to skip a source by telling it to use `/dev/null`.
+
+# ENVIRONMENT VARIABLES
+The environment variable `SHELLCHECK_OPTS` can be set with default flags:
+
+    export SHELLCHECK_OPTS='--shell=bash --exclude=SC2016'
+
+Its value will be split on spaces and prepended to the command line on each
+invocation.
 
 # AUTHOR
 ShellCheck is written and maintained by Vidar Holen.
@@ -143,6 +167,12 @@ Bugs and issues can be reported on GitHub:
 
 https://github.com/koalaman/shellcheck/issues
 
+# COPYRIGHT
+Copyright 2012-2015, Vidar Holen.
+Licensed under the GNU General Public License version 3 or later,
+see http://gnu.org/licenses/gpl.html
+
+
 # SEE ALSO
 
 sh(1) bash(1)

shellcheck.hs

@@ -1,4 +1,6 @@
 {-
+    Copyright 2012-2015 Vidar Holen
+
     This file is part of ShellCheck.
     http://www.vidarholen.net/contents/shellcheck
 
@@ -15,44 +17,58 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <http://www.gnu.org/licenses/>.
 -}
+import ShellCheck.Data
+import ShellCheck.Checker
+import ShellCheck.Interface
+import ShellCheck.Regex
+
+import ShellCheck.Formatter.Format
+import qualified ShellCheck.Formatter.CheckStyle
+import qualified ShellCheck.Formatter.GCC
+import qualified ShellCheck.Formatter.JSON
+import qualified ShellCheck.Formatter.TTY
+
 import Control.Exception
 import Control.Monad
-import Control.Monad.Trans
+import Control.Monad.Except
-import Control.Monad.Trans.Error
-import Control.Monad.Trans.List
 import Data.Char
-import Data.List
+import Data.Functor
+import Data.Either
+import qualified Data.Map as Map
 import Data.Maybe
 import Data.Monoid
-import GHC.Exts
-import GHC.IO.Device
 import Prelude hiding (catch)
-import ShellCheck.Data
-import ShellCheck.Options
-import ShellCheck.Simple
-import ShellCheck.Analytics
 import System.Console.GetOpt
 import System.Directory
 import System.Environment
 import System.Exit
-import System.Info
 import System.IO
-import Text.JSON
-import qualified Data.Map as Map
 
 data Flag = Flag String String
-data Status = NoProblems | SomeProblems | BadInput | SupportFailure | SyntaxFailure | RuntimeException deriving (Ord, Eq)
+data Status =
-
+    NoProblems
-data JsonComment = JsonComment FilePath ShellCheckComment
+    | SomeProblems
-
+    | BadInput
-instance Error Status where
+    | SupportFailure
-    noMsg = RuntimeException
+    | SyntaxFailure
+    | RuntimeException
+  deriving (Ord, Eq, Show)
 
 instance Monoid Status where
     mempty = NoProblems
     mappend = max
 
-header = "Usage: shellcheck [OPTIONS...] FILES..."
+data Options = Options {
+    checkSpec :: CheckSpec,
+    externalSources :: Bool
+}
+
+defaultOptions = Options {
+    checkSpec = emptyCheckSpec,
+    externalSources = False
+}
+
+usageHeader = "Usage: shellcheck [OPTIONS...] FILES..."
 options = [
     Option "e" ["exclude"]
         (ReqArg (Flag "exclude") "CODE1,CODE2..") "exclude types of warnings",
@@ -60,204 +76,30 @@ options = [
         (ReqArg (Flag "format") "FORMAT") "output format",
     Option "s" ["shell"]
         (ReqArg (Flag "shell") "SHELLNAME") "Specify dialect (bash,sh,ksh)",
+    Option "x" ["external-sources"]
+        (NoArg $ Flag "externals" "true") "Allow 'source' outside of FILES.",
     Option "V" ["version"]
         (NoArg $ Flag "version" "true") "Print version information"
     ]
 
-printErr = hPutStrLn stderr
+printErr = lift . hPutStrLn stderr
 
-
+parseArguments :: [String] -> ExceptT Status IO ([Flag], [FilePath])
-instance JSON (JsonComment) where
-  showJSON (JsonComment filename c) = makeObj [
-      ("file", showJSON $ filename),
-      ("line", showJSON $ scLine c),
-      ("column", showJSON $ scColumn c),
-      ("level", showJSON $ scSeverity c),
-      ("code", showJSON $ scCode c),
-      ("message", showJSON $ scMessage c)
-      ]
-  readJSON = undefined
-
-parseArguments :: [String] -> ErrorT Status IO ([Flag], [FilePath])
 parseArguments argv =
     case getOpt Permute options argv of
         (opts, files, []) -> return (opts, files)
         (_, _, errors) -> do
-            liftIO . printErr $ concat errors ++ "\n" ++ usageInfo header options
+            printErr $ concat errors ++ "\n" ++ usageInfo usageHeader options
             throwError SyntaxFailure
 
-formats :: Map.Map String (AnalysisOptions -> [FilePath] -> IO Status)
+formats :: Map.Map String (IO Formatter)
 formats = Map.fromList [
-    ("json", forJson),
+    ("checkstyle", ShellCheck.Formatter.CheckStyle.format),
-    ("gcc", forGcc),
+    ("gcc",  ShellCheck.Formatter.GCC.format),
-    ("checkstyle", forCheckstyle),
+    ("json", ShellCheck.Formatter.JSON.format),
-    ("tty", forTty)
+    ("tty",  ShellCheck.Formatter.TTY.format)
     ]
 
-toStatus = liftM (either id (const NoProblems)) . runErrorT
-
-catchExceptions :: IO Status -> IO Status
-catchExceptions action = action -- action `catch` handler
-  where
-    handler err = do
-        printErr $ show (err :: SomeException)
-        return RuntimeException
-
-checkComments comments = if null comments then NoProblems else SomeProblems
-
-forTty :: AnalysisOptions -> [FilePath] -> IO Status
-forTty options files = do
-    output <- mapM doFile files
-    return $ mconcat output
-  where
-    clear = ansi 0
-    ansi n = "\x1B[" ++ show n ++ "m"
-
-    colorForLevel "error" = 31 -- red
-    colorForLevel "warning" = 33 -- yellow
-    colorForLevel "info" = 32 -- green
-    colorForLevel "style" = 32 -- green
-    colorForLevel "message" = 1 -- bold
-    colorForLevel "source" = 0 -- none
-    colorForLevel _ = 0 -- none
-
-    colorComment level comment =
-        ansi (colorForLevel level) ++ comment ++ clear
-
-    doFile path = catchExceptions $ do
-        contents <- readContents path
-        doInput path contents
-
-    doInput filename contents = do
-        let fileLines = lines contents
-        let lineCount = length fileLines
-        let comments = getComments options contents
-        let groups = groupWith scLine comments
-        colorFunc <- getColorFunc
-        mapM_ (\x -> do
-            let lineNum = scLine (head x)
-            let line = if lineNum < 1 || lineNum > lineCount
-                            then ""
-                            else fileLines !! (lineNum - 1)
-            putStrLn ""
-            putStrLn $ colorFunc "message"
-                ("In " ++ filename ++" line " ++ show lineNum ++ ":")
-            putStrLn (colorFunc "source" line)
-            mapM_ (\c -> putStrLn (colorFunc (scSeverity c) $ cuteIndent c)) x
-            putStrLn ""
-          ) groups
-        return . checkComments $ comments
-
-    cuteIndent comment =
-        replicate (scColumn comment - 1) ' ' ++
-            "^-- " ++ code (scCode comment) ++ ": " ++ scMessage comment
-
-    code code = "SC" ++ show code
-
-    getColorFunc = do
-        term <- hIsTerminalDevice stdout
-        let windows = "mingw" `isPrefixOf` os
-        return $ if term && not windows then colorComment else const id
-
-forJson :: AnalysisOptions -> [FilePath] -> IO Status
-forJson options files = catchExceptions $ do
-    comments <- runListT $ do
-        file <- ListT $ return files
-        comment <- ListT $ commentsFor options file
-        return $ JsonComment file comment
-    putStrLn $ encodeStrict comments
-    return $ checkComments comments
-
--- Mimic GCC "file:line:col: (error|warning|note): message" format
-forGcc :: AnalysisOptions -> [FilePath] -> IO Status
-forGcc options files = do
-    files <- mapM process files
-    return $ mconcat files
-  where
-    process file = catchExceptions $ do
-        contents <- readContents file
-        let comments = makeNonVirtual (getComments options contents) contents
-        mapM_ (putStrLn . format file) comments
-        return $ checkComments comments
-
-    format filename c = concat [
-            filename, ":",
-            show $ scLine c, ":",
-            show $ scColumn c, ": ",
-            case scSeverity c of
-                "error" -> "error"
-                "warning" -> "warning"
-                _ -> "note",
-            ": ",
-            concat . lines $ scMessage c,
-            " [SC", show $ scCode c, "]"
-      ]
-
--- Checkstyle compatible output. A bit of a hack to avoid XML dependencies
-forCheckstyle :: AnalysisOptions -> [FilePath] -> IO Status
-forCheckstyle options files = do
-    putStrLn "<?xml version='1.0' encoding='UTF-8'?>"
-    putStrLn "<checkstyle version='4.3'>"
-    statuses <- mapM process files
-    putStrLn "</checkstyle>"
-    return $ mconcat statuses
-  where
-    process file = catchExceptions $ do
-        comments <- commentsFor options file
-        putStrLn (formatFile file comments)
-        return $ checkComments comments
-
-    severity "error" = "error"
-    severity "warning" = "warning"
-    severity _ = "info"
-    attr s v = concat [ s, "='", escape v, "' " ]
-    escape = concatMap escape'
-    escape' c = if isOk c then [c] else "&#" ++ show (ord c) ++ ";"
-    isOk x = any ($x) [isAsciiUpper, isAsciiLower, isDigit, (`elem` " ./")]
-
-    formatFile name comments = concat [
-        "<file ", attr "name" name, ">\n",
-            concatMap format comments,
-        "</file>"
-        ]
-
-    format c = concat [
-        "<error ",
-        attr "line" $ show . scLine $ c,
-        attr "column" $ show . scColumn $ c,
-        attr "severity" $ severity . scSeverity $ c,
-        attr "message" $ scMessage c,
-        attr "source" $ "ShellCheck.SC" ++ show (scCode c),
-        "/>\n"
-        ]
-
-commentsFor options file = liftM (getComments options) $ readContents file
-
-getComments = shellCheck
-
-readContents :: FilePath -> IO String
-readContents file =
-    if file == "-"
-    then getContents
-    else readFile file
-
--- Realign comments from a tabstop of 8 to 1
-makeNonVirtual comments contents =
-    map fix comments
-  where
-    ls = lines contents
-    fix c = c {
-        scColumn =
-            if scLine c > 0 && scLine c <= length ls
-            then real (ls !! (scLine c - 1)) 0 0 (scColumn c)
-            else scColumn c
-    }
-    real _ r v target | target <= v = r
-    real [] r v _ = r -- should never happen
-    real ('\t':rest) r v target =
-        real rest (r+1) (v + 8 - (v `mod` 8)) target
-    real (_:rest) r v target = real rest (r+1) (v+1) target
-
 getOption [] _ = Nothing
 getOption (Flag var val:_) name | name == var = return val
 getOption (_:rest) flag = getOption rest flag
@@ -280,13 +122,19 @@ getExclusions options =
     in
         map (Prelude.read . clean) elements :: [Int]
 
-excludeCodes codes =
+toStatus = liftM (either id id) . runExceptT
-    filter (not . hasCode)
+
+getEnvArgs = do
+    opts <- getEnv "SHELLCHECK_OPTS" `catch` cantWaitForLookupEnv
+    return . filter (not . null) $ opts `splitOn` mkRegex " +"
   where
-    hasCode c = scCode c `elem` codes
+    cantWaitForLookupEnv :: IOException -> IO String
+    cantWaitForLookupEnv = const $ return ""
 
 main = do
-    args <- getArgs
+    params <- getArgs
+    envOpts  <- getEnvArgs
+    let args = envOpts ++ params
     status <- toStatus $ do
         (flags, files) <- parseArguments args
         process flags files
@@ -301,53 +149,137 @@ statusToCode status =
         SupportFailure -> ExitFailure 4
         RuntimeException -> ExitFailure 2
 
-process :: [Flag] -> [FilePath] -> ErrorT Status IO ()
+process :: [Flag] -> [FilePath] -> ExceptT Status IO Status
 process flags files = do
-    options <- foldM (flip parseOption) defaultAnalysisOptions flags
+    options <- foldM (flip parseOption) defaultOptions flags
     verifyFiles files
     let format = fromMaybe "tty" $ getOption flags "format"
+    formatter <-
         case Map.lookup format formats of
             Nothing -> do
-            liftIO $ do
                 printErr $ "Unknown format " ++ format
                 printErr "Supported formats:"
                 mapM_ (printErr . write) $ Map.keys formats
                 throwError SupportFailure
               where write s = "  " ++ s
-        Just f -> ErrorT $ liftM Left $ f options files
+            Just f -> ExceptT $ fmap Right f
+    sys <- lift $ ioInterface options files
+    lift $ runFormatter sys formatter options files
+
+runFormatter :: SystemInterface IO -> Formatter -> Options -> [FilePath]
+            -> IO Status
+runFormatter sys format options files = do
+    header format
+    result <- foldM f NoProblems files
+    footer format
+    return result
+  where
+    f :: Status -> FilePath -> IO Status
+    f status file = do
+        newStatus <- process file `catch` handler file
+        return $ status `mappend` newStatus
+    handler :: FilePath -> IOException -> IO Status
+    handler file e = do
+        onFailure format file (show e)
+        return RuntimeException
+
+    process :: FilePath -> IO Status
+    process filename = do
+        contents <- inputFile filename
+        let checkspec = (checkSpec options) {
+            csFilename = filename,
+            csScript = contents
+        }
+        result <- checkScript sys checkspec
+        onResult format result contents
+        return $
+            if null (crComments result)
+            then NoProblems
+            else SomeProblems
 
 parseOption flag options =
     case flag of
         Flag "shell" str ->
             fromMaybe (die $ "Unknown shell: " ++ str) $ do
                 shell <- shellForExecutable str
-                    return $ return options { optionShellType = Just shell }
+                return $ return options {
+                            checkSpec = (checkSpec options) {
+                                csShellTypeOverride = Just shell
+                            }
+                        }
 
         Flag "exclude" str -> do
             new <- mapM parseNum $ split ',' str
-            let old = optionExcludes options
+            let old = csExcludedWarnings . checkSpec $ options
-            return options { optionExcludes = new ++ old }
+            return options {
+                checkSpec = (checkSpec options) {
+                    csExcludedWarnings = new ++ old
+                }
+            }
 
         Flag "version" _ -> do
             liftIO printVersion
             throwError NoProblems
 
+        Flag "externals" _ -> do
+            return options {
+                externalSources = True
+            }
+
         _ -> return options
   where
     die s = do
-        liftIO $ printErr s
+        printErr s
         throwError SupportFailure
     parseNum ('S':'C':str) = parseNum str
     parseNum num = do
         unless (all isDigit num) $ do
-            liftIO . printErr $ "Bad exclusion: " ++ num
+            printErr $ "Bad exclusion: " ++ num
             throwError SyntaxFailure
         return (Prelude.read num :: Integer)
 
+ioInterface options files = do
+    inputs <- mapM normalize files
+    return SystemInterface {
+        siReadFile = get inputs
+    }
+  where
+    get inputs file = do
+        ok <- allowable inputs file
+        if ok
+          then (Right <$> inputFile file) `catch` handler
+          else return $ Left (file ++ " was not specified as input (see shellcheck -x).")
+
+      where
+        handler :: IOException -> IO (Either ErrorMessage String)
+        handler ex = return . Left $ show ex
+
+    allowable inputs x =
+        if externalSources options
+        then return True
+        else do
+            path <- normalize x
+            return $ path `elem` inputs
+
+    normalize x =
+        canonicalizePath x `catch` fallback x
+      where
+        fallback :: FilePath -> IOException -> IO FilePath
+        fallback path _ = return path
+
+inputFile file = do
+    contents <-
+            if file == "-"
+            then getContents
+            else readFile file
+
+    seq (length contents) $
+        return contents
+
 verifyFiles files =
     when (null files) $ do
-        liftIO $ printErr "No files specified.\n"
+        printErr "No files specified.\n"
-        liftIO $ printErr $ usageInfo header options
+        printErr $ usageInfo usageHeader options
         throwError SyntaxFailure
 
 printVersion = do

test/shellcheck.hs

@@ -2,15 +2,17 @@ module Main where
 
 import Control.Monad
 import System.Exit
-import qualified ShellCheck.Simple
+import qualified ShellCheck.Checker
 import qualified ShellCheck.Analytics
 import qualified ShellCheck.Parser
 
 main = do
     putStrLn "Running ShellCheck tests..."
-    results <- sequence [ShellCheck.Simple.runTests,
+    results <- sequence [
+        ShellCheck.Checker.runTests,
         ShellCheck.Analytics.runTests,
-                         ShellCheck.Parser.runTests]
+        ShellCheck.Parser.runTests
-    if and results then exitSuccess
+      ]
+    if and results
+      then exitSuccess
       else exitFailure
-