{-# OPTIONS #-}
module Language.Python.Version3.Lexer (
lex,
lexOneToken,
lexer,
initLexState ) where
import Prelude hiding (lex)
import Language.Python.Version3.Parser.Lexer (lexToken, initStartCodeStack)
import Language.Python.Common.Token as Token
import Language.Python.Common.SrcLocation (initialSrcLocation)
import Language.Python.Common.ParserMonad
(ParseState (input), P, runParser, execParser, ParseError, initialState)
initLexState :: String -> String -> ParseState
initLexState :: String -> String -> ParseState
initLexState input :: String
input srcName :: String
srcName =
SrcLocation -> String -> [Int] -> ParseState
initialState (String -> SrcLocation
initialSrcLocation String
srcName) String
input [Int]
initStartCodeStack
lex :: String
-> String
-> Either ParseError [Token]
lex :: String -> String -> Either ParseError [Token]
lex input :: String
input srcName :: String
srcName =
P [Token] -> ParseState -> Either ParseError [Token]
forall a. P a -> ParseState -> Either ParseError a
execParser P [Token]
lexer (ParseState -> Either ParseError [Token])
-> ParseState -> Either ParseError [Token]
forall a b. (a -> b) -> a -> b
$ String -> String -> ParseState
initLexState String
input String
srcName
lexOneToken :: String
-> String
-> Either ParseError (Token, String)
lexOneToken :: String -> String -> Either ParseError (Token, String)
lexOneToken source :: String
source srcName :: String
srcName =
case P Token -> ParseState -> Either ParseError (Token, ParseState)
forall a. P a -> ParseState -> Either ParseError (a, ParseState)
runParser P Token
lexToken ParseState
state of
Left err :: ParseError
err -> ParseError -> Either ParseError (Token, String)
forall a b. a -> Either a b
Left ParseError
err
Right (tok :: Token
tok, state :: ParseState
state) -> (Token, String) -> Either ParseError (Token, String)
forall a b. b -> Either a b
Right (Token
tok, ParseState -> String
input ParseState
state)
where
state :: ParseState
state = String -> String -> ParseState
initLexState String
source String
srcName
lexer :: P [Token]
lexer :: P [Token]
lexer = [Token] -> P [Token]
loop []
where
loop :: [Token] -> P [Token]
loop toks :: [Token]
toks = do
Token
tok <- P Token
lexToken
case Token
tok of
EOFToken {} -> [Token] -> P [Token]
forall (m :: * -> *) a. Monad m => a -> m a
return ([Token] -> [Token]
forall a. [a] -> [a]
reverse [Token]
toks)
other :: Token
other -> [Token] -> P [Token]
loop (Token
tokToken -> [Token] -> [Token]
forall a. a -> [a] -> [a]
:[Token]
toks)