root/trunk/dlexer.d

// dlexer - D language lexer
// (C) Copyright 2004-2005 James Dunne

// Please feel free to distribute this module in any form you like.  I only ask that you
// give credit where credit is due.  You may make modifications to this module.  If you do
// so, you may provide your modifications publicly, but you are not required to.

module  dlexer;

import  std.stream;
import  std.string;
import  std.ctype;

private import std.c.stdlib;    // for strtod and strtold

alias std.ctype.isdigit isdigit;

// Set this version identifier to enable interpretation of escaped characters within parsed strings
// If this is disabled, the strings are copied directly, escape sequences and all.

// version = interpret_slashes;

// Enumeration of D language tokens taken from DMD's lexer.h
enum : uint {
    TOKreserved,

    // Other
    TOKlparen,  TOKrparen,
    TOKlbracket,    TOKrbracket,
    TOKlcurly,  TOKrcurly,
    TOKcolon,   TOKneg,
    TOKsemicolon,   TOKdotdotdot,
    TOKeof,     TOKcast,
    TOKnull,    TOKassert,
    TOKtrue,    TOKfalse,
    TOKarray,   TOKcall,
    TOKaddress, TOKtypedot,
    TOKtype,    TOKthrow,
    TOKnew,     TOKdelete,
    TOKstar,    TOKsymoff,
    TOKvar,     TOKdotvar,
    TOKdotti,   TOKdotexp,
    TOKdottype, TOKslice,
    TOKarraylength, TOKversion,
    TOKmodule,  TOKdollar,
    TOKtemplate,    TOKinstance,
    TOKdeclaration, TOKtypeof,
    TOKpragma,  TOKdsymbol,
    TOKtypeid,  TOKuadd,

    // Operators
    TOKlt,      TOKgt,
    TOKle,      TOKge,
    TOKequal,   TOKnotequal,
    TOKidentity,    TOKnotidentity,
    TOKindex,

    // NCEG floating point compares
    // !<>=     <>    <>=    !>     !>=   !<     !<=   !<>
    TOKunord,TOKlg,TOKleg,TOKule,TOKul,TOKuge,TOKug,TOKue,

    TOKshl,     TOKshr,
    TOKshlass,  TOKshrass,
    TOKushr,    TOKushrass,
    TOKcat,     TOKcatass,  // ~ ~=
    TOKadd,     TOKmin,     TOKaddass,  TOKminass,
    TOKmul,     TOKdiv,     TOKmod,
    TOKmulass,  TOKdivass,  TOKmodass,
    TOKand,     TOKor,      TOKxor,
    TOKandass,  TOKorass,   TOKxorass,
    TOKassign,  TOKnot,     TOKtilde,
    TOKplusplus,    TOKminusminus,
    TOKdot,     TOKarrow,   TOKcomma,
    TOKquestion,    TOKandand,  TOKoror,

    // Numeric literals
    TOKint32v, TOKuns32v,
    TOKint64v, TOKuns64v,
    TOKfloat32v, TOKfloat64v, TOKfloat80v,
    TOKimaginary32v, TOKimaginary64v, TOKimaginary80v,

    // Char constants
    TOKcharv, TOKwcharv, TOKdcharv,

    // Leaf operators
    TOKidentifier,  TOKstring,
    TOKthis,    TOKsuper,

    // Basic types
    TOKvoid,
    TOKint8, TOKuns8,
    TOKint16, TOKuns16,
    TOKint32, TOKuns32,
    TOKint64, TOKuns64,
    TOKfloat32, TOKfloat64, TOKfloat80,
    TOKimaginary32, TOKimaginary64, TOKimaginary80,
    TOKcomplex32, TOKcomplex64, TOKcomplex80,
    TOKchar, TOKwchar, TOKdchar, TOKbit,

    // Aggregates
    TOKstruct, TOKclass, TOKinterface, TOKunion, TOKenum, TOKimport,
    TOKtypedef, TOKalias, TOKoverride, TOKdelegate, TOKfunction,
    TOKmixin,

    TOKalign, TOKextern, TOKprivate, TOKprotected, TOKpublic, TOKexport,
    TOKstatic, /*TOKvirtual,*/ TOKfinal, TOKconst, TOKabstract, TOKvolatile,
    TOKdebug, TOKdeprecated, TOKin, TOKout, TOKinout,
    TOKauto, TOKpackage,

    // Statements
    TOKif, TOKelse, TOKwhile, TOKfor, TOKdo, TOKswitch,
    TOKcase, TOKdefault, TOKbreak, TOKcontinue, TOKwith,
    TOKsynchronized, TOKreturn, TOKgoto, TOKtry, TOKcatch, TOKfinally,
    TOKasm, TOKforeach,

    // Contracts
    TOKbody, TOKinvariant,

    // Testing
    TOKunittest,

    TOKmax
}

// A table converting token values into string representations:
char[]  toktostr[TOKmax] = [
    TOKreserved : "reserved",

    // Other
    TOKlparen : "(",
    TOKrparen : ")",
    TOKlbracket : "[",
    TOKrbracket : "]",
    TOKlcurly : "{",
    TOKrcurly : "}",
    TOKcolon : ":",
    TOKneg : "-",
    TOKsemicolon : ";",
    TOKdotdotdot : "...",
    TOKeof : "EOF",
    TOKcast : "cast",
    TOKnull : "null",
    TOKassert : "assert",
    TOKtrue : "true",
    TOKfalse : "false",
    TOKarray : "[]",
    TOKcall : "call",
    TOKaddress : "#",
    TOKtypedot : "typedot",
    TOKtype : "type",
    TOKthrow : "throw",
    TOKnew : "new",
    TOKdelete : "delete",
    TOKstar : "*",
    TOKsymoff : "symoff",
    TOKvar : "var",
    TOKdotvar : "dotvar",
    TOKdotti : "dotti",
    TOKdotexp : "dotexp",
    TOKdottype : "dottype",
    TOKslice : "..",
    TOKarraylength : "arraylength",
    TOKversion : "version",
    TOKmodule : "module",
    TOKdollar : "$",
    TOKtemplate : "template",
    TOKinstance : "instance",
    TOKdeclaration : "declaration",
    TOKtypeof : "typeof",
    TOKpragma : "pragma",
    TOKdsymbol : "dsymbol",
    TOKtypeid : "typeid",
    TOKuadd : "uadd",

    // Operators
    TOKlt : "<",
    TOKgt : ">",
    TOKle : "<=",
    TOKge : ">=",
    TOKequal : "==",
    TOKnotequal : "!=",
    TOKidentity : "===",
    TOKnotidentity : "!==",
    TOKindex : "[]",

    // NCEG floating point compares
    // !<>=     <>    <>=    !>     !>=   !<     !<=   !<>

    // NOTE:  These could be horribly wrong
    TOKunord : "!<>=",
    TOKue : "!<>",
    TOKlg : "<>",
    TOKleg : "<>=",
    TOKule : "!>",
    TOKul : "!>=",
    TOKuge : "!<",
    TOKug : "!<=",

    TOKshl : "<<",
    TOKshr : ">>",
    TOKshlass : "<<=",
    TOKshrass : ">>=",
    TOKushr : ">>>",
    TOKushrass : ">>>=",
    TOKcat : "~",
    TOKcatass : "~=",   // ~ ~=
    TOKadd : "+",
    TOKmin : "-",
    TOKaddass : "+=",
    TOKminass : "-=",
    TOKmul : "*",
    TOKdiv : "/",
    TOKmod : "%",
    TOKmulass : "*=",
    TOKdivass : "/=",
    TOKmodass : "%=",
    TOKand : "&",
    TOKor : "|",
    TOKxor : "^",
    TOKandass : "&=",
    TOKorass : "|=",
    TOKxorass : "^=",
    TOKassign : "=",
    TOKnot : "!",
    TOKtilde : "~",
    TOKplusplus : "++",
    TOKminusminus : "--",
    TOKdot : ".",
    TOKarrow : "->",
    TOKcomma : ",",
    TOKquestion : "?",
    TOKandand : "&&",
    TOKoror : "||",

    // Numeric literals
    TOKint32v : "int32v", TOKuns32v : "uns32v",
    TOKint64v : "int64v", TOKuns64v : "uns64v",
    TOKfloat32v : "float32v", TOKfloat64v : "float64v", TOKfloat80v : "float80v",
    TOKimaginary32v : "imaginary32v", TOKimaginary64v : "imaginary64v", TOKimaginary80v : "imaginary80v",

    // Char constants
    TOKcharv : "charv", TOKwcharv : "wcharv", TOKdcharv : "dcharv",

    // Leaf operators
    TOKidentifier : "identifier",   TOKstring : "string",
    TOKthis : "this",   TOKsuper : "super",

    // Basic types
    TOKvoid : "void",
    TOKint8 : "byte", TOKuns8 : "ubyte",
    TOKint16 : "short", TOKuns16 : "ushort",
    TOKint32 : "int", TOKuns32 : "uint",
    TOKint64 : "long", TOKuns64 : "ulong",
    TOKfloat32 : "float", TOKfloat64 : "double", TOKfloat80 : "real",
    TOKimaginary32 : "ifloat", TOKimaginary64 : "idouble", TOKimaginary80 : "ireal",
    TOKcomplex32 : "cfloat", TOKcomplex64 : "cdouble", TOKcomplex80 : "creal",
    TOKchar : "char", TOKwchar : "wchar", TOKdchar : "dchar", TOKbit : "bit",

    // Aggregates
    TOKstruct : "struct", TOKclass : "class", TOKinterface : "interface", TOKunion : "union", TOKenum : "enum", TOKimport : "import",
    TOKtypedef : "typedef", TOKalias : "alias", TOKoverride : "override", TOKdelegate : "delegate", TOKfunction : "function",
    TOKmixin : "mixin",

    TOKalign : "align", TOKextern : "extern", TOKprivate : "private", TOKprotected : "protected", TOKpublic : "public", TOKexport : "export",
    TOKstatic : "static", /*TOKvirtual : "virtual",*/ TOKfinal : "final", TOKconst : "const", TOKabstract : "abstract", TOKvolatile : "volatile",
    TOKdebug : "debug", TOKdeprecated : "deprecated", TOKin : "in", TOKout : "out", TOKinout : "inout",
    TOKauto : "auto", TOKpackage : "package",

    // Statements
    TOKif : "if", TOKelse : "else", TOKwhile : "while", TOKfor : "for", TOKdo : "do", TOKswitch : "switch",
    TOKcase : "case", TOKdefault : "default", TOKbreak : "break", TOKcontinue : "continue", TOKwith : "with",
    TOKsynchronized : "synchronized", TOKreturn : "return", TOKgoto : "goto", TOKtry : "try", TOKcatch : "catch", TOKfinally : "finally",
    TOKasm : "asm", TOKforeach : "foreach",

    // Contracts
    TOKbody : "body", TOKinvariant : "invariant",

    // Testing
    TOKunittest : "unittest",
];

class Identifier {
    int     value;
    char[]  string;

    this(char[] string, int value) {
        this.value = value;
        this.string = string;
    };
};

// A language token:
class Token {
    // The kind of token:
    uint    value;
    Token   next;

    // The value for the token:
    union {
        // The identifier or value of the token:
        Identifier  ident;

        char[]      ustring;

        // Integers
        int         int32value;
        uint        uns32value;
        long        int64value;
        ulong       uns64value;

        // Floats
        real        float80value;
    };

    char[] toString() {
        return toktostr[value];
    };

    static char[] toChars(uint value) {
        return toktostr[value];
    };
}

// Exception thrown during parsing of D language:
class DLexerException : Error {
    public:
        this(DLexer dlx, char[] msg) {
            // Construct an error message with the filename and line number:
            super(dlx.filename ~ "(" ~ format("%d", dlx.line) ~ ")" ~ ": " ~ msg);
        }
}

// Check for octal digit:
int isodigit(dchar x) {
    if (!isdigit(x) || (x == '8') || (x == '9')) return 0;
    return -1;
}

class Loc {
    public:
        int linnum;

        this() {
        }
}

// The D language lexer (tokenizer):
class DLexer {
        static uint[char[]] keywords;
        // Initialize the keyword->tokenvalue AA:
        static this() {
            // Add all the keywords' values into the AA:
            keywords["this"] = TOKthis;
            keywords["super"] = TOKsuper;
            keywords["assert"] = TOKassert;
            keywords["null"] = TOKnull;
            keywords["true"] = TOKtrue;
            keywords["false"] = TOKfalse;
            keywords["cast"] = TOKcast;
            keywords["new"] = TOKnew;
            keywords["delete"] = TOKdelete;
            keywords["throw"] = TOKthrow;
            keywords["module"] = TOKmodule;
            keywords["pragma"] = TOKpragma;
            keywords["typeof"] = TOKtypeof;
            keywords["typeid"] = TOKtypeid;

            keywords["template"] = TOKtemplate;
            keywords["instance"] = TOKinstance;

            keywords["void"] = TOKvoid;
            keywords["byte"] = TOKint8;
            keywords["ubyte"] = TOKuns8;
            keywords["short"] = TOKint16;
            keywords["ushort"] = TOKuns16;
            keywords["int"] = TOKint32;
            keywords["uint"] = TOKuns32;
            keywords["long"] = TOKint64;
            keywords["ulong"] = TOKuns64;
            keywords["float"] = TOKfloat32;
            keywords["double"] = TOKfloat64;
            keywords["real"] = TOKfloat80;

            keywords["bit"] = TOKbit;
            keywords["char"] = TOKchar;
            keywords["wchar"] = TOKwchar;
            keywords["dchar"] = TOKdchar;

            keywords["ifloat"] = TOKimaginary32;
            keywords["idouble"] = TOKimaginary64;
            keywords["ireal"] = TOKimaginary80;

            keywords["cfloat"] = TOKcomplex32;
            keywords["cdouble"] = TOKcomplex64;
            keywords["creal"] = TOKcomplex80;

            keywords["delegate"] = TOKdelegate;
            keywords["function"] = TOKfunction;

            keywords["is"] = TOKidentity;
            keywords["if"] = TOKif;
            keywords["else"] = TOKelse;
            keywords["while"] = TOKwhile;
            keywords["for"] = TOKfor;
            keywords["do"] = TOKdo;
            keywords["switch"] = TOKswitch;
            keywords["case"] = TOKcase;
            keywords["default"] = TOKdefault;
            keywords["break"] = TOKbreak;
            keywords["continue"] = TOKcontinue;
            keywords["synchronized"] = TOKsynchronized;
            keywords["return"] = TOKreturn;
            keywords["goto"] = TOKgoto;
            keywords["try"] = TOKtry;
            keywords["catch"] = TOKcatch;
            keywords["finally"] = TOKfinally;
            keywords["with"] = TOKwith;
            keywords["asm"] = TOKasm;
            keywords["foreach"] = TOKforeach;

            keywords["struct"] = TOKstruct;
            keywords["class"] = TOKclass;
            keywords["interface"] = TOKinterface;
            keywords["union"] = TOKunion;
            keywords["enum"] = TOKenum;
            keywords["import"] = TOKimport;
            keywords["mixin"] = TOKmixin;
            keywords["static"] = TOKstatic;
            /*keywords["virtual"] = TOKvirtual;*/
            keywords["final"] = TOKfinal;
            keywords["const"] = TOKconst;
            keywords["typedef"] = TOKtypedef;
            keywords["alias"] = TOKalias;
            keywords["override"] = TOKoverride;
            keywords["abstract"] = TOKabstract;
            keywords["volatile"] = TOKvolatile;
            keywords["debug"] = TOKdebug;
            keywords["deprecated"] = TOKdeprecated;
            keywords["in"] = TOKin;
            keywords["out"] = TOKout;
            keywords["inout"] = TOKinout;
            keywords["auto"] = TOKauto;

            keywords["align"] = TOKalign;
            keywords["extern"] = TOKextern;
            keywords["private"] = TOKprivate;
            keywords["package"] = TOKpackage;
            keywords["protected"] = TOKprotected;
            keywords["public"] = TOKpublic;
            keywords["export"] = TOKexport;

            keywords["body"] = TOKbody;
            keywords["invariant"] = TOKinvariant;
            keywords["unittest"] = TOKunittest;
            keywords["version"] = TOKversion;
        }

    protected:
        Token               token;
        Identifier[char[]]  identifiers;
        Loc                 loc;

    private:
        char[]  file;       // The input file
        uint    p;          // Current character

        char[]  wysiwygString(char tc) {
            char    c;
            char[]  s;
            uint    i;

            i = 0;
            ++p;
            s.length = 16;
            while (p < file.length) {
                c = file[p++];
                switch (c) {
                    case '\n':
                        ++line;
                        break;

                    case '\r':
                        if (file[p] == '\n')
                            continue;   // ignore
                        c = '\n';   // treat EndOfLine as \n character
                        ++line;
                        break;

                    case 0x1A:
                        error("unterminated string constant starting at %s", s);
                        return null;

                    case '"', '`':
                        if (c == tc) {
                            s.length = i;
                            return s;
                        }
                        break;

                    default:
                        break;
                }

                if (i == s.length) s.length = s.length * 2;
                s[i++] = c;
            }
            s.length = i;
            return s;
        }

        // Test this!
        char[]  hexString() {
            char    c;
            uint    n = 0, i = 0;
            char[]  s;
            ubyte   v;

            p++;
            s.length = 16;
            while (p < file.length) {
                c = file[p++];
                switch (c) {
                    case ' ', '\t', '\v', '\f':
                        continue;           // skip white space

                    case '\r':
                        if (file[p] == '\n')
                            continue;           // ignore
                        // Treat isolated '\r' as if it were a '\n'
                    case '\n':
                        ++line;
                        continue;

                    case 0x1A:
                        error("unterminated string constant starting at %s", s);
                        return null;

                    case '"':
                        if (n & 1) {
                            error("odd number (%d) of hex characters in hex string", n);
                            return null;
                        }
                        s.length = i;
                        return s;

                    default:
                        if (c >= '0' && c <= '9')
                            c -= '0';
                        else if (c >= 'a' && c <= 'f')
                            c -= 'a' - 10;
                        else if (c >= 'A' && c <= 'F')
                            c -= 'A' - 10;
                        else
                            error("non-hex character '%c'", c);
                        if (n & 1) {
                            v = (v << 4) | c;
                            if (i == s.length) s.length = s.length * 2;
                            s[i++] = v;
                        } else
                            v = c;
                        ++n;
                        break;
                }
            }
            return null;
        }

    public:
        char[]  filename;   // Filename
        uint    line;       // Current line

        // Initialize the lexer with the full source code as a string:
        this(char[] filename, char[] src) {<