dopp0/source/dopp_lexer.d

import dopp;

// helper - is lexeme keyword
export bool isKeyword(string lexeme){
  static immutable keywords = ["if", "else", "while", "for", "return"];
  return keywords.canFind(lexeme);
}


// lexer - Tokenizer makes tokens from text
export Token[] tokenize(string source){
  Token[] tokens;
  int i = 0;
  while(i < source.length){
    
    if(source[i].isWhite){ // skip whitespaces
      i++;
    
    }else if(source[i].isAlpha || source[i] == '_'){
      auto start = i;
      while(i < source.length && (source[i].isAlphaNum || source[i] == '_')){
        i++;
      }
      string lexeme = source[start .. i];
      tokens ~= Token(lexeme.isKeyword ? TokenType.Keyword : TokenType.Identifier, lexeme);
    
    }else if(source[i].isDigit){
      auto start = i;
      while(i < source.length && source[i].isDigit){
        i++;
      }
      if(i < source.length && source[i] == '.'){ // include dot for float
        i++;
        while(i < source.length && source[i].isDigit){
          i++;
        }
        tokens ~= Token(TokenType.Float, source[start .. i]);
      
      }else{
        tokens ~= Token(TokenType.Integer, source[start .. i]);
      }
    
    }else if(source[i] == '"'){
      auto start = i++;
      while(i < source.length && source[i] != '"'){
        i++;
      }
      if(i < source.length){ // close quotes
        i++;
      }
      tokens ~= Token(TokenType.String, source[start .. i]);
    
    }else{ // common symbols as tokens
      tokens ~= Token(TokenType.Symbol, source[i].to!string);
      i++;
    }
  }
  return tokens;
}