add comments, underscore in numbers

source/dopp.d

@@ -21,7 +21,7 @@ TOMLDocument toml_s;
 
 
 export enum TokenType{
-  Identifier,
+  Identifier, // todo split to Type - Function - Variable
   Keyword,
   Integer,
   Float,
@@ -204,14 +204,24 @@ int main(string[] argv){
   
   
   //string source = `if (x > 0) { writeln("Positive"); }`;
+  /+
   string source = q"[
   if (x > 0) {
     writeln("Pos`it`i've'");
   }
   ]";
+  +/
+  
+  string source = q"[
+auto x = 5;
+auto x2 = 5_001;
+auto y = "this is test";
+auto z = 1;
+  ]";
+  
   auto tokens = tokenize(source);
   writeln(tokens);
-  parse(tokens);
+  //parse(tokens);
   
   
   

source/dopp_lexer.d

@@ -25,9 +25,9 @@ export Token[] tokenize(string source){
       string lexeme = source[start .. i];
       tokens ~= Token(lexeme.isKeyword ? TokenType.Keyword : TokenType.Identifier, lexeme);
     
-    }else if(source[i].isDigit){
+    }else if(source[i].isDigit){ // number
       auto start = i;
-      while(i < source.length && source[i].isDigit){
+      while(i < source.length && (source[i].isDigit || source[i] == '_')){ // underscore can be inside number like 5_000 etc
         i++;
       }
       if(i < source.length && source[i] == '.'){ // include dot for float
@@ -41,9 +41,9 @@ export Token[] tokenize(string source){
         tokens ~= Token(TokenType.Integer, source[start .. i]);
       }
     
-    }else if(source[i] == '"'){
-      auto start = i++;
-      while(i < source.length && source[i] != '"'){
+    }else if(source[i] == '"'){ // type string begins
+      auto start = i++; // string begin position
+      while(i < source.length && source[i] != '"'){ // goto type string end position
         i++;
       }
       if(i < source.length){ // close quotes