[LEX] Token definition

src/lexer.py

@@ -0,0 +1,28 @@
+import ply.lex as lex
+
+tokens = [
+    "ID",
+    "STR"
+    "DATE",
+    "TIME",
+    "DATETIME",
+    "INT",
+    "FLOAT", # need to implement exponents check https://toml.io/en/
+    "HEX",
+    "BIN",
+    "OCT",
+    "INF",
+    "NAN",
+    "COMMENT",
+]
+
+def t_COMMENT(t):
+    r"#.*"
+    pass
+
+t_ignore = "\n\t "
+
+literals = "[]{},="
+
+
+lexer = lex.lex()