From 842c344ef10c58559266ba6e179e2aaf12a520ab Mon Sep 17 00:00:00 2001
From: Tiago Sousa <tsousa@DESKTOP-PH9JJHB>
Date: Sun, 30 Apr 2023 23:09:21 +0100
Subject: [PATCH] [LEX] Token definition

---
 src/lexer.py | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)
 create mode 100644 src/lexer.py

diff --git a/src/lexer.py b/src/lexer.py
new file mode 100644
index 0000000..b98974a
--- /dev/null
+++ b/src/lexer.py
@@ -0,0 +1,28 @@
+import ply.lex as lex
+
+tokens = [
+    "ID",
+    "STR"
+    "DATE",
+    "TIME",
+    "DATETIME",
+    "INT",
+    "FLOAT", # need to implement exponents check https://toml.io/en/
+    "HEX",
+    "BIN",
+    "OCT",
+    "INF",
+    "NAN",
+    "COMMENT",
+]
+
+def t_COMMENT(t):
+    r"#.*"
+    pass
+
+t_ignore = "\n\t "
+
+literals = "[]{},="
+
+
+lexer = lex.lex()