From 19f0d670acc7f3dd947d36e12abe2de266eeac88 Mon Sep 17 00:00:00 2001 From: Nemo Date: Tue, 16 Jun 2020 02:16:37 +0530 Subject: [PATCH] [10] Tokenizer works on all files correctly --- README.md | 5 +- compiler/__init__.py | 18 +- projects/10/ArrayTest/Main.jackT.xml | 142 +++++++++++ projects/10/Square/Main.jackT.xml | 126 ++++++++++ projects/10/Square/SquareGame.jackT.xml | 315 ++++++++++++++++++++++++ 5 files changed, 597 insertions(+), 9 deletions(-) create mode 100644 projects/10/ArrayTest/Main.jackT.xml create mode 100644 projects/10/Square/Main.jackT.xml create mode 100644 projects/10/Square/SquareGame.jackT.xml diff --git a/README.md b/README.md index bbab3fd..c44d5b1 100644 --- a/README.md +++ b/README.md @@ -135,9 +135,8 @@ Final hack instruction set count in brackets as before. ### Tokenizer -- [ ] Square -- [ ] ExpressionLessSquare -- [ ] TestArray +- [x] Square +- [x] TestArray ### Parser (Compilation Engine) diff --git a/compiler/__init__.py b/compiler/__init__.py index fbde61b..867e0a7 100644 --- a/compiler/__init__.py +++ b/compiler/__init__.py @@ -8,7 +8,7 @@ class Token(Enum): SYMBOL = 2 IDENTIFIER = 3 INTEGERCONSTANT = 4 - STRING_CONST = 5 + STRINGCONSTANT = 5 UNKNOWN = 6 class Keyword(Enum): @@ -50,13 +50,16 @@ class JackTokenizer: elif re.compile("\d+").match(t): return Token.INTEGERCONSTANT elif re.compile("\".*\"").match(t): - return Token.STRING_CONST + return Token.STRINGCONSTANT else: return Token.IDENTIFIER pass def printable_token(self): - return escape(self.current_token(), True) + if self.tokenType() == Token.STRINGCONSTANT: + return self.current_token()[1:-1] + else: + return escape(self.current_token(), True) """ Returns the character which is the current token """ def symbol(self): @@ -113,9 +116,12 @@ class JackTokenizer: if len(line) == 0: return [] else: - regex = re.compile("(class|constructor|function|method|field|static|var|int|char|boolean|void|true|false|null|this|let|do|if|else|while|return)|(\(|\)|\[|\]|,|\+|-|;|<|>|=|~|&|{|}|\*|\/|\||\.)|\s+") - tokens = regex.split(line) - return [e.strip() for e in tokens if e != None and e.strip()!=''] + # Regex contains 3 parts: + # 1. Keywords + # 2. Symbols + # 3. Identifiers + regex = re.compile("(class|constructor|function|method|field|static|var|int|char|boolean|void|true|false|null|this|let|do|if|else|while|return|\(|\)|\[|\]|,|\+|-|;|<|>|=|~|&|{|}|\*|\/|\||\.|[a-zA-Z_]+\w*|\".*\")") + return [e.strip() for e in regex.split(line) if e != None and e.strip()!=''] def has_more_tokens(self): return self.ptr < len(self.tokens) diff --git a/projects/10/ArrayTest/Main.jackT.xml b/projects/10/ArrayTest/Main.jackT.xml new file mode 100644 index 0000000..a9f9b69 --- /dev/null +++ b/projects/10/ArrayTest/Main.jackT.xml @@ -0,0 +1,142 @@ + + class + Main + { + function + void + main + ( + ) + { + var + Array + a + ; + var + int + length + ; + var + int + i + , + sum + ; + let + length + = + Keyboard + . + readInt + ( + HOW MANY NUMBERS? + ) + ; + let + a + = + Array + . + new + ( + length + ) + ; + let + i + = + 0 + ; + while + ( + i + < + length + ) + { + let + a + [ + i + ] + = + Keyboard + . + readInt + ( + ENTER THE NEXT NUMBER: + ) + ; + let + i + = + i + + + 1 + ; + } + let + i + = + 0 + ; + let + sum + = + 0 + ; + while + ( + i + < + length + ) + { + let + sum + = + sum + + + a + [ + i + ] + ; + let + i + = + i + + + 1 + ; + } + do + Output + . + printString + ( + THE AVERAGE IS: + ) + ; + do + Output + . + printInt + ( + sum + / + length + ) + ; + do + Output + . + println + ( + ) + ; + return + ; + } + } + diff --git a/projects/10/Square/Main.jackT.xml b/projects/10/Square/Main.jackT.xml new file mode 100644 index 0000000..ce52bf7 --- /dev/null +++ b/projects/10/Square/Main.jackT.xml @@ -0,0 +1,126 @@ + + class + Main + { + static + boolean + test + ; + function + void + main + ( + ) + { + var + SquareGame + game + ; + let + game + = + SquareGame + . + new + ( + ) + ; + do + game + . + run + ( + ) + ; + do + game + . + dispose + ( + ) + ; + return + ; + } + function + void + test + ( + ) + { + var + int + i + , + j + ; + var + String + s + ; + var + Array + a + ; + if + ( + false + ) + { + let + s + = + string constant + ; + let + s + = + null + ; + let + a + [ + 1 + ] + = + a + [ + 2 + ] + ; + } + else + { + let + i + = + i + * + ( + - + j + ) + ; + let + j + = + j + / + ( + - + 2 + ) + ; + let + i + = + i + | + j + ; + } + return + ; + } + } + diff --git a/projects/10/Square/SquareGame.jackT.xml b/projects/10/Square/SquareGame.jackT.xml new file mode 100644 index 0000000..62d53f2 --- /dev/null +++ b/projects/10/Square/SquareGame.jackT.xml @@ -0,0 +1,315 @@ + + class + SquareGame + { + field + Square + square + ; + field + int + direction + ; + constructor + SquareGame + new + ( + ) + { + let + square + = + Square + . + new + ( + 0 + , + 0 + , + 30 + ) + ; + let + direction + = + 0 + ; + return + this + ; + } + method + void + dispose + ( + ) + { + do + square + . + dispose + ( + ) + ; + do + Memory + . + deAlloc + ( + this + ) + ; + return + ; + } + method + void + moveSquare + ( + ) + { + if + ( + direction + = + 1 + ) + { + do + square + . + moveUp + ( + ) + ; + } + if + ( + direction + = + 2 + ) + { + do + square + . + moveDown + ( + ) + ; + } + if + ( + direction + = + 3 + ) + { + do + square + . + moveLeft + ( + ) + ; + } + if + ( + direction + = + 4 + ) + { + do + square + . + moveRight + ( + ) + ; + } + do + Sys + . + wait + ( + 5 + ) + ; + return + ; + } + method + void + run + ( + ) + { + var + char + key + ; + var + boolean + exit + ; + let + exit + = + false + ; + while + ( + ~ + exit + ) + { + while + ( + key + = + 0 + ) + { + let + key + = + Keyboard + . + keyPressed + ( + ) + ; + do + moveSquare + ( + ) + ; + } + if + ( + key + = + 81 + ) + { + let + exit + = + true + ; + } + if + ( + key + = + 90 + ) + { + do + square + . + decSize + ( + ) + ; + } + if + ( + key + = + 88 + ) + { + do + square + . + incSize + ( + ) + ; + } + if + ( + key + = + 131 + ) + { + let + direction + = + 1 + ; + } + if + ( + key + = + 133 + ) + { + let + direction + = + 2 + ; + } + if + ( + key + = + 130 + ) + { + let + direction + = + 3 + ; + } + if + ( + key + = + 132 + ) + { + let + direction + = + 4 + ; + } + while + ( + ~ + ( + key + = + 0 + ) + ) + { + let + key + = + Keyboard + . + keyPressed + ( + ) + ; + do + moveSquare + ( + ) + ; + } + } + return + ; + } + } +