diff --git a/src/components/Footer.astro b/src/components/Footer.astro index 0411db1..281aa33 100644 --- a/src/components/Footer.astro +++ b/src/components/Footer.astro @@ -6,7 +6,7 @@ const columns = [ title: "Projects", links: [ { label: "Comprehensive Config", href: "/projects/comprehensiveconfig" }, - { label: "Compiler Toolkit", href: "#" }, + { label: "Compiler Toolkit", href: "/projects/compilertoolkit" }, ], }, { @@ -22,7 +22,7 @@ const columns = [ links: [ { label: "ComprehensiveConfig Docs", - href: "https://github.com/Summersweet-Software/ComprehensiveConfig", + href: "https://comprehensiveconfig.readthedocs.io/", }, ], }, diff --git a/src/pages/index.astro b/src/pages/index.astro index 0c1ce34..e9b2bf7 100644 --- a/src/pages/index.astro +++ b/src/pages/index.astro @@ -10,7 +10,7 @@ const projects = [ { name: "Compiler Toolkit", description: "A toolkit for building compilers and language tooling. Lexers, parsers, and code generation.", - href: "#", + href: "/projects/compilertoolkit", }, ]; --- diff --git a/src/pages/projects/compilertoolkit.md b/src/pages/projects/compilertoolkit.md new file mode 100644 index 0000000..5b1e1d3 --- /dev/null +++ b/src/pages/projects/compilertoolkit.md @@ -0,0 +1,348 @@ +--- +layout: ../../layouts/Project.astro +title: Compiler Toolkit +description: An opinionated library for creating parsers and compilers in python. +tech: Python +github: https://github.com/Summersweet-Software/compilertoolkit +--- + +## Overview + +CompilerToolkit is a library designed to help you create well-organized, large compilers in python. It helps with ensuring proper annotations and typing support throughout your entire codebase. +It also provides many utilities for pretty error printing and tree handling. + +## Release/Version Information + +Currently this project is unfinished and does not have a `v1.0` on pypi or github. + +## Features + +- [x] Ast Creation Tools +- [x] Ast Walk functionality built in +- [x] Ast node typing garauntees (ensure that all nodes are well defined) +- [x] Ast nodes having configurable parser patterns (WIP, needs to be more ergonomic) +- [x] Utility decorators for annotation of individual parts of the compilers +- [x] Parser pattern builder + - [ ] unions on patterns (`or` clause) + - [ ] `and` clause on patterns +- [ ] Parser builder (WIP, needs to be more ergonomic) +- [ ] Parser check functions built into patterns to allow automatic syntax error parsing. +- [x] Source error highlighting (fine grained highlights) +- [x] Package and module tree utilities +- [x] Lexing via rply library (and utilities) +- [x] Parser token class builtin +- [x] `NTree` utility type to help make things like module/package trees + +## Example usage of `NTree`'s + +```python +class ModuleName: + ... # implemented in full example on github + +class Module: + ... # implemented in full example on github + +class ModuleView: + ... # implemented in full example on github + +class Package(NTree[ModuleView, str]): + """An example of an extension of NTree""" + ... # implemented in full example on github + + +imports = Package( + identifier="base", + leaves=[ + Module("main", private=True), + Module("other_mod"), + Package( + identifier="lib", + leaves=[ + Module("math"), + Module("system"), + Module("err"), + Package( + identifier="ui", + leaves=[ + Module("application"), + Module("widgets"), + Module("internal_stuff", private=True), + Package( + identifier="bindings", + leaves=[ + Module("application", private=True), + Module("widgets", private=True), + ], + ), + ], + ), + ], + ), + ], +) + +# NTree supports: +# - intersection via `&` operator +# - combining via `|` op +# - custom leaf initialization (useful for those ModuleView classes) +# - Two kinds of indexing (NTree indexing and Leaf node indexing) via the index operator +# - Comparison via `==` +# - arbitrary tree/leaf matching (via common interface/protocol) +# - Fully supported type annotation that work fantastically (avoiding bare type unions and instead using overloads to provide more accurate typing when calling a function or using an operator) + + +# Setup trees to do testing against +# =================================== + +trying_to_import = NTree[ModuleName, str]( + identifier="base", leaves=[ModuleName("other_mod")] +) + +trying_to_import_2 = NTree[ModuleName, str]( + identifier="base", + leaves=[NTree(identifier="lib", leaves=[ModuleName("math")])], +) + +trying_to_import_partial = NTree[ModuleName, str]( + identifier="base", + leaves=[ + NTree(identifier="lib", leaves=[ModuleName("something_that_does_not_exist")]) + ], +) +# Test Basic overlapping +# ======================== +assert imports.overlaps(trying_to_import) +assert imports.overlaps(trying_to_import_2) +assert not imports.overlaps(trying_to_import_partial) + + +# Test Basic intersection +# ========================= + +print() +print(imports & trying_to_import) +print(imports & trying_to_import_2) +print(imports & trying_to_import_partial) + +assert (imports & trying_to_import).children == [imports[ModuleName("other_mod")]] +assert (imports & trying_to_import_2).children == [ + imports["lib"].copy().set_leaves((imports["lib"]["math"],)) +] +assert (imports & trying_to_import_partial).children == [ + imports["lib"].copy().set_leaves([]) +] + +# Test Basic combining +# ====================== +print() + +trying_to_import_resolved = imports & trying_to_import +trying_to_import_2_resolved = imports & trying_to_import_2 +trying_to_import_partial_resolved = Package( + identifier="base", + leaves=[ + Package(identifier="lib", leaves=[Module("something_that_does_not_exist")]) + ], +) + +print(imports | trying_to_import_resolved) +print(imports | trying_to_import_2_resolved) +print(imports | trying_to_import_partial_resolved) + +assert (imports | trying_to_import_resolved) == imports +assert (imports | trying_to_import_2_resolved) == imports +changed_tree = imports.copy() +changed_tree["lib"] = ( + changed_tree["lib"] + .copy() + .add_leaf( + trying_to_import_partial_resolved["lib"][ + ModuleName("something_that_does_not_exist") + ] + ) +) +assert (imports | trying_to_import_partial_resolved) == changed_tree + +# Test Basic tree indexing +# ========================== +print() + +lib_pkg = imports["lib"] +print(lib_pkg) +assert isinstance(lib_pkg, Package) and lib_pkg.identifier == "lib" + +# Test private imports +# ###################### +try: + imports[ModuleName("main")] # should not work + raise Exception("Expected an error") +except PrivateImportExc: + main_mod = imports[ModuleName("main", parent=imports)] + print(main_mod) + assert isinstance(main_mod, ModuleView) and main_mod.mod.name == "main" + +``` + +## Test usage of AST utilities + +```python +from typing import Any +from compilertoolkit.ast import ( + AbstractAstNode, + abstractcompilationstep, + compilationstep, +) +from compilertoolkit.exceptions import ( + CompilerError, + ParsingError, + UnexpectedToken, + ansi_highlight_line, + create_underline, + format_file_position, +) +from compilertoolkit.parsing import ParseThenCheck, Parser, ParsingPattern, TokenHasType +from compilertoolkit.tokens import ( + Ignore, + Source, + SourcePosition, + TokenEnum, + TokenType, + create_lexer, +) + + +class Token[T](TokenEnum[T]): + """Part of our own Stuff""" + + Comma: TokenType[str] = TokenType(pattern=r"\,") + Number: TokenType[str] = TokenType(pattern=r"\d+") + Keyword: TokenType[str] = TokenType(pattern=r"\w+") + Plus: TokenType[str] = TokenType(pattern=r"\+") + + Expression: TokenType["ExpressionNode"] = TokenType() + Statement: TokenType["AstNode"] = TokenType() + EOF: TokenType[None] = TokenType() + + whitespace = Ignore(r"\s+") # ignore all whitespace + + +class AstNode(AbstractAstNode): + """Basic Ast Node""" + + __slots__ = () + + @abstractcompilationstep(0) + def analyze_types(self, ctx): + pass + + @abstractcompilationstep(1) + def compile(self, ctx) -> Any: + pass + + +class ExpressionNode(AstNode): + """Basic Ast Node""" + + __slots__ = "return_type" + + # instance variables + return_type: None | type # Your own type class + + +class NumberLiteral(ExpressionNode): + """Basic Ast Node""" + + __slots__ = "value" + + class ParserPattern(ParsingPattern, token_type=Token.Expression): + value = TokenHasType(Token.Number) + + # instance variables + value: int + + def __init__(self, tokens: ParserPattern): + super().__init__(tokens) + self.value = int(tokens.value.value) + + @compilationstep + def analyze_types(self, ctx): + self.return_type = int + + @compilationstep + def compile(self, ctx): + return self.value + + +class SumNode(ExpressionNode): + """Basic Ast Node""" + + __slots__ = ("lhs", "rhs") + + class ParserPattern(ParsingPattern, token_type=Token.Expression, precedence=1): + lhs = TokenHasType(Token.Expression) + operation = TokenHasType(Token.Plus) + # Parses, then checks for the specified case, errors if the value of the token is unparsed + rhs = ParseThenCheck( + TokenHasType(Token.Expression), + err_on_false="Expected expression on right hand side", + ) + + # instance variables + lhs: ExpressionNode + rhs: ExpressionNode + + def __init__(self, tokens: ParserPattern): + super().__init__(tokens) + self.rhs = tokens.rhs.value + self.lhs = tokens.lhs.value + + @compilationstep + def analyze_types(self, ctx): + self.lhs.analyze_types(ctx) + self.rhs.analyze_types(ctx) + self.return_type = int + + @compilationstep + def compile(self, ctx): + return self.lhs.compile(ctx) + self.rhs.compile(ctx) + + +def print_error(e: CompilerError, title: str): + print(f"""{title}: +| {e.msg} +| {f"line: {format_file_position(e.positions[0])}" if e.positions[0].source.filename else ""} +Line: +| {e.positions[0].line-1}. {ansi_highlight_line(source.lines[e.positions[0].line-1], e.positions)} +| {e.positions[0].line-1}. {create_underline(source.lines[e.positions[0].line-1], e.positions, e.pattern_position)} +""") + exit(1) + + +source = Source(" borger 8 + gaming", filename="borger.json") + +lexer = create_lexer(Token) +tokens = lexer.lex(source) +EOF = Token.EOF(SourcePosition(-1, -1, -1, -1, source), None) +try: + parser = Parser(EOF) + parser.add_rule(NumberLiteral.ParserPattern)\ + .add_rule(SumNode.ParserPattern) + parsed_tokens = parser.parse(tokens, 0, 0) + if any(not isinstance(tok.value, AstNode) for tok in parsed_tokens): + raise ParsingError( + [ + tok.position + for tok in parsed_tokens + if not isinstance(tok.value, AstNode) + ], + "Unexpected Token", + ) + + parsed_tokens[0].value.analyze_types({}) + +except UnexpectedToken as e: + print_error(e, "Unexpected Token") + +except CompilerError as e: + print_error(e, "Error") +``` diff --git a/src/pages/projects/rply-stubs.md b/src/pages/projects/rply-stubs.md new file mode 100644 index 0000000..b16f03e --- /dev/null +++ b/src/pages/projects/rply-stubs.md @@ -0,0 +1,35 @@ +--- +layout: ../../layouts/Project.astro +title: RPLY-stubs +description: A type stubs library for the popular `rply` python package. +tech: Python +github: https://github.com/Summersweet-Software/rply-stubs +--- + +## Overview + +A stubs library for [RPLY](https://pypi.org/project/rply/). + +## Correctness + +This is an independent static analysis of the types being used. The original rply codebase is quite messy. +This is all to say that the type annotations provided are not always correct but provide a firm guideline in an otherwise untyped library. + +## Completeness + +| symbol | meaning | +| ------ | ------------- | +| ⚪ | partial | +| ❌ | Not worked on | +| ✅ | completed | + +| Module | Status | +| --------------- | :----: | +| errors | ✅ | +| grammar | ⚪ | +| lexer | ✅ | +| lexergenerator | ✅ | +| parser | ✅ | +| parsergenerator | ⚪ | +| token | ✅ | +| utils | ✅ |