Skip to content

Commit

Permalink
Compiler now supports UTF-8. Polishing editor again
Browse files Browse the repository at this point in the history
  • Loading branch information
@VonTum
VonTum committed Aug 13, 2023
1 parent d18d935 commit ff85f67
Show file tree
Hide file tree
Showing 6 changed files with 151 additions and 103 deletions.
9 changes: 5 additions & 4 deletions multiply_add.sus
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,7 @@ module ExactlyOne : bool[Size] inputs -> bool exactlyOne {

module multiply_add2 : int a, int b, int c -> int r {
int tmp = a * b;
@
r = tmp + c;
reg r = tmp + c;
}


Expand All @@ -45,6 +44,8 @@ module parallel_mul_add_reg : int a, int b -> int p, int q {
reg int b2 = b + b;
reg int b3 = b + b2;
reg q = b + b3;


}

/* a module to test the syntax */
Expand All @@ -66,6 +67,8 @@ module MultiplyAdd : i32 a, i32 b, i32 c -> i32 result {
//[
beep boop;
}

Lööwe 老虎 Léopard;
}

module beep : i32 a {
Expand Down Expand Up @@ -128,5 +131,3 @@ module seq_adder : i32 a -> i32 result {
}
result = sum;
}


94 changes: 64 additions & 30 deletions src/ast.rs
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@

use num_bigint::BigUint;

use crate::{tokenizer::{TokenTypeIdx, TokenExtraInfo}, errors::{ParsingError, error_basic_str}};
use crate::tokenizer::{TokenTypeIdx, TokenExtraInfo};
use core::ops::Range;

use std::collections::HashMap;
Expand All @@ -20,16 +20,16 @@ impl Span {

#[derive(Debug,Clone,Copy,PartialEq,Eq)]
pub struct FilePos {
pub char_idx : usize,
pub char_idx : usize, // Byte index
pub row : usize,
pub col : usize
pub col : usize // Char index
}

// Char span, for chars in file. start is INCLUSIVE, end is EXCLUSIVE. It's a bit weird to make the distinction, but it works out
#[derive(Clone,Copy,Debug,PartialEq,Eq)]
pub struct CharSpan{
pub file_pos : FilePos,
pub length : usize
pub length : usize // in bytes. Can just do file_text[file_pos.char_idx .. file_pos.char_idx = length]
}


Expand Down Expand Up @@ -113,7 +113,7 @@ pub struct IdentifierToken {

#[derive(Debug, Clone)]
pub enum TypeExpression {
Named(usize),
Named(u64),
Array(Box<(SpanTypeExpression, SpanExpression)>)
}

Expand Down Expand Up @@ -155,9 +155,14 @@ impl Expression {
}
}
pub type SpanExpression = (Expression, Span);
pub type SpanAssignableExpression = (AssignableExpression, Span);
pub type SpanStatement = (Statement, Span);

pub type SpanAssignableExpression = SpanExpression;
#[derive(Debug)]
pub enum AssignableExpression {
Named{local_idx : usize, num_regs : usize},
ArrayIndex(Box<(SpanAssignableExpression, SpanExpression)>)
}

#[derive(Debug)]
pub enum Statement {
Expand All @@ -181,6 +186,7 @@ pub struct ASTRoot {

pub trait IterIdentifiers {
fn for_each_value<F>(&self, func : &mut F) where F : FnMut(IdentifierIdx, usize) -> ();
fn for_each_type<F>(&self, func : &mut F) where F : FnMut(TokenExtraInfo, usize) -> ();
}

impl IterIdentifiers for SpanExpression {
Expand Down Expand Up @@ -213,6 +219,25 @@ impl IterIdentifiers for SpanExpression {
}
}
}
fn for_each_type<F>(&self, _func : &mut F) where F : FnMut(TokenExtraInfo, usize) -> () {}
}

impl IterIdentifiers for SpanAssignableExpression {
fn for_each_value<F>(&self, func : &mut F) where F : FnMut(IdentifierIdx, usize) -> () {
let (expr, span) = self;
match expr {
AssignableExpression::Named{local_idx: id, num_regs : _} => {
assert!(span.0 == span.1);
func(IdentifierIdx::new_local(*id), span.0)
}
AssignableExpression::ArrayIndex(b) => {
let (array, idx) = &**b;
array.for_each_value(func);
idx.for_each_value(func);
}
}
}
fn for_each_type<F>(&self, _func : &mut F) where F : FnMut(TokenExtraInfo, usize) -> () {}
}

impl IterIdentifiers for SpanTypeExpression {
Expand All @@ -229,34 +254,52 @@ impl IterIdentifiers for SpanTypeExpression {
}
}
}
fn for_each_type<F>(&self, func : &mut F) where F : FnMut(TokenExtraInfo, usize) -> () {
let (typ, span) = self;
match typ {
TypeExpression::Named(n) => {
func(*n, span.1)
}
TypeExpression::Array(b) => {
let (arr_typ, arr_size) = &**b;
arr_typ.for_each_type(func);
arr_size.for_each_type(func);
}
}
}
}

pub fn for_each_expression_in_block<F>(block : &Vec<SpanStatement>, func : &mut F) where F: FnMut(&SpanExpression) {
pub fn for_each_assign_in_block<F>(block : &Vec<SpanStatement>, func : &mut F) where F: FnMut(&Vec<SpanAssignableExpression>, &SpanExpression) {
for (stmt, _span) in block {
match stmt {
Statement::Assign(to, v) => {
for t in to {
func(t);
}
func(v);
func(to, v);
},
Statement::Block(b) => {
for_each_expression_in_block(b, func);
for_each_assign_in_block(b, func);
},
_other => {}
}
}
}

pub fn for_each_expression_in_module<F>(m : &Module, func : &mut F) where F : FnMut(&SpanExpression) {
for (idx, d) in m.declarations.iter().enumerate() {
/*if d.identifier_type != IdentifierType::Input && d.identifier_type != IdentifierType::Output {
break;
}*/ // Allow potential duplicates for locals
let local_expr = (Expression::Named(IdentifierIdx::new_local(idx)), Span::from(d.span.1));
func(&local_expr);
impl IterIdentifiers for Module {
fn for_each_value<F>(&self, func : &mut F) where F : FnMut(IdentifierIdx, usize) -> () {
for (pos, decl) in self.declarations.iter().enumerate() {
func(IdentifierIdx::new_local(pos), decl.span.1);
}
for_each_assign_in_block(&self.code, &mut |to, v| {
for assign_to in to {
assign_to.for_each_value(func);
}
v.for_each_value(func);
});
}
fn for_each_type<F>(&self, func : &mut F) where F : FnMut(TokenExtraInfo, usize) -> () {
for decl in &self.declarations {
decl.typ.for_each_type(func);
}
}
for_each_expression_in_block(&m.code, func);
}

pub struct GlobalContext {
Expand All @@ -265,15 +308,6 @@ pub struct GlobalContext {
}

impl GlobalContext {
pub fn parse_to_type((expr, span) : &SpanExpression, errors : &mut Vec<ParsingError<Span>>) -> Option<TypeExpression> {
match expr {
Expression::Named(idx) => {todo!();},
Expression::Array(args) => {todo!();},
other => {
errors.push(error_basic_str(*span, "Unexpected part"));
return None
}
}
}

}

45 changes: 22 additions & 23 deletions src/dev_aid/lsp.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,12 +142,13 @@ fn get_semantic_token_type_from_ide_token(tok : &IDEToken) -> u32 {
IDETokenType::Comment => 0,
IDETokenType::Keyword => 1,
IDETokenType::Operator => 2,
IDETokenType::TimelineStage => 7,// EVENT seems to get a good colour
IDETokenType::TimelineStage => 8,// EVENT seems to get a good colour
IDETokenType::Identifier(IDEIdentifierType::Value(IdentifierType::Input)) => 4,
IDETokenType::Identifier(IDEIdentifierType::Value(IdentifierType::Output)) => 4,
IDETokenType::Identifier(IDEIdentifierType::Value(IdentifierType::State)) => 3,
IDETokenType::Identifier(IDEIdentifierType::Value(IdentifierType::Local)) => 3,
IDETokenType::Identifier(IDEIdentifierType::Unknown) => 2, // make it 'OPERATOR'?
IDETokenType::Identifier(IDEIdentifierType::Interface) => 7, // FUNCTION
IDETokenType::Identifier(_) => 5, // All others are 'TYPE'
IDETokenType::Number => 6,
IDETokenType::Invalid => 2, // make it 'OPERATOR'?
Expand Down Expand Up @@ -202,38 +203,36 @@ fn do_syntax_highlight(file_data : &LoadedFile, full_parse : &FullParseResult) -

for (idx, tok) in ide_tokens.iter().enumerate() {
let tok_file_pos = full_parse.tokens.token_spans[idx];
let token_text = &file_text[tok_file_pos.as_range()];
let char_iter = token_text.chars();

let typ = get_semantic_token_type_from_ide_token(tok);
let mod_bits = get_modifiers_for_token(tok);
if tok.typ == IDETokenType::Comment {
// Comments can be multiline, editor doesn't support this. Have to split them up myself. Eurgh
let mut comment_piece_start = tok_file_pos.file_pos.char_idx;
let mut char_iter = file_text.char_indices();
let mut line_char_offset = 0;
let mut line = tok_file_pos.file_pos.row;
let mut col = tok_file_pos.file_pos.col;
char_iter.nth(tok_file_pos.file_pos.char_idx);
for _pos in 0..tok_file_pos.length {
if let Some((idx, c)) = char_iter.next() {
if c == '\n' {
semantic_tokens_acc.push(line, col, idx - comment_piece_start, typ, mod_bits);

comment_piece_start = idx + 1;
line += 1;
col = 0;
}
} else {
break;
let mut length_in_chars : usize = 0;
for (idx, c) in char_iter.enumerate() {
length_in_chars += 1;
if c == '\n' {
semantic_tokens_acc.push(line, col, idx - line_char_offset, typ, mod_bits);

line_char_offset = idx + 1;
line += 1;
col = 0;
}
}
let leftover_length = tok_file_pos.file_pos.char_idx + tok_file_pos.length - comment_piece_start;
let leftover_length = length_in_chars - line_char_offset;
if leftover_length > 0 {
semantic_tokens_acc.push(line, col, leftover_length, typ, mod_bits);
}
} else {
semantic_tokens_acc.push(tok_file_pos.file_pos.row, tok_file_pos.file_pos.col, tok_file_pos.length, typ, mod_bits);
semantic_tokens_acc.push(tok_file_pos.file_pos.row, tok_file_pos.file_pos.col, char_iter.count(), typ, mod_bits);
}

//println!("{}: typ={typ} {delta_line}:{delta_col}", file_text.get(tok_file_pos.as_range()).unwrap());
//println!("{}: typ={typ} {delta_line}:{delta_col}", file_text[tok_file_pos.as_range()]);
}

SemanticTokensResult::Tokens(lsp_types::SemanticTokens {
Expand All @@ -246,15 +245,15 @@ use lsp_types::Diagnostic;

fn cvt_char_span_to_lsp_range(ch_sp : CharSpan, file_text : &str) -> lsp_types::Range {
let mut last_char_line = ch_sp.file_pos.row;
let mut last_newline_idx = ch_sp.file_pos.char_idx - ch_sp.file_pos.col;
let last_char_idx = ch_sp.file_pos.char_idx+ch_sp.length;
for (i, c) in file_text.get(ch_sp.file_pos.char_idx..last_char_idx).unwrap().char_indices() {
let mut last_char_col = ch_sp.file_pos.col;
for c in file_text[ch_sp.file_pos.char_idx..ch_sp.file_pos.char_idx + ch_sp.length].chars() {
if c == '\n' {
last_char_line += 1;
last_newline_idx = i;
last_char_col = 0;
} else {
last_char_col += 1;
}
}
let last_char_col = last_char_idx - last_newline_idx;
Range{
start : Position{
line : ch_sp.file_pos.row as u32,
Expand Down
33 changes: 14 additions & 19 deletions src/dev_aid/syntax_highlighting.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,9 @@ pub struct IDEToken {
}

fn pretty_print_chunk_with_whitespace(whitespace_start : usize, file_text : &str, text_span : CharSpan, st : Style) {
let whitespace_text = file_text.get(whitespace_start..text_span.file_pos.char_idx).unwrap();
let whitespace_text = &file_text[whitespace_start..text_span.file_pos.char_idx];

print!("{}{}", whitespace_text, st.apply_to(file_text.get(text_span.as_range()).unwrap()));
print!("{}{}", whitespace_text, st.apply_to(&file_text[text_span.as_range()]));
}

fn print_tokens(file_text : &str, token_spans : &[CharSpan]) {
Expand All @@ -47,7 +47,7 @@ fn print_tokens(file_text : &str, token_spans : &[CharSpan]) {
whitespace_start = tok_span.end_pos();
}

print!("{}\n", file_text.get(whitespace_start..file_text.len()).unwrap());
print!("{}\n", &file_text[whitespace_start..file_text.len()]);
}

fn pretty_print(file_text : &str, token_spans : &[CharSpan], ide_infos : &[IDEToken]) {
Expand All @@ -66,7 +66,7 @@ fn pretty_print(file_text : &str, token_spans : &[CharSpan], ide_infos : &[IDETo
IDETokenType::Identifier(IDEIdentifierType::Value(IdentifierType::Input)) => Style::new().blue().bright(),
IDETokenType::Identifier(IDEIdentifierType::Value(IdentifierType::Output)) => Style::new().blue().dim(),
IDETokenType::Identifier(IDEIdentifierType::Type) => Style::new().magenta().bright(),
IDETokenType::Identifier(IDEIdentifierType::Interface) => Style::new().magenta().dim(),
IDETokenType::Identifier(IDEIdentifierType::Interface) => Style::new().yellow(),
IDETokenType::Number => Style::new().green().bright(),
IDETokenType::Invalid | IDETokenType::InvalidBracket => Style::new().red().underlined(),
IDETokenType::OpenBracket(depth) | IDETokenType::CloseBracket(depth) => {
Expand All @@ -79,7 +79,7 @@ fn pretty_print(file_text : &str, token_spans : &[CharSpan], ide_infos : &[IDETo
whitespace_start = tok_span.end_pos();
}

print!("{}\n", file_text.get(whitespace_start..file_text.len()).unwrap());
print!("{}\n", &file_text[whitespace_start..file_text.len()]);
}

fn add_ide_bracket_depths_recursive<'a>(result : &mut [IDEToken], current_depth : usize, token_hierarchy : &[TokenTreeNode]) {
Expand All @@ -94,22 +94,17 @@ fn add_ide_bracket_depths_recursive<'a>(result : &mut [IDEToken], current_depth

fn walk_name_color(ast : &ASTRoot, result : &mut [IDEToken]) {
for module in &ast.modules {
for decl in &module.declarations {
decl.typ.for_each_value(&mut |_name, position| {
result[position].typ = IDETokenType::Identifier(IDEIdentifierType::Type);
});
//result[decl.name.position].typ = IDETokenType::Identifier(IDEIdentifierType::Value(decl.identifier_type));
}

for_each_expression_in_module(&module, &mut |expr| {
expr.for_each_value(&mut |name, position| {
result[position].typ = IDETokenType::Identifier(if let Some(l) = name.get_local() {
IDEIdentifierType::Value(module.declarations[l].identifier_type)
} else {
IDEIdentifierType::Unknown
});
module.for_each_type(&mut |_name, position| {
result[position].typ = IDETokenType::Identifier(IDEIdentifierType::Type);
});
module.for_each_value(&mut |name, position| {
result[position].typ = IDETokenType::Identifier(if let Some(l) = name.get_local() {
IDEIdentifierType::Value(module.declarations[l].identifier_type)
} else {
IDEIdentifierType::Unknown
});
});
result[module.name.position].typ = IDETokenType::Identifier(IDEIdentifierType::Interface);
}
}

Expand Down
Loading

0 comments on commit ff85f67

Please sign in to comment.