Allow reading with preserving source location in syntax objects

This commit is contained in:
Konstantin Nazarov 2024-09-06 23:32:05 +01:00
parent aa1d8d7fa9
commit 4b7d845757
Signed by: knazarov
GPG key ID: 4CFE0A42FA409C22
6 changed files with 165 additions and 31 deletions

View file

@ -110,8 +110,8 @@ Result<short> SrcLoc::cmp(const SrcLoc& rhs) const {
return sourcerange_cmp(lhs_sourcerange, rhs_sourcerange); return sourcerange_cmp(lhs_sourcerange, rhs_sourcerange);
} }
Result<Syntax> Syntax::create(String filename, SrcLoc srcloc, Result<Syntax> Syntax::create(const String& filename, const SrcLoc& srcloc,
Value expression) { const Value& expression) {
auto pod = TRY(arena_alloc<PodSyntax>()); auto pod = TRY(arena_alloc<PodSyntax>());
pod->header.tag = Tag::Syntax; pod->header.tag = Tag::Syntax;
pod->filename = filename.pod(); pod->filename = filename.pod();
@ -121,6 +121,15 @@ Result<Syntax> Syntax::create(String filename, SrcLoc srcloc,
return Syntax(TRY(MkGcRoot(pod))); return Syntax(TRY(MkGcRoot(pod)));
} }
Result<Syntax> Syntax::create(const String& filename,
const SourcePosition& start,
const SourcePosition& end,
const Value& expression) {
auto srcloc = TRY(SrcLoc::create(start, end));
return Syntax::create(filename, srcloc, expression);
}
Result<Value> Syntax::filename() const { Result<Value> Syntax::filename() const {
return Value::create(_value->filename.get()); return Value::create(_value->filename.get());
} }

View file

@ -9,6 +9,7 @@
#include "error.hpp" #include "error.hpp"
#include "opcode.hpp" #include "opcode.hpp"
#include "pod.hpp" #include "pod.hpp"
#include "sourcerange.hpp"
#include "stdlib.hpp" #include "stdlib.hpp"
#include "utf8.hpp" #include "utf8.hpp"
@ -660,6 +661,17 @@ class SrcLoc : public Object {
return SrcLoc(TRY(MkGcRoot(obj))); return SrcLoc(TRY(MkGcRoot(obj)));
} }
static Result<SrcLoc> create(const SourcePosition& start,
const SourcePosition& end) {
auto pod = TRY(arena_alloc<PodSrcLoc>());
pod->header.tag = Tag::SrcLoc;
pod->sourcerange.start = start;
pod->sourcerange.end = end;
return SrcLoc(TRY(MkGcRoot(pod)));
}
SourceRange sourcerange() const; SourceRange sourcerange() const;
virtual Result<Value> copy_value() const final; virtual Result<Value> copy_value() const final;
@ -674,8 +686,12 @@ class Syntax : public Object {
Syntax() {} Syntax() {}
Syntax(Syntax&& rhs) : _value(std::move(rhs._value)) {} Syntax(Syntax&& rhs) : _value(std::move(rhs._value)) {}
Syntax(GcRoot<PodSyntax>&& val) : _value(std::move(val)) {} Syntax(GcRoot<PodSyntax>&& val) : _value(std::move(val)) {}
static Result<Syntax> create(String filename, SrcLoc srcloc, static Result<Syntax> create(const String& filename, const SrcLoc& srcloc,
Value expression); const Value& expression);
static Result<Syntax> create(const String& filename,
const SourcePosition& start,
const SourcePosition& end,
const Value& expression);
virtual Tag tag() const final { return Tag::Syntax; } virtual Tag tag() const final { return Tag::Syntax; }
virtual PodObject* pod() const final { return _value.get(); } virtual PodObject* pod() const final { return _value.get(); }
virtual Result<short> cmp(const Object& rhs) const final { virtual Result<short> cmp(const Object& rhs) const final {

View file

@ -2,6 +2,7 @@
#include "common.hpp" #include "common.hpp"
#include "error.hpp" #include "error.hpp"
#include "sourcerange.hpp"
static bool is_digit(char32_t c) { return c >= '0' && c <= '9'; } static bool is_digit(char32_t c) { return c >= '0' && c <= '9'; }
static bool is_alpha(char32_t c) { static bool is_alpha(char32_t c) {
@ -88,7 +89,7 @@ Result<Value> Reader::read_multiple() {
return ERROR(ReadError); return ERROR(ReadError);
} }
Result<Value> Reader::read_list_rec(SourcePosition& end) { Result<Value> Reader::read_list_rec() {
forward_whitespace(); forward_whitespace();
if (is_eof()) { if (is_eof()) {
@ -97,12 +98,11 @@ Result<Value> Reader::read_list_rec(SourcePosition& end) {
if (match(')')) { if (match(')')) {
forward(); forward();
end = position_;
return Value(TRY(Nil::create())); return Value(TRY(Nil::create()));
} }
auto val = TRY(read_one()); auto val = TRY(read_one());
auto rest = TRY(read_list_rec(end)); auto rest = TRY(read_list_rec());
auto res = Value(TRY(Pair::create(val, rest))); auto res = Value(TRY(Pair::create(val, rest)));
@ -111,18 +111,43 @@ Result<Value> Reader::read_list_rec(SourcePosition& end) {
Result<Value> Reader::read_list() { Result<Value> Reader::read_list() {
if (!match('(')) return ERROR(ReadError); if (!match('(')) return ERROR(ReadError);
// SourcePosition start = position_;
SourcePosition start = position_;
forward(); forward();
SourcePosition end = position_; auto res = Value(TRY(Nil::create()));
auto res = read_list_rec(end);
while (1) {
forward_whitespace();
if (is_eof()) {
return ERROR(ReadError);
}
if (match(')')) {
forward();
SourcePosition end = position_;
res = TRY(reverse(res));
if (!_syntax) {
return res; return res;
} else {
return Value(TRY(Syntax::create(_fname, start, end, res)));
}
}
auto val = TRY(read_one());
res = TRY(Pair::create(val, res));
}
return ERROR(ReadError);
} }
Result<Value> Reader::read_array_code() { Result<Value> Reader::read_array_code() {
if (!match('[')) return ERROR(ReadError); if (!match('[')) return ERROR(ReadError);
SourcePosition start = position_;
forward(); forward();
auto sym = Value(TRY(Symbol::create("array"))); auto sym = Value(TRY(Symbol::create("array")));
@ -137,7 +162,15 @@ Result<Value> Reader::read_array_code() {
if (match(']')) { if (match(']')) {
forward(); forward();
return reverse(res);
SourcePosition end = position_;
res = TRY(reverse(res));
if (!_syntax) {
return res;
} else {
return Value(TRY(Syntax::create(_fname, start, end, res)));
}
} }
auto val = TRY(read_one()); auto val = TRY(read_one());
@ -151,6 +184,7 @@ Result<Value> Reader::read_array_code() {
Result<Value> Reader::read_array_data() { Result<Value> Reader::read_array_data() {
if (!match('[')) return ERROR(ReadError); if (!match('[')) return ERROR(ReadError);
SourcePosition start = position_;
forward(); forward();
auto res = TRY(Array::create()); auto res = TRY(Array::create());
@ -164,7 +198,14 @@ Result<Value> Reader::read_array_data() {
if (match(']')) { if (match(']')) {
forward(); forward();
SourcePosition end = position_;
if (!_syntax) {
return Value(std::move(res)); return Value(std::move(res));
} else {
return Value(
TRY(Syntax::create(_fname, start, end, Value(std::move(res)))));
}
} }
auto val = TRY(read_one()); auto val = TRY(read_one());
@ -184,6 +225,7 @@ Result<Value> Reader::read_array() {
Result<Value> Reader::read_dict_code() { Result<Value> Reader::read_dict_code() {
if (!match('{')) return ERROR(ReadError); if (!match('{')) return ERROR(ReadError);
SourcePosition start = position_;
forward(); forward();
auto sym = Value(TRY(Symbol::create("dict"))); auto sym = Value(TRY(Symbol::create("dict")));
@ -198,7 +240,15 @@ Result<Value> Reader::read_dict_code() {
if (match('}')) { if (match('}')) {
forward(); forward();
return reverse(res);
SourcePosition end = position_;
res = TRY(reverse(res));
if (!_syntax) {
return res;
} else {
return Value(TRY(Syntax::create(_fname, start, end, res)));
}
} }
auto val1 = TRY(read_one()); auto val1 = TRY(read_one());
@ -214,6 +264,7 @@ Result<Value> Reader::read_dict_code() {
Result<Value> Reader::read_dict_data() { Result<Value> Reader::read_dict_data() {
if (!match('{')) return ERROR(ReadError); if (!match('{')) return ERROR(ReadError);
SourcePosition start = position_;
forward(); forward();
auto res = TRY(Dict::create()); auto res = TRY(Dict::create());
@ -227,7 +278,14 @@ Result<Value> Reader::read_dict_data() {
if (match('}')) { if (match('}')) {
forward(); forward();
SourcePosition end = position_;
if (!_syntax) {
return Value(std::move(res)); return Value(std::move(res));
} else {
return Value(
TRY(Syntax::create(_fname, start, end, Value(std::move(res)))));
}
} }
auto val1 = TRY(read_one()); auto val1 = TRY(read_one());
@ -246,25 +304,48 @@ Result<Value> Reader::read_dict() {
} }
Result<Value> Reader::read_bool() { Result<Value> Reader::read_bool() {
SourcePosition start = position_;
if (match("true")) { if (match("true")) {
forward(4); forward(4);
if (!is_separator(get()) && !is_eof()) return ERROR(ReadError); if (!is_separator(get()) && !is_eof()) return ERROR(ReadError);
return Value(TRY(Bool::create(true))); SourcePosition end = position_;
auto res = Value(TRY(Bool::create(true)));
if (!_syntax) {
return res;
} else {
return Value(TRY(Syntax::create(_fname, start, end, res)));
}
} }
if (match("false")) { if (match("false")) {
forward(5); forward(5);
if (!is_separator(get()) && !is_eof()) return ERROR(ReadError); if (!is_separator(get()) && !is_eof()) return ERROR(ReadError);
return Value(TRY(Bool::create(false))); SourcePosition end = position_;
auto res = Value(TRY(Bool::create(false)));
if (!_syntax) {
return res;
} else {
return Value(TRY(Syntax::create(_fname, start, end, res)));
}
} }
return ERROR(ReadError); return ERROR(ReadError);
} }
Result<Value> Reader::read_nil() { Result<Value> Reader::read_nil() {
SourcePosition start = position_;
if (match("nil")) { if (match("nil")) {
forward(3); forward(3);
if (!is_separator(get()) && !is_eof()) return ERROR(ReadError); if (!is_separator(get()) && !is_eof()) return ERROR(ReadError);
return Value(TRY(Nil::create())); SourcePosition end = position_;
auto res = Value(TRY(Nil::create()));
if (!_syntax) {
return res;
} else {
return Value(TRY(Syntax::create(_fname, start, end, res)));
}
} }
return ERROR(ReadError); return ERROR(ReadError);
@ -274,6 +355,7 @@ Result<Value> Reader::read_string() {
if (!match('"')) return ERROR(ReadError); if (!match('"')) return ERROR(ReadError);
size_t start = position_.offset + 1; size_t start = position_.offset + 1;
SourcePosition startloc = position_;
forward(); forward();
while (!match('"') && !match('\r') && !match('\n') && !is_eof()) { while (!match('"') && !match('\r') && !match('\n') && !is_eof()) {
@ -331,13 +413,20 @@ Result<Value> Reader::read_string() {
} }
if (!is_separator(get()) && !is_eof()) return ERROR(ReadError); if (!is_separator(get()) && !is_eof()) return ERROR(ReadError);
SourcePosition endloc = position_;
return Value(std::move(result)); auto res = Value(std::move(result));
if (!_syntax) {
return res;
} else {
return Value(TRY(Syntax::create(_fname, startloc, endloc, res)));
}
} }
Result<Value> Reader::read_number() { Result<Value> Reader::read_number() {
if (!is_numeric_start()) return ERROR(ReadError); if (!is_numeric_start()) return ERROR(ReadError);
size_t start = position_.offset; size_t start = position_.offset;
SourcePosition startloc = position_;
bool is_float = false; bool is_float = false;
@ -397,13 +486,20 @@ Result<Value> Reader::read_number() {
if (!is_separator(get()) && !is_eof()) return ERROR(ReadError); if (!is_separator(get()) && !is_eof()) return ERROR(ReadError);
SourcePosition endloc = position_;
if (!_syntax) {
return res; return res;
} else {
return Value(TRY(Syntax::create(_fname, startloc, endloc, res)));
}
} }
Result<Value> Reader::read_symbol() { Result<Value> Reader::read_symbol() {
if (!is_symbol_char(get())) return ERROR(ReadError); if (!is_symbol_char(get())) return ERROR(ReadError);
size_t start = position_.offset; size_t start = position_.offset;
SourcePosition startloc = position_;
while (is_symbol_char(get())) forward(); while (is_symbol_char(get())) forward();
@ -413,7 +509,14 @@ Result<Value> Reader::read_symbol() {
if (!is_separator(get()) && !is_eof()) return ERROR(ReadError); if (!is_separator(get()) && !is_eof()) return ERROR(ReadError);
return Value(TRY(Symbol::create(str))); SourcePosition endloc = position_;
auto res = Value(TRY(Symbol::create(str)));
if (!_syntax) {
return res;
} else {
return Value(TRY(Syntax::create(_fname, startloc, endloc, res)));
}
} }
char32_t Reader::get(size_t offset) { char32_t Reader::get(size_t offset) {
@ -524,20 +627,24 @@ bool Reader::forward_exponent() {
Result<Value> read_one(const Value& value, bool syntax, bool as_code) { Result<Value> read_one(const Value& value, bool syntax, bool as_code) {
if (!value.is<String>()) return ERROR(TypeMismatch); if (!value.is<String>()) return ERROR(TypeMismatch);
auto r = Reader(*value.to<String>(), syntax, as_code); auto fname = TRY(String::create(""));
auto r = Reader(fname, *value.to<String>(), syntax, as_code);
return r.read_one(); return r.read_one();
} }
Result<Value> read_one(const String& value, bool syntax, bool as_code) { Result<Value> read_one(const String& value, bool syntax, bool as_code) {
auto r = Reader(value, syntax, as_code); auto fname = TRY(String::create(""));
auto r = Reader(fname, value, syntax, as_code);
return r.read_one(); return r.read_one();
} }
Result<Value> read_one(const char* value, bool syntax, bool as_code) { Result<Value> read_one(const char* value, bool syntax, bool as_code) {
auto fname = TRY(String::create(""));
auto s = TRY(String::create(value)); auto s = TRY(String::create(value));
auto r = Reader(s, syntax, as_code); auto r = Reader(fname, s, syntax, as_code);
return r.read_one(); return r.read_one();
} }
Result<Value> read_multiple(const String& value, bool syntax, bool as_code) { Result<Value> read_multiple(const String& value, bool syntax, bool as_code) {
auto r = Reader(value, syntax, as_code); auto fname = TRY(String::create(""));
auto r = Reader(fname, value, syntax, as_code);
return r.read_multiple(); return r.read_multiple();
} }

View file

@ -5,8 +5,9 @@
class Reader { class Reader {
public: public:
Reader(const String& str, bool syntax = true, bool as_code = true) Reader(const String& fname, const String& str, bool syntax = true,
: _str(str), _syntax(syntax), _as_code(as_code) {} bool as_code = true)
: _fname(fname), _str(str), _syntax(syntax), _as_code(as_code) {}
Result<Value> read_one(); Result<Value> read_one();
Result<Value> read_multiple(); Result<Value> read_multiple();
@ -28,7 +29,7 @@ class Reader {
bool is_string_start(); bool is_string_start();
Result<Value> read_list(); Result<Value> read_list();
Result<Value> read_list_rec(SourcePosition& end); Result<Value> read_list_rec();
Result<Value> read_string(); Result<Value> read_string();
Result<Value> read_array(); Result<Value> read_array();
Result<Value> read_array_code(); Result<Value> read_array_code();
@ -45,6 +46,7 @@ class Reader {
bool match(const char* str); bool match(const char* str);
bool match(char c); bool match(char c);
const String& _fname;
const String& _str; const String& _str;
bool _syntax; bool _syntax;
bool _as_code; bool _as_code;

View file

@ -13,7 +13,7 @@
StaticArena<64 * 1024 * 1024> arena; StaticArena<64 * 1024 * 1024> arena;
Result<Value> run_string(const String& src) { Result<Value> run_string(const String& src) {
auto parsed = TRY(read_multiple(src)); auto parsed = TRY(read_multiple(src, false));
TRY(arena_gc()); TRY(arena_gc());
@ -32,7 +32,7 @@ Result<void> run_repl() {
while (true) { while (true) {
auto src = TRY(read_line("valeri> ")); auto src = TRY(read_line("valeri> "));
auto parsed = TRY(read_multiple(src)); auto parsed = TRY(read_multiple(src, false));
auto compiled = TRY(compile(parsed)); auto compiled = TRY(compile(parsed));
Module& mod = *compiled.to<Module>(); Module& mod = *compiled.to<Module>();
auto vm = TRY(VM::create()); auto vm = TRY(VM::create());

View file

@ -196,7 +196,7 @@ Result<String> Writer::write_srcloc(const SrcLoc& val) {
} }
Result<String> Writer::write_syntax(const Syntax& val) { Result<String> Writer::write_syntax(const Syntax& val) {
return ERROR(NotImplemented); return TRY(String::create("#<syntax>"));
} }
Result<String> Writer::write_pair(const Pair& val) { Result<String> Writer::write_pair(const Pair& val) {