syntax: add support for raw strings

CHANGELOG.md

@@ -9,8 +9,12 @@
 - sha256 sums of release artefacts in the releases
 - the debugger now display the instruction and page pointers in the traces
 - checking that we can properly create error messages when code is run from a string
+- add support for raw strings: `r"string..."`, backslashes do not have to be escaped in them
+- stdlib:
+  - new draft module `re`, using `google/re2` as the regex engine
 
 ### Changed
+- `@` can be used with dictionaries: `(@ dict key)` will behave the same as `(dict:get dict key)`
 
 ### Removed
 - removed `std.Range` which had been deprecated in 4.6.0

CMakeLists.txt

@@ -27,6 +27,11 @@ else ()
     set(ARK_BUILD_DATE "2026-01-02T17:56:00Z")
 endif ()
 
+if (CMAKE_SOURCE_DIR STREQUAL ${PROJECT_SOURCE_DIR})
+    set(CMAKE_CXX_STANDARD 20)
+    set(CMAKE_CXX_STANDARD_REQUIRED ON)
+endif ()
+
 # needed so that cppcheck has a cache folder, when launched through pre-commit
 file(MAKE_DIRECTORY cache-cppcheck)
 
@@ -83,7 +88,7 @@ file(GLOB_RECURSE SOURCE_FILES
 
 if (NOT ARK_EMSCRIPTEN AND NOT ARK_STATIC)
     add_library(ArkReactor SHARED ${SOURCE_FILES})
-    # enable_lto(ArkReactor)
+    enable_lto(ArkReactor)
     set_target_properties(ArkReactor PROPERTIES
             INSTALL_NAME_DIR "@rpath"
             LIBRARY_OUTPUT_DIRECTORY ${CMAKE_INSTALL_BINDIR})
@@ -336,7 +341,7 @@ if (ARK_BENCHMARKS)
     target_link_libraries(bench PUBLIC ArkReactor benchmark::benchmark)
     target_compile_features(bench PRIVATE cxx_std_20)
     target_compile_definitions(bench PRIVATE ARK_TESTS_ROOT="${CMAKE_CURRENT_SOURCE_DIR}/")
-    # enable_lto(bench)
+    enable_lto(bench)
 endif ()
 
 if (ARK_BUILD_EXE)
@@ -367,7 +372,7 @@ if (ARK_BUILD_EXE)
             INSTALL_RPATH_USE_LINK_PATH ON)
     set_target_rpath(arkscript)
 
-    # enable_lto(arkscript)
+    enable_lto(arkscript)
 
     # Installs the arkscript executable.
     install(TARGETS arkscript

docs/arkdoc/Dict.txt

@@ -17,3 +17,15 @@
 * (print (len (dict "a" 1 "b" 2 "c" 3)))  # 3
 * =end
 #--
+
+--#
+* @name @
+# @brief Get a value from a given dictionary using a key, or nil if it doesn't exist
+# @param _D dictionary
+# @param _key key to get
+# =begin
+# (let data (dict "key" "value"))
+# (print (@ data "key"))  # value
+# =end
+* =end
+#--

include/Ark/Compiler/AST/Node.hpp

@@ -195,6 +195,19 @@ namespace Ark::internal
          */
         [[nodiscard]] bool isAnonymousFunction() const noexcept;
 
+        /**
+         * @brief Set the m_is_raw_string flag on the node
+         * @param is_raw_string true to mark the node as a raw string
+         */
+        void setRawString(bool is_raw_string);
+
+        /**
+         * @brief Check if a node is a raw string
+         * @return true if the node is a raw string
+         * @return false
+         */
+        [[nodiscard]] bool isRawString() const noexcept;
+
         /**
          * @brief Get the span of the node (start and end)
          *
@@ -239,16 +252,19 @@ namespace Ark::internal
         friend class Parser;
 
     private:
-        NodeType m_type { NodeType::Unused };
         Value m_value;
+        // 5 bits is more than enough, we can have 32 different nodes types
+        NodeType m_type : 5 = NodeType::Unused;
+        bool m_alt_syntax : 1 = false;            ///< Used to tell if a node uses the alternative syntax (if available), eg (begin) / {}, (list) / []
+        bool m_is_anonymous_function : 1 = true;  ///< Function nodes are marked as anonymous/non-anonymous by the ASTLowerer, to enable some optimisations
+        bool m_is_raw_string : 1 = false;         ///< Strings are by default not raw ; raw strings have all their backslashes escaped
+
         std::optional<std::string> m_unqualified_name { std::nullopt };  ///< Used by Capture nodes, to have the FQN in the value, and the captured name here
         // position of the node in the original code, useful when it comes to parser errors
         FileSpan m_pos;
         std::string m_filename;
         std::string m_comment;
-        std::string m_after_comment;          ///< Comment after node
-        bool m_alt_syntax = false;            ///< Used to tell if a node uses the alternative syntax (if available), eg (begin) / {}, (list) / []
-        bool m_is_anonymous_function = true;  ///< Function nodes are marked as anonymous/non-anonymous by the ASTLowerer, to enable some optimisations
+        std::string m_after_comment;  ///< Comment after node
     };
 
     const Node& getTrueNode();

include/Ark/VM/Helpers.hpp

@@ -73,41 +73,48 @@ namespace Ark::helper
 
     inline ARK_ALWAYS_INLINE Value at(Value& container, Value& index, VM& vm)
     {
-        if (index.valueType() != ValueType::Number)
-            throw types::TypeCheckingError(
-                "@",
-                { { types::Contract { { types::Typedef("src", ValueType::List), types::Typedef("idx", ValueType::Number) } },
-                    types::Contract { { types::Typedef("src", ValueType::String), types::Typedef("idx", ValueType::Number) } } } },
-                { container, index });
-
-        const auto num = static_cast<long>(index.number());
-
-        if (container.valueType() == ValueType::List)
+        if (container.valueType() != ValueType::Dict)
         {
-            const auto i = static_cast<std::size_t>(num < 0 ? static_cast<long>(container.list().size()) + num : num);
-            if (i < container.list().size())
-                return container.list()[i];
+            if (index.valueType() != ValueType::Number)
+                throw types::TypeCheckingError(
+                    "@",
+                    { { types::Contract { { types::Typedef("src", ValueType::List), types::Typedef("idx", ValueType::Number) } },
+                        types::Contract { { types::Typedef("src", ValueType::String), types::Typedef("idx", ValueType::Number) } },
+                        types::Contract { { types::Typedef("src", ValueType::Dict), types::Typedef("key", ValueType::Any) } } } },
+                    { container, index });
+
+            const auto num = static_cast<long>(index.number());
+
+            if (container.valueType() == ValueType::List)
+            {
+                const auto i = static_cast<std::size_t>(num < 0 ? static_cast<long>(container.list().size()) + num : num);
+                if (i < container.list().size())
+                    return container.list()[i];
+                else
+                    VM::throwVMError(
+                        ErrorKind::Index,
+                        fmt::format("{} out of range {} (length {})", num, container.toString(vm), container.list().size()));
+            }
+            else if (container.valueType() == ValueType::String)
+            {
+                const auto i = static_cast<std::size_t>(num < 0 ? static_cast<long>(container.string().size()) + num : num);
+                if (i < container.string().size())
+                    return Value(std::string(1, container.string()[i]));
+                else
+                    VM::throwVMError(
+                        ErrorKind::Index,
+                        fmt::format("{} out of range \"{}\" (length {})", num, container.string(), container.string().size()));
+            }
             else
-                VM::throwVMError(
-                    ErrorKind::Index,
-                    fmt::format("{} out of range {} (length {})", num, container.toString(vm), container.list().size()));
-        }
-        else if (container.valueType() == ValueType::String)
-        {
-            const auto i = static_cast<std::size_t>(num < 0 ? static_cast<long>(container.string().size()) + num : num);
-            if (i < container.string().size())
-                return Value(std::string(1, container.string()[i]));
-            else
-                VM::throwVMError(
-                    ErrorKind::Index,
-                    fmt::format("{} out of range \"{}\" (length {})", num, container.string(), container.string().size()));
+                throw types::TypeCheckingError(
+                    "@",
+                    { { types::Contract { { types::Typedef("src", ValueType::List), types::Typedef("idx", ValueType::Number) } },
+                        types::Contract { { types::Typedef("src", ValueType::String), types::Typedef("idx", ValueType::Number) } },
+                        types::Contract { { types::Typedef("src", ValueType::Dict), types::Typedef("key", ValueType::Any) } } } },
+                    { container, index });
         }
         else
-            throw types::TypeCheckingError(
-                "@",
-                { { types::Contract { { types::Typedef("src", ValueType::List), types::Typedef("idx", ValueType::Number) } },
-                    types::Contract { { types::Typedef("src", ValueType::String), types::Typedef("idx", ValueType::Number) } } } },
-                { container, index });
+            return container.dictRef().get(index);
     }
 
     inline ARK_ALWAYS_INLINE Value atAt(const Value* x, const Value* y, Value& list)

src/arkreactor/Compiler/AST/Node.cpp

@@ -9,7 +9,7 @@
 namespace Ark::internal
 {
     Node::Node(const NodeType node_type, const std::string& value) :
-        m_type(node_type), m_value(value), m_pos()
+        m_value(value), m_type(node_type), m_pos()
     {}
 
     Node::Node(const NodeType node_type) :
@@ -20,19 +20,19 @@ namespace Ark::internal
     }
 
     Node::Node(double value) :
-        m_type(NodeType::Number), m_value(value), m_pos()
+        m_value(value), m_type(NodeType::Number), m_pos()
     {}
 
     Node::Node(const long value) :
-        m_type(NodeType::Number), m_value(static_cast<double>(value)), m_pos()
+        m_value(static_cast<double>(value)), m_type(NodeType::Number), m_pos()
     {}
 
     Node::Node(Keyword value) :
-        m_type(NodeType::Keyword), m_value(value), m_pos()
+        m_value(value), m_type(NodeType::Keyword), m_pos()
     {}
 
     Node::Node(const Namespace& namespace_) :
-        m_type(NodeType::Namespace), m_value(namespace_), m_pos()
+        m_value(namespace_), m_type(NodeType::Namespace), m_pos()
     {}
 
     const std::string& Node::string() const noexcept
@@ -156,6 +156,16 @@ namespace Ark::internal
         return m_is_anonymous_function;
     }
 
+    void Node::setRawString(const bool is_raw_string)
+    {
+        m_is_raw_string = is_raw_string;
+    }
+
+    bool Node::isRawString() const noexcept
+    {
+        return m_is_raw_string;
+    }
+
     FileSpan Node::position() const noexcept
     {
         return m_pos;
@@ -202,7 +212,7 @@ namespace Ark::internal
                 break;
 
             case NodeType::String:
-                data += "\"" + string() + "\"";
+                data += (m_is_raw_string ? "r\"" : "\"") + string() + "\"";
                 break;
 
             case NodeType::Number:
@@ -321,7 +331,10 @@ namespace Ark::internal
                 break;
 
             case NodeType::String:
-                os << "String:" << string();
+                if (m_is_raw_string)
+                    os << "RawString:" << string();
+                else
+                    os << "String:" << string();
                 break;
 
             case NodeType::Number:
@@ -367,30 +380,6 @@ namespace Ark::internal
         return os;
     }
 
-    const Node& getTrueNode()
-    {
-        static const Node TrueNode(NodeType::Symbol, "true");
-        return TrueNode;
-    }
-
-    const Node& getFalseNode()
-    {
-        static const Node FalseNode(NodeType::Symbol, "false");
-        return FalseNode;
-    }
-
-    const Node& getNilNode()
-    {
-        static const Node NilNode(NodeType::Symbol, "nil");
-        return NilNode;
-    }
-
-    const Node& getListNode()
-    {
-        static const Node ListNode(NodeType::Symbol, "list");
-        return ListNode;
-    }
-
     bool operator==(const Node& A, const Node& B)
     {
         if (A.m_type != B.m_type)  // should have the same types
@@ -419,4 +408,28 @@ namespace Ark::internal
                 return false;
         }
     }
+
+    const Node& getTrueNode()
+    {
+        static const Node TrueNode(NodeType::Symbol, "true");
+        return TrueNode;
+    }
+
+    const Node& getFalseNode()
+    {
+        static const Node FalseNode(NodeType::Symbol, "false");
+        return FalseNode;
+    }
+
+    const Node& getNilNode()
+    {
+        static const Node NilNode(NodeType::Symbol, "nil");
+        return NilNode;
+    }
+
+    const Node& getListNode()
+    {
+        static const Node ListNode(NodeType::Symbol, "list");
+        return ListNode;
+    }
 }

src/arkreactor/Compiler/AST/Parser.cpp

@@ -840,14 +840,16 @@ namespace Ark::internal
 
     std::optional<Node> Parser::string(const FilePosition filepos)
     {
-        std::string res;
-        if (accept(IsChar('"')))
+        if (const bool is_normal_string = accept(IsChar('"')); is_normal_string || (accept(IsChar('r')) && accept(IsChar('"'))))
         {
+            const bool is_raw_string = !is_normal_string;
+
+            std::string res;
             while (true)
             {
                 const auto pos = getCursor();
 
-                if (accept(IsChar('\\')))
+                if (!is_raw_string && accept(IsChar('\\')))
                 {
                     if (m_mode != ParserMode::Interpret)
                         res += '\\';
@@ -960,16 +962,20 @@ namespace Ark::internal
                         error("Unknown escape sequence", pos);
                     }
                 }
+                else if (is_raw_string)
+                    accept(IsNot(IsChar('"')), &res);
                 else
                     accept(IsNot(IsEither(IsChar('\\'), IsChar('"'))), &res);
 
                 if (accept(IsChar('"')))
                     break;
                 if (isEOF())
-                    expectSuffixOrError('"', "after string");
+                    expectSuffixOrError('"', fmt::format("after {}string", is_raw_string ? "raw " : ""));
             }
 
-            return positioned(Node(NodeType::String, res), filepos);
+            Node str_node = Node(NodeType::String, res);
+            str_node.setRawString(is_raw_string);
+            return positioned(str_node, filepos);
         }
         return std::nullopt;
     }

src/arkscript/Formatter.cpp

@@ -238,7 +238,7 @@ std::string Formatter::format(const Node& node, std::size_t indent, bool after_n
             result += std::string(keywords[static_cast<std::size_t>(node.keyword())]);
             break;
         case NodeType::String:
-            result += fmt::format("\"{}\"", node.string());
+            result += fmt::format("{}\"{}\"", node.isRawString() ? "r" : "", node.string());
             break;
         case NodeType::Number:
             result += fmt::format("{}", node.number());

tests/unittests/resources/DiagnosticsSuite/typeChecking/at_list_str.expected

@@ -14,6 +14,13 @@ Arguments
   → `src' (expected String), got [] (List)
   → `idx' (expected Number), got "1" (String)
 
+Alternative 3:
+Signature
+  ↳ (@ src key)
+Arguments
+  → `src' (expected Dict), got [] (List)
+  → `key' (expected any) ✓
+
 In file tests/unittests/resources/DiagnosticsSuite/typeChecking/at_list_str.ark:1
     1 | (@ [] "1")
       | ^~~~~~~~~