Drastically improve parsing and unparsing speed and fix bug

The parsing code still isn't easy to read but at least it's faster and 
not horribly bugged (I hope).

core.lua

@@ -31,63 +31,77 @@ local formspec_ast, minetest = formspec_ast, formspec_ast.minetest
 
 -- Parse a formspec into a "raw" non-AST state.
 -- Input: size[5,2]button[0,0;5,1;name;Label] image[0,1;1,1;air.png]
--- Output: {{'size', '5', '2'}, {'button', '0,0', '5,1', 'name', 'label'},
+-- Output:
---             {'image', '0,1', '1,1', 'air.png'}}
+-- {
+--     {type='size', '5', '2'},
+--     {type='button', {'0', '0'}, {'5', '1'}, 'name', 'label'},
+--     {type='image', {'0', '1'}, {'1', '1'}, 'air.png'},
+-- }
+local table_concat = table.concat
 local function raw_parse(spec)
     local res = {}
-    while spec do
+    local end_idx = 0
-        -- Get the first element
+    local bracket_idx
-        local name
+    local spec_length = #spec
-        name, spec = spec:match('([%w_%-:]*[^%s\\])%s*(%[.*)')
-        if not name or not spec then return res end
-        local elem = {}
-        elem[1] = name
 
-        -- Get the parameters
+    while end_idx < spec_length do
-        local s, e = spec:find('[^\\]%]')
+        -- Get the element type
-        local rawargs
+        bracket_idx = spec:find('[', end_idx + 1, true)
-        if s and e then
+        if not bracket_idx then break end
-            rawargs, spec = spec:sub(2, s), spec:sub(e + 1)
+        local parts = {type = spec:sub(end_idx + 1, bracket_idx - 1):trim()}
-        else
-            rawargs, spec = spec:sub(2), false
-        end
 
         -- Split everything
-        -- TODO: Make this a RegEx
+        -- This tries and avoids creating small strings where possible
-        local i = ''
+        end_idx = spec_length + 1
+        local part = {}
         local esc = false
         local inner = {}
-        for c = 1, #rawargs do
+        local start_idx = bracket_idx + 1
-            local char = rawargs:sub(c, c)
+        for idx = bracket_idx, spec_length do
+            local byte = spec:byte(idx)
             if esc then
+                -- The current character is escaped
                 esc = false
-                i = i .. char
+                start_idx = idx
-            elseif char == '\\' then
+            elseif byte == 0x5c then
-                    esc = true
+                -- Backslashes
-            elseif char == ';' then
+                part[#part + 1] = spec:sub(start_idx, idx - 1)
+                esc = true
+            elseif byte == 0x3b then
+                -- Semicolons
+                part[#part + 1] = spec:sub(start_idx, idx - 1)
+                start_idx = idx + 1
                 if #inner > 0 then
-                    table.insert(inner, i)
+                    inner[#inner + 1] = table_concat(part)
-                    table.insert(elem, inner)
+                    parts[#parts + 1] = inner
                     inner = {}
                 else
-                    table.insert(elem, i)
+                    parts[#parts + 1] = table_concat(part)
                 end
-                i = ''
+                part = {}
-            elseif char == ',' then
+            elseif byte == 0x2c then
-                table.insert(inner, i)
+                -- Commas
-                i = ''
+                part[#part + 1] = spec:sub(start_idx, idx - 1)
-            else
+                start_idx = idx + 1
-                i = i .. char
+                inner[#inner + 1] = table_concat(part)
+                part = {}
+            elseif byte == 0x5d then
+                -- ]
+                end_idx = idx
+                break
             end
         end
+
+        -- Add the last part
+        part[#part + 1] = spec:sub(start_idx, end_idx - 1)
         if #inner > 0 then
-            table.insert(inner, i)
+            inner[#inner + 1] = table_concat(part)
-            table.insert(elem, inner)
+            parts[#parts + 1] = inner
         else
-            table.insert(elem, i)
+            parts[#parts + 1] = table_concat(part)
         end
 
-        table.insert(res, elem)
+        res[#res + 1] = parts
     end
 
     return res
@@ -96,22 +110,24 @@ end
 -- Unparse raw formspecs
 -- WARNING: This will modify the table passed to it.
 local function raw_unparse(data)
-    local res = ''
+    local res = {}
-    for _, elem in ipairs(data) do
+    for _, parts in ipairs(data) do
-        for i = 2, #elem do
+        res[#res + 1] = parts.type
-            if type(elem[i]) == 'table' then
+        for i = 1, #parts do
-                for j, e in ipairs(elem[i]) do
+            if type(parts[i]) == 'table' then
-                    elem[i][j] = minetest.formspec_escape(e)
+                for j, e in ipairs(parts[i]) do
+                    parts[i][j] = minetest.formspec_escape(e)
                 end
-                elem[i] = table.concat(elem[i], ',')
+                parts[i] = table_concat(parts[i], ',')
             else
-                elem[i] = minetest.formspec_escape(elem[i])
+                parts[i] = minetest.formspec_escape(parts[i])
             end
         end
-        res = res .. table.remove(elem, 1) .. '[' ..
+        res[#res + 1] = '['
-            table.concat(elem, ';') .. ']'
+        res[#res + 1] = table_concat(parts, ';')
+        res[#res + 1] = ']'
     end
-    return res
+    return table_concat(res)
 end
 
 -- Elements
@@ -186,7 +202,7 @@ local function parse_value(elems, template)
             local func = types[obj[2]] or types.undefined
             local elem = elems[i]
             if type(elem) == 'table' then
-                elem = table.concat(elem, ',')
+                elem = table_concat(elem, ',')
             end
             res[obj[1]] = func(elem, obj[1])
         else
@@ -237,7 +253,6 @@ end
 
 local parse_mt
 local function parse_elem(elem, custom_handlers)
-    elem.type = table.remove(elem, 1)
     local data = elements[elem.type]
     if not data then
         if not custom_handlers or not custom_handlers[elem.type] then
@@ -252,7 +267,6 @@ local function parse_elem(elem, custom_handlers)
         if good then
             return ast_elem, true
         else
-            table.insert(elem, 1, elem.type)
             return nil, 'Invalid element "' .. raw_unparse({elem}) .. '": ' ..
                 tostring(ast_elem)
         end
@@ -274,7 +288,6 @@ local function parse_elem(elem, custom_handlers)
         end
     end
 
-    table.insert(elem, 1, elem.type)
     return nil, 'Invalid element "' .. raw_unparse({elem}) .. '": ' ..
         tostring(ast_elem)
 end
@@ -318,8 +331,8 @@ end
 -- }
 
 
-function formspec_ast.parse(spec, custom_handlers)
+function formspec_ast.parse(fs, custom_handlers)
-    spec = raw_parse(spec)
+    local spec = raw_parse(fs)
     local res = {formspec_version=1}
     local containers = {}
     local container = res
@@ -463,7 +476,7 @@ local function unparse_elem(elem, res, force)
             good, raw_elem = pcall(unparse_value, elem, template)
         end
         if good then
-            table.insert(raw_elem, 1, elem.type)
+            raw_elem.type = elem.type
             table.insert(possible_elems, raw_elem)
         end
     end
@@ -508,7 +521,10 @@ end
 function formspec_ast.unparse(tree)
     local raw_spec = {}
     if tree.formspec_version and tree.formspec_version ~= 1 then
-        raw_spec[1] = {'formspec_version', tostring(tree.formspec_version)}
+        raw_spec[1] = {
+            type = 'formspec_version',
+            tostring(tree.formspec_version)
+        }
     end
 
     for _, elem in ipairs(tree) do
@@ -536,7 +552,7 @@ function parse_mt:__index(key)
     if func then
         return function(obj)
             if type(obj) == 'table' then
-                obj = table.concat(obj, ',')
+                obj = table_concat(obj, ',')
             end
             return func(obj or '')
         end

tests.lua

@@ -64,12 +64,23 @@ local function test_parse_unparse(fs, expected_tree)
     assert_equal(fs, unparsed_fs)
 end
 
+test_parse_unparse([=[label[123,456;yay abc def\, ghi\; jkl mno \]\\]]=], {
+    formspec_version = 1,
+    {
+        type = 'label',
+        x = 123,
+        y = 456,
+        label = 'yay abc def, ghi; jkl mno ]\\'
+    }
+})
+
 local fs = [[
     formspec_version[2]
     size[5,2]
     padding[1,2]
+    no_prepend[]
     container[1,1]
-        label[0,0;Containers are fun]
+        label[0,0;Containers are fun\]\\]
         container[-1,-1]
             button[0.5,0;4,1;name;Label]
         container_end[]
@@ -109,6 +120,9 @@ test_parse_unparse(fs, {
         x = 1,
         y = 2,
     },
+    {
+        type = "no_prepend"
+    },
     {
         type = "container",
         x = 1,
@@ -117,7 +131,7 @@ test_parse_unparse(fs, {
             type = "label",
             x = 0,
             y = 0,
-            label = "Containers are fun",
+            label = "Containers are fun]\\",
         },
         {
             type = "container",