Blocktrans utilities

src/erlydtl_scanner.erl

@@ -128,19 +128,19 @@ scan("<!--{{" ++ T, Scanned, {Row, Column}, in_text) ->
     scan(T, [{open_var, {Row, Column}, '<!--{{'} | Scanned], {Row, Column + length("<!--{{")}, {in_code, "}}-->"});
 
 scan("{{" ++ T, Scanned, {Row, Column}, in_text) ->
-    scan(T, [{open_var, {Row, Column}, '{{'} | Scanned], {Row, Column + 2}, {in_code, "}}"});
+    scan(T, [{open_var, {Row, Column}, '{{'} | Scanned], {Row, Column + length("{{")}, {in_code, "}}"});
 
 scan("<!--{#" ++ T, Scanned, {Row, Column}, in_text) ->
     scan(T, Scanned, {Row, Column + length("<!--{#")}, {in_comment, "#}-->"});
 
 scan("{#" ++ T, Scanned, {Row, Column}, in_text) ->
-    scan(T, Scanned, {Row, Column + 2}, {in_comment, "#}"});
+    scan(T, Scanned, {Row, Column + length("{#")}, {in_comment, "#}"});
 
 scan("#}-->" ++ T, Scanned, {Row, Column}, {in_comment, "#}-->"}) ->
     scan(T, Scanned, {Row, Column + length("#}-->")}, in_text);
 
 scan("#}" ++ T, Scanned, {Row, Column}, {in_comment, "#}"}) ->
-    scan(T, Scanned, {Row, Column + 2}, in_text);
+    scan(T, Scanned, {Row, Column + length("#}")}, in_text);
 
 scan("<!--{%" ++ T, Scanned, {Row, Column}, in_text) ->
     scan(T, [{open_tag, {Row, Column}, '<!--{%'} | Scanned], 
@@ -148,7 +148,7 @@ scan("<!--{%" ++ T, Scanned, {Row, Column}, in_text) ->
 
 scan("{%" ++ T, Scanned, {Row, Column}, in_text) ->
     scan(T, [{open_tag, {Row, Column}, '{%'} | Scanned], 
-        {Row, Column + 2}, {in_code, "%}"});
+        {Row, Column + length("{%")}, {in_code, "%}"});
 
 scan([_ | T], Scanned, {Row, Column}, {in_comment, Closer}) ->
     scan(T, Scanned, {Row, Column + 1}, {in_comment, Closer});
@@ -289,30 +289,19 @@ scan([H | T], Scanned, {Row, Column}, {in_identifier, Closer}) ->
 
 % internal functions
 
-append_char(Scanned, Char) ->
-    [String | Scanned1] = Scanned,
-    [setelement(3, String, [Char | element(3, String)]) | Scanned1].
+append_char([{Type, Pos, Chars}|Scanned], Char) ->
+    [{Type, Pos, [Char | Chars]} | Scanned].
 
+append_text_char([], {Row, Column}, Char) ->
+    [{string, {Row, Column}, [Char]}];
+append_text_char([{string, StrPos, Chars} |Scanned1], _, Char) ->
+    [{string, StrPos, [Char | Chars]} | Scanned1];
 append_text_char(Scanned, {Row, Column}, Char) ->
-    case length(Scanned) of
-        0 ->
-            [{string, {Row, Column}, [Char]}];
-        _ ->
-            [Token | Scanned1] = Scanned,
-            case element(1, Token) of
-                string ->
-                    [{string, element(2, Token), [Char | element(3, Token)]} | Scanned1];
-                _ ->
-                    [{string, element(2, Token), [Char]} | Scanned]
-            end
-    end.
-
-char_type(Char) ->
-    case Char of 
-        C when ((C >= $a) and (C =< $z)) or ((C >= $A) and (C =< $Z)) or (C == $_) ->
-            letter_underscore;
-        C when ((C >= $0) and (C =< $9)) ->
-            digit;
-        _ ->
-            undefined
-    end.
+    [{string, {Row, Column}, [Char]} | Scanned].
+
+char_type(C) when ((C >= $a) andalso (C =< $z)) orelse ((C >= $A) andalso (C =< $Z)) orelse (C == $_) ->
+    letter_underscore;
+char_type(C) when ((C >= $0) andalso (C =< $9)) ->
+    digit;
+char_type(_C) ->
+    undefined.

src/i18n/blocktrans_parser.erl

@@ -0,0 +1,13 @@
+-module(blocktrans_parser).
+
+-export([parse/1]).
+
+parse(Tokens) ->
+    parse(Tokens, []).
+
+parse([], Acc) ->
+    lists:reverse(Acc);
+parse([{open_blocktrans, _, Name}, {text, _, Text}, {close_blocktrans, _}|Rest], Acc) ->
+    parse(Rest, [{Name, unicode:characters_to_binary(Text)}|Acc]);
+parse([{text, _, _}|Rest], Acc) ->
+    parse(Rest, Acc).

src/i18n/blocktrans_scanner.erl

@@ -0,0 +1,146 @@
+% Module for extracting blocktrans blocks with original source formatting preserved.
+
+-module(blocktrans_scanner).
+
+-export([scan/1]).
+
+scan(Template) ->
+    scan(Template, [], {1, 1}, in_text).
+
+scan([], Scanned, _, in_text) ->
+    {ok, lists:reverse(lists:map(
+                fun
+                    ({text, Pos, Text}) ->
+                        {text, Pos, lists:reverse(Text)};
+                    (Other) ->
+                        Other
+                end, Scanned))};
+
+scan([], _Scanned, _, {in_comment, _}) ->
+    {error, "Reached end of file inside a comment."};
+
+scan([], _Scanned, _, _) ->
+    {error, "Reached end of file inside a code block."};
+
+scan("<!--{{" ++ T, Scanned, {Row, Column}, in_text) ->
+    scan(T, append_text("<!--{{", {Row, Column}, Scanned), {Row, Column + length("<!--{{")}, {in_code, "}}-->"});
+
+scan("{{" ++ T, Scanned, {Row, Column}, in_text) ->
+    scan(T, append_text("{{", {Row, Column}, Scanned), {Row, Column + 2}, {in_code, "}}"});
+
+scan("<!--{#" ++ T, Scanned, {Row, Column}, in_text) ->
+    scan(T, append_text("<!--{#", {Row, Column}, Scanned), {Row, Column + length("<!--{#")}, {in_comment, "#}-->"});
+
+scan("{#" ++ T, Scanned, {Row, Column}, in_text) ->
+    scan(T, append_text("{#", {Row, Column}, Scanned), {Row, Column + length("{#")}, {in_comment, "#}"});
+
+scan("#}-->" ++ T, Scanned, {Row, Column}, {in_comment, "#}-->"}) ->
+    scan(T, append_text("#}-->", {Row, Column}, Scanned), {Row, Column + length("#}-->")}, in_text);
+
+scan("#}" ++ T, Scanned, {Row, Column}, {in_comment, "#}"}) ->
+    scan(T, append_text("#}", {Row, Column}, Scanned), {Row, Column + length("#}")}, in_text);
+
+scan("<!--{% blocktrans " ++ T, Scanned, {Row, Column}, in_text) ->
+    scan(T, [{open_blocktrans, {Row, Column}, ""} | Scanned], 
+        {Row, Column + length("<!--{% blocktrans ")}, {in_code, "%}-->"});
+
+scan("{% blocktrans " ++ T, Scanned, {Row, Column}, in_text) ->
+    scan(T, [{open_blocktrans, {Row, Column}, ""} | Scanned], 
+        {Row, Column + length("{% blocktrans ")}, {in_code, "%}"});
+
+scan("{% endblocktrans %}" ++ T, Scanned, {Row, Column}, in_text) ->
+    scan(T, [{close_blocktrans, {Row, Column}} | Scanned],
+        {Row, Column + length("{% endblocktrans %}")}, in_text);
+
+scan("<!--{% endblocktrans %}-->" ++ T, Scanned, {Row, Column}, in_text) ->
+    scan(T, [{close_blocktrans, {Row, Column}} | Scanned],
+        {Row, Column + length("<!--{% endblocktrans %}-->")}, {in_text});
+
+scan("<!--{%" ++ T, Scanned, {Row, Column}, in_text) ->
+    scan(T, append_text("<!--{%", {Row, Column}, Scanned),
+        {Row, Column + length("<!--{%")}, {in_code, "%}-->"});
+
+scan("{%" ++ T, Scanned, {Row, Column}, in_text) ->
+    scan(T, append_text("{%", {Row, Column}, Scanned),
+        {Row, Column + length("{%")}, {in_code, "%}"});
+
+scan([H | T], Scanned, {Row, Column}, {in_comment, Closer}) ->
+    scan(T, append_text([H], {Row, Column}, Scanned), {Row, Column + 1}, {in_comment, Closer});
+
+scan("\n" ++ T, Scanned, {Row, Column}, in_text) ->
+    scan(T, append_text("\n", {Row, Column}, Scanned), {Row + 1, 1}, in_text);
+
+scan([H | T], Scanned, {Row, Column}, in_text) ->
+    scan(T, append_text([H], {Row, Column}, Scanned), {Row, Column + 1}, in_text);
+
+scan("\"" ++ T, Scanned, {Row, Column}, {in_code, Closer}) ->
+    scan(T, append_text("\"", {Row, Column}, Scanned), {Row, Column + 1} , {in_double_quote, Closer});
+
+scan("\'" ++ T, Scanned, {Row, Column}, {in_code, Closer}) ->
+    scan(T, append_text("\'", {Row, Column}, Scanned), {Row, Column + 1}, {in_single_quote, Closer});
+
+scan("\\" ++ T, Scanned, {Row, Column}, {in_double_quote, Closer}) ->
+    scan(T, append_text("\\", {Row, Column}, Scanned), {Row, Column + 1}, {in_double_quote_slash, Closer});
+
+scan([H | T], Scanned, {Row, Column}, {in_double_quote_slash, Closer}) ->
+    scan(T, append_text([H], {Row, Column}, Scanned), {Row, Column + 1}, {in_double_quote, Closer});
+
+scan("\\" ++ T, Scanned, {Row, Column}, {in_single_quote, Closer}) ->
+    scan(T, append_text("\\", {Row, Column}, Scanned), {Row, Column + 1}, {in_single_quote_slash, Closer});
+
+scan([H | T], Scanned, {Row, Column}, {in_single_quote_slash, Closer}) ->
+    scan(T, append_text([H], {Row, Column}, Scanned), {Row, Column + 1}, {in_single_quote, Closer});
+
+% end quote
+scan("\"" ++ T, Scanned, {Row, Column}, {in_double_quote, Closer}) ->
+    scan(T, append_text("\"", {Row, Column}, Scanned), {Row, Column + 1}, {in_code, Closer});
+
+scan("\'" ++ T, Scanned, {Row, Column}, {in_single_quote, Closer}) ->
+    scan(T, append_text("\'", {Row, Column}, Scanned), {Row, Column + 1}, {in_code, Closer});
+
+scan([H | T], Scanned, {Row, Column}, {in_double_quote, Closer}) ->
+    scan(T, append_text([H], {Row, Column}, Scanned), {Row, Column + 1}, {in_double_quote, Closer});
+
+scan([H | T], Scanned, {Row, Column}, {in_single_quote, Closer}) ->
+    scan(T, append_text([H], {Row, Column}, Scanned), {Row, Column + 1}, {in_single_quote, Closer});
+
+
+scan("}}-->" ++ T, Scanned, {Row, Column}, {_, "}}-->"}) ->
+    scan(T, append_text("}}-->", {Row, Column}, Scanned),
+        {Row, Column + length("}}-->")}, in_text);
+
+scan("}}" ++ T, Scanned, {Row, Column}, {_, "}}"}) ->
+    scan(T, append_text("}}", {Row, Column}, Scanned), {Row, Column + length("}}")}, in_text);
+
+scan("%}-->" ++ T, Scanned, {Row, Column}, {_, "%}-->"}) ->
+    scan(T, append_text("%}-->", {Row, Column}, Scanned),
+        {Row, Column + length("%}-->")}, in_text);
+
+scan("%}" ++ T, Scanned, {Row, Column}, {_, "%}"}) ->
+    scan(T, append_text("%}", {Row, Column}, Scanned),
+        {Row, Column + length("%}")}, in_text);
+
+
+scan([H | T], Scanned, {Row, Column}, {in_code, Closer}) ->
+    scan(T, append_text([H], {Row, Column}, Scanned), {Row, Column + 1}, {in_code, Closer}).
+
+% internal functions
+
+append_text(Text, Pos, []) ->
+    [{text, Pos, Text}];
+append_text(Text, Pos, [{close_blocktrans, _}|_] = Scanned) ->
+    [{text, Pos, Text}|Scanned];
+append_text([C], _Pos, [{open_blocktrans, BPos, ""}|Rest]) when ((C >= $a) and (C =< $z)) or ((C >= $A) and (C =< $Z)) or (C =:= $_) ->
+    [{open_blocktrans, BPos, [C]}|Rest];
+append_text(" ", _Pos, [{open_blocktrans, BPos, ""}|Rest]) ->
+    [{open_blocktrans, BPos, ""}|Rest];
+append_text([C], _Pos, [{open_blocktrans, BPos, Name}|Rest]) when ((C >= $a) and (C =< $z)) or ((C >= $A) and (C =< $Z)) or (C =:= $_) orelse (C >= $0 andalso C =< $9) ->
+    [{open_blocktrans, BPos, [C|Name]}|Rest];
+append_text(" ", _Pos, [{open_blocktrans, BPos, Name}|Rest]) when is_list(Name) ->
+    [{open_blocktrans, BPos, list_to_atom(lists:reverse(Name))}|Rest];
+append_text("%}", {Row, Column}, [{open_blocktrans, _BPos, _Name}|_] = Scanned) ->
+    [{text, {Row, Column + 2}, ""}|Scanned];
+append_text(_Chars, _Pos, [{open_blocktrans, _BPos, _Name}|_] = Scanned) ->
+    Scanned;
+append_text(Chars, _Pos, [{text, TPos, TChars}|Rest]) ->
+    [{text, TPos, lists:reverse(Chars, TChars)}|Rest].