Unofficial Dub Package Documentation 
1 /**
2   * DJinja parser
3   *
4   * Copyright:
5   *     Copyright (c) 2018, Maxim Tyapkin.
6   * Authors:
7   *     Maxim Tyapkin
8   * License:
9   *     This software is licensed under the terms of the BSD 3-clause license.
10   *     The full terms of the license can be found in the LICENSE.md file.
11   */
12 
13 module djinja.parser;
14 
15 public
16 {
17     import djinja.lexer : Position;
18 }
19 
20 private
21 {
22     import std.array : appender;
23     import std.conv : to;
24     import std.file : exists, read;
25     import std.format: fmt = format;
26     import std.range;
27 
28     import djinja.ast;
29     import djinja.lexer;
30     import djinja.exception : JinjaParserException,
31                               assertJinja = assertJinjaParser;
32 }
33 
34 
35 struct Parser(Lexer)
36 {
37     struct ParserState
38     {
39         Token[] tokens;
40         BlockNode[string] blocks;
41     }
42 
43     private
44     {
45         TemplateNode[string] _parsedFiles;
46 
47         Token[] _tokens;
48         BlockNode[string] _blocks;
49 
50         ParserState[] _states;
51     }
52 
53 
54     void preprocess()
55     {
56         import std..string : stripRight, stripLeft;
57 
58         auto newTokens = appender!(Token[]);
59 
60         for (int i = 0; i < _tokens.length;)
61         {
62             if (i < _tokens.length - 1
63                 && _tokens[i] == Type.StmtBegin
64                 && _tokens[i+1] == Operator.Minus)
65             {
66                 newTokens.put(_tokens[i]);
67                 i += 2;
68             }
69             else if(i < _tokens.length - 1
70                     && _tokens[i] == Operator.Minus
71                     && _tokens[i+1] == Type.StmtEnd)
72             {
73                 newTokens.put(_tokens[i+1]);
74                 i += 2;
75             }
76             else if (_tokens[i] == Type.Raw)
77             {
78                 bool stripR = false;
79                 bool stripL = false;
80                 bool stripInlineR = false;
81 
82                 if (i >= 2 
83                     && _tokens[i-2] == Operator.Minus
84                     && _tokens[i-1] == Type.StmtEnd
85                     )
86                     stripL = true;
87 
88                 if (i < _tokens.length - 2 && _tokens[i+1] == Type.StmtBegin)
89                 {
90                     if (_tokens[i+2] == Operator.Minus)
91                         stripR = true;
92                     else if (_tokens[i+1].value == Lexer.stmtInline)
93                         stripInlineR = true;
94                 }
95 
96                 auto str = _tokens[i].value;
97                 str = stripR ? str.stripRight : str;
98                 str = stripL ? str.stripLeft : str;
99                 str = stripInlineR ? str.stripOnceRight : str;
100                 newTokens.put(Token(Type.Raw, str, _tokens[i].pos));
101                 i++;
102             }
103             else
104             {
105                 newTokens.put(_tokens[i]);
106                 i++;
107             }
108         }
109         _tokens = newTokens.data;
110     }
111 
112 
113     TemplateNode parseTree(string str, string filename = "main")
114     {
115         stashState();
116 
117         auto lexer = Lexer(str, filename);
118         auto newTokens = appender!(Token[]);
119 
120         while (true)
121         {
122             auto tkn = lexer.nextToken;
123             newTokens.put(tkn); 
124             if (tkn.type == Type.EOF)
125                 break;
126         }
127         _tokens = newTokens.data;
128 
129         preprocess();
130 
131         auto root = parseStatementBlock();
132         auto blocks = _blocks;
133 
134         if (front.type != Type.EOF)
135             assertJinja(0, "Expected EOF found %s(%s)".fmt(front.type, front.value), front.pos);
136 
137         popState();
138 
139         return new TemplateNode(Position(filename, 1, 1), root, blocks);
140     }
141 
142 
143     TemplateNode parseTreeFromFile(string path)
144     {
145         path = path.absolute;
146 
147         if (auto cached = path in _parsedFiles)
148         {
149             if (*cached is null)
150                 assertJinja(0, "Recursive imports/includes/extends not allowed: ".fmt(path), front.pos);
151             else
152                 return *cached;
153         }
154 
155         // Prevent recursive imports
156         _parsedFiles[path] = null;
157         auto str = cast(string)read(path);
158         _parsedFiles[path] = parseTree(str, path);
159 
160         return _parsedFiles[path];
161     }
162 
163 
164 private: 
165 
166 
167     /**
168       * exprblock = EXPRBEGIN expr (IF expr (ELSE expr)? )? EXPREND
169       */
170     ExprNode parseExpression()
171     {
172         Node expr;
173         auto pos = front.pos;
174 
175         pop(Type.ExprBegin);
176         expr = parseHighLevelExpression();
177         pop(Type.ExprEnd);
178 
179         return new ExprNode(pos, expr);
180     }
181 
182     StmtBlockNode parseStatementBlock()
183     {
184         auto block = new StmtBlockNode(front.pos);
185 
186         while (front.type != Type.EOF)
187         {
188             auto pos = front.pos;
189             switch(front.type) with (Type)
190             {
191                 case Raw:
192                     auto raw = pop.value;
193                     if (raw.length)
194                         block.children ~= new RawNode(pos, raw);
195                     break;
196 
197                 case ExprBegin:
198                     block.children ~= parseExpression();
199                     break;
200 
201                 case CmntBegin:
202                     parseComment();
203                     break;
204 
205                 case CmntInline:
206                     pop();
207                     break;
208 
209                 case StmtBegin:
210                     if (next.type == Type.Keyword
211                         && next.value.toKeyword.isBeginingKeyword)
212                         block.children ~= parseStatement();
213                     else
214                         return block;
215                     break;
216 
217                 default:
218                     return block;
219             }
220         }
221 
222         return block;
223     }
224 
225 
226     Node parseStatement()
227     {
228         pop(Type.StmtBegin);
229 
230         switch(front.value) with (Keyword)
231         {
232             case If:      return parseIf();
233             case For:     return parseFor();
234             case Set:     return parseSet();
235             case Macro:   return parseMacro();
236             case Call:    return parseCall();
237             case Filter:  return parseFilterBlock();
238             case With:    return parseWith();
239             case Import:  return parseImport();
240             case From:    return parseImportFrom();
241             case Include: return parseInclude();
242             case Extends: return parseExtends();
243 
244             case Block:
245                 auto block = parseBlock();
246                 _blocks[block.name] = block;
247                 return block;
248             default:
249                 assert(0, "Not implemented kw %s".fmt(front.value));
250         }
251     }
252 
253 
254     ForNode parseFor()
255     {
256         string[] keys;
257         bool isRecursive = false;
258         Node cond = null;
259         auto pos = front.pos;
260 
261         pop(Keyword.For);
262 
263         keys ~= pop(Type.Ident).value;
264         while(front != Operator.In)
265         {
266             pop(Type.Comma);
267             keys ~= pop(Type.Ident).value;
268         }
269 
270         pop(Operator.In);
271         
272         Node iterable;
273 
274         switch (front.type) with (Type)
275         {
276             case LParen:  iterable = parseTuple(); break;
277             case LSParen: iterable = parseList(); break;
278             case LBrace:  iterable = parseDict; break;
279             default:      iterable = parseIdent();
280         }
281 
282         if (front == Keyword.If)
283         {
284             pop(Keyword.If);
285             cond = parseHighLevelExpression();
286         }
287 
288         if (front == Keyword.Recursive)
289         {
290             pop(Keyword.Recursive);
291             isRecursive = true;
292         }
293 
294         pop(Type.StmtEnd);
295 
296         auto block = parseStatementBlock();
297 
298         pop(Type.StmtBegin);
299 
300         switch (front.value) with (Keyword)
301         {
302             case EndFor:
303                 pop(Keyword.EndFor);
304                 pop(Type.StmtEnd);
305                 return new ForNode(pos, keys, iterable, block, null, cond, isRecursive);
306             case Else:
307                 pop(Keyword.Else);
308                 pop(Type.StmtEnd);
309                 auto other = parseStatementBlock();
310                 pop(Type.StmtBegin);
311                 pop(Keyword.EndFor);
312                 pop(Type.StmtEnd);
313                 return new ForNode(pos, keys, iterable, block, other, cond, isRecursive);
314             default:
315                 assertJinja(0, "Unexpected token %s(%s)".fmt(front.type, front.value), front.pos);
316                 assert(0);
317         }
318     }
319 
320 
321     IfNode parseIf()
322     {
323         auto pos = front.pos;
324         assertJinja(front == Keyword.If || front == Keyword.ElIf, "Expected If/Elif", pos);
325         pop();
326         auto cond = parseHighLevelExpression();
327         pop(Type.StmtEnd);
328 
329         auto then = parseStatementBlock();
330 
331         pop(Type.StmtBegin);
332 
333         switch (front.value) with (Keyword)
334         {
335             case ElIf:
336                 auto other = parseIf();
337                 return new IfNode(pos, cond, then, other);
338             case Else:
339                 pop(Keyword.Else, Type.StmtEnd);
340                 auto other = parseStatementBlock();
341                 pop(Type.StmtBegin, Keyword.EndIf, Type.StmtEnd);
342                 return new IfNode(pos, cond, then, other);
343             case EndIf:
344                 pop(Keyword.EndIf, Type.StmtEnd);
345                 return new IfNode(pos, cond, then, null);
346             default:
347                 assertJinja(0, "Unexpected token %s(%s)".fmt(front.type, front.value), front.pos);
348                 assert(0);
349         }
350     }
351 
352 
353     SetNode parseSet()
354     {
355         auto setPos = front.pos;
356 
357         pop(Keyword.Set);
358 
359         auto assigns = parseSequenceOf!parseAssignable(Type.Operator);
360 
361         pop(Operator.Assign);
362 
363         auto listPos = front.pos;
364         auto exprs = parseSequenceOf!parseHighLevelExpression(Type.StmtEnd);
365         Node expr = exprs.length == 1 ? exprs[0] : new ListNode(listPos, exprs);
366 
367         pop(Type.StmtEnd);
368 
369         return new SetNode(setPos, assigns, expr);
370     }
371 
372 
373     AssignableNode parseAssignable()
374     {
375         auto pos = front.pos;
376         string name = pop(Type.Ident).value;
377         Node[] subIdents = [];
378 
379         while (true)
380         {
381             switch (front.type) with (Type)
382             {
383                 case Dot:
384                     pop(Dot);
385                     auto strPos = front.pos;
386                     subIdents ~= new StringNode(strPos, pop(Ident).value);
387                     break;
388                 case LSParen:
389                     pop(LSParen);
390                     subIdents ~= parseHighLevelExpression();
391                     pop(RSParen);
392                     break;
393                 default:
394                     return new AssignableNode(pos, name, subIdents);
395             }
396         }
397     }
398 
399 
400     MacroNode parseMacro()
401     {
402         auto pos = front.pos;
403         pop(Keyword.Macro);
404 
405         auto name = pop(Type.Ident).value;
406         Arg[] args;
407 
408         if (front.type == Type.LParen)
409         {
410             pop(Type.LParen);
411             args = parseFormalArgs();
412             pop(Type.RParen);
413         }
414 
415         pop(Type.StmtEnd);
416 
417         auto block = parseStatementBlock();
418 
419         pop(Type.StmtBegin, Keyword.EndMacro);
420 
421         bool ret = false;
422         if (front.type == Type.Keyword && front.value == Keyword.Return)
423         {
424             pop(Keyword.Return);
425             block.children ~= parseHighLevelExpression();
426             ret = true;
427         }
428         else
429             block.children ~= new NilNode; // void return
430 
431         pop(Type.StmtEnd);
432 
433         return new MacroNode(pos, name, args, block, ret);
434     }
435 
436     
437     CallNode parseCall()
438     {
439         auto pos = front.pos;
440         pop(Keyword.Call);
441         
442         Arg[] formalArgs;
443 
444         if (front.type == Type.LParen)
445         {
446             pop(Type.LParen);
447             formalArgs = parseFormalArgs();
448             pop(Type.RParen);
449         }
450 
451         auto macroName = front.value;
452         auto factArgs = parseCallExpr();
453 
454         pop(Type.StmtEnd);
455 
456         auto block = parseStatementBlock();
457         block.children ~= new NilNode; // void return
458 
459         pop(Type.StmtBegin, Keyword.EndCall, Type.StmtEnd);
460 
461         return new CallNode(pos, macroName, formalArgs, factArgs, block);
462     }
463 
464 
465     FilterBlockNode parseFilterBlock()
466     {
467         auto pos = front.pos;
468         pop(Keyword.Filter);
469 
470         auto filterName = front.value;
471         auto args = parseCallExpr();
472 
473         pop(Type.StmtEnd);
474 
475         auto block = parseStatementBlock();
476 
477         pop(Type.StmtBegin, Keyword.EndFilter, Type.StmtEnd);
478 
479         return new FilterBlockNode(pos, filterName, args, block);
480     }
481     
482 
483     StmtBlockNode parseWith()
484     {
485         pop(Keyword.With, Type.StmtEnd);
486         auto block = parseStatementBlock();
487         pop(Type.StmtBegin, Keyword.EndWith, Type.StmtEnd);
488 
489         return block;
490     }
491 
492 
493     ImportNode parseImport()
494     {
495         auto pos = front.pos;
496         pop(Keyword.Import);
497         auto path = pop(Type.String).value.absolute;
498         bool withContext = false;
499 
500         if (front == Keyword.With)
501         {
502             withContext = true;
503             pop(Keyword.With, Keyword.Context);
504         }
505 
506         if (front == Keyword.Without)
507         {
508             withContext = false;
509             pop(Keyword.Without, Keyword.Context);
510         }
511 
512         pop(Type.StmtEnd);
513 
514         assertJinja(path.exists, "Non existing file `%s`".fmt(path), pos);
515         
516         auto stmtBlock = parseTreeFromFile(path);
517         
518         return new ImportNode(pos, path, cast(ImportNode.Rename[])[], stmtBlock, withContext);
519     }
520 
521 
522     ImportNode parseImportFrom()
523     {
524         auto pos = front.pos;
525         pop(Keyword.From);
526         auto path = pop(Type.String).value.absolute;
527         pop(Keyword.Import);
528 
529         ImportNode.Rename[] macros;
530 
531         bool firstName = true;
532         while (front == Type.Comma || firstName)
533         {
534             if (!firstName)
535                 pop(Type.Comma);
536 
537             auto was = pop(Type.Ident).value;
538             auto become = was;
539 
540             if (front == Keyword.As)
541             {
542                 pop(Keyword.As);
543                 become = pop(Type.Ident).value;
544             }
545 
546             macros ~= ImportNode.Rename(was, become);
547 
548             firstName = false;
549         }
550 
551         bool withContext = false;
552 
553         if (front == Keyword.With)
554         {
555             withContext = true;
556             pop(Keyword.With, Keyword.Context);
557         }
558 
559         if (front == Keyword.Without)
560         {
561             withContext = false;
562             pop(Keyword.Without, Keyword.Context);
563         }
564 
565         pop(Type.StmtEnd);
566 
567         assertJinja(path.exists, "Non existing file `%s`".fmt(path), pos);
568         
569         auto stmtBlock = parseTreeFromFile(path);
570         
571         return new ImportNode(pos, path, macros, stmtBlock, withContext);
572     }
573 
574 
575     IncludeNode parseInclude()
576     {
577         auto pos = front.pos;
578         pop(Keyword.Include);
579 
580         string[] names;
581 
582         if (front == Type.LSParen)
583         {
584             pop(Type.LSParen);
585 
586             names ~= pop(Type.String).value;
587             while (front == Type.Comma)
588             {
589                 pop(Type.Comma);
590                 names ~= pop(Type.String).value;
591             }
592 
593             pop(Type.RSParen);
594         }
595         else
596             names ~= pop(Type.String).value;
597 
598 
599         bool ignoreMissing = false;
600         if (front == Keyword.Ignore)
601         {
602             pop(Keyword.Ignore, Keyword.Missing);
603             ignoreMissing = true;
604         }
605 
606         bool withContext = true;
607 
608         if (front == Keyword.With)
609         {
610             withContext = true;
611             pop(Keyword.With, Keyword.Context);
612         }
613 
614         if (front == Keyword.Without)
615         {
616             withContext = false;
617             pop(Keyword.Without, Keyword.Context);
618         }
619 
620         pop(Type.StmtEnd);
621 
622         foreach (name; names)
623             if (name.exists)
624                 return new IncludeNode(pos, name, parseTreeFromFile(name), withContext);
625  
626         assertJinja(ignoreMissing, "No existing files `%s`".fmt(names), pos);
627         
628         return new IncludeNode(pos, "", null, withContext);
629     }
630 
631 
632     ExtendsNode parseExtends()
633     {
634         auto pos = front.pos;
635         pop(Keyword.Extends);
636         auto path = pop(Type.String).value.absolute;
637         pop(Type.StmtEnd);
638 
639         assertJinja(path.exists, "Non existing file `%s`".fmt(path), pos);
640         
641         auto stmtBlock = parseTreeFromFile(path);
642         
643         return new ExtendsNode(pos, path, stmtBlock);
644     }
645 
646 
647     BlockNode parseBlock()
648     {
649         auto pos = front.pos;
650         pop(Keyword.Block);
651         auto name = pop(Type.Ident).value;
652         pop(Type.StmtEnd);
653 
654         auto stmt = parseStatementBlock();
655 
656         pop(Type.StmtBegin, Keyword.EndBlock);
657 
658         auto posNameEnd = front.pos;
659         if (front == Type.Ident)
660             assertJinja(pop.value == name, "Missmatching block's begin/end names", posNameEnd);
661 
662         pop(Type.StmtEnd);
663 
664         return new BlockNode(pos, name, stmt);
665     }
666 
667     Arg[] parseFormalArgs()
668     {
669         Arg[] args = [];
670         bool isVarargs = true;
671 
672         while(front.type != Type.EOF && front.type != Type.RParen)
673         {
674             auto name = pop(Type.Ident).value;
675             Node def = null;
676 
677             if (!isVarargs || front.type == Type.Operator && front.value == Operator.Assign)
678             {
679                 isVarargs = false;
680                 pop(Operator.Assign);
681                 def = parseHighLevelExpression();
682             }
683 
684             args ~= Arg(name, def);
685             
686             if (front.type != Type.RParen)
687                 pop(Type.Comma);
688         }
689         return args;
690     }
691 
692 
693     Node parseHighLevelExpression()
694     {
695         return parseInlineIf();
696     }
697 
698 
699     /**
700       * inlineif = orexpr (IF orexpr (ELSE orexpr)? )?
701       */
702     Node parseInlineIf()
703     {
704         Node expr;
705         Node cond = null;
706         Node other = null;
707 
708         auto pos = front.pos;
709         expr = parseOrExpr();
710 
711         if (front == Keyword.If)
712         {
713             pop(Keyword.If);
714             cond = parseOrExpr();
715 
716             if (front == Keyword.Else)
717             {
718                 pop(Keyword.Else);
719                 other = parseOrExpr();
720             }
721 
722             return new InlineIfNode(pos, expr, cond, other);
723         }
724 
725         return expr;
726     }
727 
728     /**
729       * Parse Or Expression
730       * or = and (OR or)?
731       */
732     Node parseOrExpr()
733     {
734         auto lhs = parseAndExpr();
735 
736         while(true)
737         {
738             if (front.type == Type.Operator && front.value == Operator.Or)
739             {
740                 auto pos = front.pos;
741                 pop(Operator.Or);
742                 auto rhs = parseAndExpr();
743                 lhs = new BinOpNode(pos, Operator.Or, lhs, rhs);
744             }
745             else
746                 return lhs;
747         }
748     }
749 
750     /**
751       * Parse And Expression:
752       * and = inis (AND inis)*
753       */
754     Node parseAndExpr()
755     {
756         auto lhs = parseInIsExpr();
757 
758         while(true)
759         {
760             if (front.type == Type.Operator && front.value == Operator.And)
761             {
762                 auto pos = front.pos;
763                 pop(Operator.And);
764                 auto rhs = parseInIsExpr();
765                 lhs = new BinOpNode(pos, Operator.And, lhs, rhs);
766             }
767             else
768                 return lhs;
769         }
770     }
771 
772     /**
773       * Parse inis:
774       * inis = cmp ( (NOT)? (IN expr |IS callexpr) )?
775       */
776     Node parseInIsExpr()
777     {
778         auto inis = parseCmpExpr();
779 
780         auto notPos = front.pos;
781         bool hasNot = false;
782         if (front == Operator.Not && (next == Operator.In || next == Operator.Is))
783         {
784             pop(Operator.Not);
785             hasNot = true;
786         }
787 
788         auto inisPos = front.pos;
789 
790         if (front == Operator.In)
791         {
792             auto op = pop().value;
793             auto rhs = parseHighLevelExpression();
794             inis = new BinOpNode(inisPos, op, inis, rhs);
795         }
796 
797         if (front == Operator.Is)
798         {
799             auto op = pop().value;
800             auto rhs = parseCallExpr();
801             inis = new BinOpNode(inisPos, op, inis, rhs);
802         }
803 
804         if (hasNot)
805             inis = new UnaryOpNode(notPos, Operator.Not, inis);
806 
807         return inis;
808     }
809 
810 
811     /**
812       * Parse compare expression:
813       * cmp = concatexpr (CMPOP concatexpr)?
814       */
815     Node parseCmpExpr()
816     {
817         auto lhs = parseConcatExpr();
818 
819         if (front.type == Type.Operator && front.value.toOperator.isCmpOperator)
820         {
821             auto pos = front.pos;
822             auto op = pop(Type.Operator).value;
823             return new BinOpNode(pos, op, lhs, parseConcatExpr());
824         }
825 
826         return lhs;
827     }
828 
829     /**
830       * Parse expression:
831       * concatexpr = filterexpr (CONCAT filterexpr)*
832       */
833     Node parseConcatExpr()
834     {
835         auto lhsTerm = parseFilterExpr();
836 
837         while (front == Operator.Concat)
838         {
839             auto pos = front.pos;
840             auto op = pop(Operator.Concat).value;
841             lhsTerm = new BinOpNode(pos, op, lhsTerm, parseFilterExpr());
842         }
843 
844         return lhsTerm;
845     }
846 
847     /**
848       * filterexpr = mathexpr (FILTER callexpr)*
849       */
850     Node parseFilterExpr()
851     {
852         auto filterexpr = parseMathExpr();
853 
854         while (front == Operator.Filter)
855         {
856             auto pos = front.pos;
857             auto op = pop(Operator.Filter).value;
858             filterexpr = new BinOpNode(pos, op, filterexpr, parseCallExpr());
859         }
860 
861         return filterexpr;
862     }
863 
864     /**
865       * Parse math expression:
866       * mathexpr = term((PLUS|MINUS)term)*
867       */
868     Node parseMathExpr()
869     {
870         auto lhsTerm = parseTerm();
871 
872         while (true)
873         {
874             if (front.type != Type.Operator)
875                 return lhsTerm;
876 
877             auto pos = front.pos;
878             switch (front.value) with (Operator)
879             {
880                 case Plus:
881                 case Minus:
882                     auto op = pop.value;
883                     lhsTerm = new BinOpNode(pos, op, lhsTerm, parseTerm());
884                     break;
885                 default:
886                     return lhsTerm;
887             }
888         }
889     }
890 
891     /**
892       * Parse term:
893       * term = unary((MUL|DIVI|DIVF|REM)unary)*
894       */
895     Node parseTerm()
896     {
897         auto lhsFactor = parseUnary();
898 
899         while(true)
900         {
901             if (front.type != Type.Operator)
902                 return lhsFactor;
903         
904             auto pos = front.pos;
905             switch (front.value) with (Operator)
906             {
907                 case DivInt:
908                 case DivFloat:
909                 case Mul:
910                 case Rem:
911                     auto op = pop.value;
912                     lhsFactor = new BinOpNode(pos, op, lhsFactor, parseUnary());
913                     break;
914                 default:
915                     return lhsFactor;
916             }
917         } 
918     }
919 
920     /**
921       * Parse unary:
922       * unary = (pow | (PLUS|MINUS|NOT)unary)
923       */
924     Node parseUnary()
925     {
926         if (front.type != Type.Operator)
927             return parsePow();
928 
929         auto pos = front.pos;
930         switch (front.value) with (Operator)
931         {
932             case Plus:
933             case Minus:
934             case Not:
935                 auto op = pop.value;
936                 return new UnaryOpNode(pos, op, parseUnary());
937             default:
938                 assertJinja(0, "Unexpected operator `%s`".fmt(front.value), front.pos);
939                 assert(0);
940         }
941     }
942 
943     /**
944       * Parse pow:
945       * pow = factor (POW pow)?
946       */
947     Node parsePow()
948     {
949         auto lhs = parseFactor();
950 
951         if (front.type == Type.Operator && front.value == Operator.Pow)
952         {
953             auto pos = front.pos;
954             auto op = pop(Operator.Pow).value;
955             return new BinOpNode(pos, op, lhs, parsePow());
956         }
957 
958         return lhs;
959     }
960 
961 
962     /**
963       * Parse factor:
964       * factor = (ident|(tuple|LPAREN HighLevelExpr RPAREN)|literal)
965       */
966     Node parseFactor()
967     {
968         switch (front.type) with (Type)
969         {
970             case Ident:
971                 return parseIdent();
972 
973             case LParen:
974                 auto pos = front.pos;
975                 pop(LParen);
976                 bool hasCommas;
977                 auto exprList = parseSequenceOf!parseHighLevelExpression(RParen, hasCommas);
978                 pop(RParen);
979                 return hasCommas ? new ListNode(pos, exprList) : exprList[0];
980 
981             default:
982                 return parseLiteral();
983         }
984     }
985 
986     /**
987       * Parse ident:
988       * ident = IDENT (LPAREN ARGS RPAREN)? (DOT IDENT (LP ARGS RP)?| LSPAREN STR LRPAREN)*
989       */
990     Node parseIdent()
991     {
992         string name = "";
993         Node[] subIdents = [];
994         auto pos = front.pos;
995 
996         if (next.type == Type.LParen)
997             subIdents ~= parseCallExpr();
998         else
999             name = pop(Type.Ident).value;
1000 
1001         while (true)
1002         {
1003             switch (front.type) with (Type)
1004             {
1005                 case Dot:
1006                     pop(Dot);
1007                     auto posStr = front.pos;
1008                     if (next.type == Type.LParen)
1009                         subIdents ~= parseCallExpr();
1010                     else
1011                         subIdents ~= new StringNode(posStr, pop(Ident).value);
1012                     break;
1013                 case LSParen:
1014                     pop(LSParen);
1015                     subIdents ~= parseHighLevelExpression();
1016                     pop(RSParen);
1017                     break;
1018                 default:
1019                     return new IdentNode(pos, name, subIdents);
1020             }
1021         }
1022     }
1023 
1024 
1025     IdentNode parseCallIdent()
1026     {
1027         auto pos = front.pos;
1028         return new IdentNode(pos, "", [parseCallExpr()]);
1029     }
1030 
1031 
1032     DictNode parseCallExpr()
1033     {
1034         auto pos = front.pos;
1035         string name = pop(Type.Ident).value;
1036         Node[] varargs;
1037         Node[string] kwargs;
1038 
1039         bool parsingKwargs = false;
1040         void parse()
1041         {
1042             if (parsingKwargs || front.type == Type.Ident && next.value == Operator.Assign)
1043             {
1044                 parsingKwargs = true;
1045                 auto name = pop(Type.Ident).value;
1046                 pop(Operator.Assign);
1047                 kwargs[name] = parseHighLevelExpression();
1048             }
1049             else
1050                 varargs ~= parseHighLevelExpression();
1051         }
1052 
1053         if (front.type == Type.LParen)
1054         {
1055             pop(Type.LParen);
1056 
1057             while (front.type != Type.EOF && front.type != Type.RParen)
1058             {
1059                 parse();
1060 
1061                 if (front.type != Type.RParen)
1062                     pop(Type.Comma);
1063             }
1064 
1065             pop(Type.RParen);
1066         }
1067 
1068         Node[string] callDict;
1069         callDict["name"] = new StringNode(pos, name);
1070         callDict["varargs"] = new ListNode(pos, varargs);
1071         callDict["kwargs"] = new DictNode(pos, kwargs);
1072 
1073         return new DictNode(pos, callDict);
1074     }
1075 
1076     /**
1077       * literal = string|number|list|tuple|dict
1078       */
1079     Node parseLiteral()
1080     {
1081         auto pos = front.pos;
1082         switch (front.type) with (Type)
1083         {
1084             case Integer: return new NumNode(pos, pop.value.to!long);
1085             case Float:   return new NumNode(pos, pop.value.to!double);
1086             case String:  return new StringNode(pos, pop.value);
1087             case Boolean: return new BooleanNode(pos, pop.value.to!bool);
1088             case LParen:  return parseTuple();
1089             case LSParen: return parseList();
1090             case LBrace:  return parseDict();
1091             default:
1092                 assertJinja(0, "Unexpected token while parsing expression: %s(%s)".fmt(front.type, front.value), front.pos);
1093                 assert(0);
1094         }
1095     }
1096 
1097 
1098     Node parseTuple()
1099     {
1100         //Literally array right now
1101 
1102         auto pos = front.pos;
1103         pop(Type.LParen);
1104         auto tuple = parseSequenceOf!parseHighLevelExpression(Type.RParen);
1105         pop(Type.RParen);
1106 
1107         return new ListNode(pos, tuple);
1108     }
1109 
1110 
1111     Node parseList()
1112     {
1113         auto pos = front.pos;
1114         pop(Type.LSParen);
1115         auto list = parseSequenceOf!parseHighLevelExpression(Type.RSParen);
1116         pop(Type.RSParen);
1117 
1118         return new ListNode(pos, list);
1119     }
1120 
1121 
1122     Node[] parseSequenceOf(alias parser)(Type stopSymbol)
1123     {
1124         bool hasCommas;
1125         return parseSequenceOf!parser(stopSymbol, hasCommas);
1126     }
1127 
1128 
1129     Node[] parseSequenceOf(alias parser)(Type stopSymbol, ref bool hasCommas)
1130     {
1131         Node[] seq;
1132 
1133         hasCommas = false;
1134         while (front.type != stopSymbol && front.type != Type.EOF)
1135         {
1136             seq ~= parser();
1137 
1138             if (front.type != stopSymbol)
1139             {
1140                 pop(Type.Comma);
1141                 hasCommas = true;
1142             }
1143         }
1144 
1145         return seq;
1146     }
1147 
1148 
1149     Node parseDict()
1150     {
1151         Node[string] dict;
1152         auto pos = front.pos;
1153 
1154         pop(Type.LBrace);
1155 
1156         bool isFirst = true;
1157         while (front.type != Type.RBrace && front.type != Type.EOF)
1158         {
1159             if (!isFirst)
1160                 pop(Type.Comma);
1161 
1162             string key;
1163             if (front.type == Type.Ident)
1164                 key = pop(Type.Ident).value;
1165             else
1166                 key = pop(Type.String).value;
1167 
1168             pop(Type.Colon);
1169             dict[key] = parseHighLevelExpression();
1170             isFirst = false;
1171         }
1172 
1173         if (front.type == Type.Comma)
1174             pop(Type.Comma);
1175 
1176         pop(Type.RBrace);
1177 
1178         return new DictNode(pos, dict);
1179     }
1180 
1181 
1182     void parseComment()
1183     {
1184         pop(Type.CmntBegin);
1185         while (front.type != Type.CmntEnd && front.type != Type.EOF)
1186             pop();
1187         pop(Type.CmntEnd);
1188     }
1189 
1190 
1191     Token front()
1192     {
1193         if (_tokens.length)
1194             return _tokens[0];
1195         return Token.EOF;
1196     }
1197 
1198     Token next()
1199     {
1200         if (_tokens.length > 1)
1201             return _tokens[1];
1202         return Token.EOF;
1203     }
1204 
1205 
1206     Token pop()
1207     {
1208         auto tkn = front();
1209         if (_tokens.length)
1210             _tokens = _tokens[1 .. $];
1211         return tkn;
1212     }
1213 
1214 
1215     Token pop(Type t)
1216     {
1217         if (front.type != t)
1218             assertJinja(0, "Unexpected token %s(%s), expected: `%s`".fmt(front.type, front.value, t), front.pos);
1219         return pop();
1220     }
1221 
1222 
1223     Token pop(Keyword kw)
1224     {
1225         if (front.type != Type.Keyword || front.value != kw)
1226             assertJinja(0, "Unexpected token %s(%s), expected kw: %s".fmt(front.type, front.value, kw), front.pos);
1227         return pop();
1228     }
1229 
1230 
1231     Token pop(Operator op)
1232     {
1233         if (front.type != Type.Operator || front.value != op)
1234             assertJinja(0, "Unexpected token %s(%s), expected op: %s".fmt(front.type, front.value, op), front.pos);
1235         return pop();
1236     }
1237 
1238 
1239     void pop(T...)(T args)
1240         if (args.length > 1)
1241     {
1242         foreach(arg; args)
1243             pop(arg);
1244     }
1245 
1246 
1247     void stashState()
1248     {
1249         ParserState old;
1250         old.tokens = _tokens;
1251         old.blocks = _blocks;
1252         _states ~= old;
1253         _tokens = [];
1254         _blocks = (BlockNode[string]).init;
1255     }
1256 
1257 
1258     void popState()
1259     {
1260         assertJinja(_states.length > 0, "Unexpected empty state stack");
1261 
1262         auto state = _states.back;
1263         _states.popBack;
1264         _tokens = state.tokens;
1265         _blocks = state.blocks;
1266     }
1267 }
1268 
1269 
1270 private:
1271 
1272 
1273 string absolute(string path)
1274 {
1275     //TODO
1276     return path;
1277 }
1278 
1279 
1280 string stripOnceRight(string str)
1281 {
1282     import std.uni;
1283     import std.utf : codeLength;
1284 
1285     import std.traits;
1286     alias C = Unqual!(ElementEncodingType!string);
1287 
1288     bool stripped = false;
1289     foreach_reverse (i, dchar c; str)
1290     {
1291         if (!isWhite(c) || c == '\n' || c == '\r' || c == 0x2028 || c == 0x2029)
1292             return str[0 .. i + codeLength!C(c)];
1293     }
1294 
1295     return str[0 .. 0];
1296 }
Page generated by adrdox