diff --git a/ASTNode.h b/ASTNode.h index 98760dbf5..8aea23caf 100644 --- a/ASTNode.h +++ b/ASTNode.h @@ -18,7 +18,7 @@ class ASTNode { NODE_COMPREHENSION, NODE_LOADBUILDCLASS, NODE_AWAITABLE, NODE_FORMATTEDVALUE, NODE_JOINEDSTR, NODE_CONST_MAP, NODE_ANNOTATED_VAR, NODE_CHAINSTORE, NODE_TERNARY, - NODE_KW_NAMES_MAP, + NODE_KW_NAMES_MAP, NODE_CALL_INTRINSIC_1, NODE_CALL_INTRINSIC_2, NODE_UNPACKED_TUPLE, NODE_ASSERT, // Empty node types NODE_LOCALS, @@ -286,13 +286,28 @@ class ASTCall : public ASTNode { typedef std::list> pparam_t; typedef std::list, PycRef>> kwparam_t; + ASTCall(PycRef func, pparam_t pparams, kwparam_t kwparams) : ASTNode(NODE_CALL), m_func(std::move(func)), m_pparams(std::move(pparams)), m_kwparams(std::move(kwparams)) { } + ASTCall(): ASTNode(NODE_CALL){} + + bool isKwparamUnpacked(std::pair, PycRef> value) + { + return value.first == NULL; + } + + std::pair, PycRef> genKwparamUnpacked(PycRef value){ + return std::make_pair(m_unpacked_marker, value); + } PycRef func() const { return m_func; } const pparam_t& pparams() const { return m_pparams; } const kwparam_t& kwparams() const { return m_kwparams; } + void setFunc(PycRef func) { m_func = std::move(func); } + void setPparams(pparam_t pparams) { m_pparams = std::move(pparams); } + void setKwparams(kwparam_t kwparams) { m_kwparams = std::move(kwparams); } + PycRef var() const { return m_var; } PycRef kw() const { return m_kw; } @@ -308,6 +323,7 @@ class ASTCall : public ASTNode { kwparam_t m_kwparams; PycRef m_var; PycRef m_kw; + const PycRef m_unpacked_marker = NULL; }; @@ -339,6 +355,7 @@ class ASTTuple : public ASTNode { ASTTuple(value_t values) : ASTNode(NODE_TUPLE), m_values(std::move(values)), m_requireParens(true) { } + ASTTuple(enum ASTNode::Type subtype, value_t values, bool requireParens) : ASTNode(subtype), m_values(std::move(values)), m_requireParens(requireParens) {} const value_t& values() const { return m_values; } void add(PycRef name) { m_values.emplace_back(std::move(name)); } @@ -346,11 +363,39 @@ class ASTTuple : public ASTNode { void setRequireParens(bool require) { m_requireParens = require; } bool requireParens() const { return m_requireParens; } -private: +protected: value_t m_values; bool m_requireParens; }; +class ASTUnpackedTuple : public ASTTuple { +public: + ASTUnpackedTuple(u_int8_t before, u_int8_t after) : ASTTuple(NODE_UNPACKED_TUPLE, value_t(), true), m_before(std::move(before)), m_after(std::move(after)){} + + void add(PycRef value){ + if (m_before > 0){ + m_before--; + } else if (!m_unpackedValueAdded){ + m_unpackedValueAdded = true; + value.setUnpacked(true); + } else if (m_after > 0){ + m_after--; + } else{ + fputs("Cannot add new value to unpacked tuple!\n", stderr); + return; + } + setRequireParens(false); + m_values.emplace_back(std::move(value)); + } + bool isFull() { return m_unpackedValueAdded and (m_values.size() > (m_before + m_after)); } + u_int8_t unpackedBefore() { return m_before; } + u_int8_t unpackedAfter() { return m_after; } + +private: + u_int8_t m_before; + bool m_unpackedValueAdded = false; + u_int8_t m_after; +}; class ASTList : public ASTNode { public: @@ -383,33 +428,31 @@ class ASTMap : public ASTNode { typedef std::list, PycRef>> map_t; ASTMap() : ASTNode(NODE_MAP) { } + ASTMap(enum ASTNode::Type subtype) : ASTNode(subtype) {} void add(PycRef key, PycRef value) { m_values.emplace_back(std::move(key), std::move(value)); } + void add_unpacked_value(PycRef variable) + { + m_values.emplace_back(m_unpacked_marker, std::move(variable)); + } + bool is_unpacked(std::pair , PycRef> value) + { + return value.first == m_unpacked_marker; + } const map_t& values() const { return m_values; } private: map_t m_values; + const PycRef m_unpacked_marker = NULL; }; -class ASTKwNamesMap : public ASTNode { +class ASTKwNamesMap : public ASTMap { public: - typedef std::list, PycRef>> map_t; - - ASTKwNamesMap() : ASTNode(NODE_KW_NAMES_MAP) { } - - void add(PycRef key, PycRef value) - { - m_values.emplace_back(std::move(key), std::move(value)); - } - - const map_t& values() const { return m_values; } - -private: - map_t m_values; + ASTKwNamesMap() : ASTMap(NODE_KW_NAMES_MAP) { } }; class ASTConstMap : public ASTNode { @@ -509,6 +552,19 @@ class ASTRaise : public ASTNode { param_t m_params; }; +class ASTAssert : public ASTNode { +public: + ASTAssert() : ASTNode(NODE_ASSERT) {} + + const PycRef cond() const { return m_cond; } + const PycRef msg() const { return m_msg; } + void setCond(PycRef cond) { m_cond = std::move(cond); } + void setMsg(PycRef msg) { m_msg = std::move(msg); } + +private : + PycRef m_cond; + PycRef m_msg; +}; class ASTExec : public ASTNode { public: @@ -618,12 +674,37 @@ class ASTContainerBlock : public ASTBlock { int except() const { return m_except; } void setExcept(int except) { m_except = except; } + void setFinally(int finally) { m_finally = finally; } private: int m_finally; int m_except; }; +class ASTExceptBlock : public ASTBlock +{ +public: + ASTExceptBlock(int end) + : ASTBlock(ASTBlock::BLK_EXCEPT, end) {} + ASTExceptBlock(int end, PycRef expr) + : ASTBlock(ASTBlock::BLK_EXCEPT, end), m_expr(std::move(expr)) {} + + PycRef expr() const { return m_expr; } + PycRef var() const { return m_var; } + + void setExpr(PycRef expr) + { + m_expr = std::move(expr); + init(); + } + void setVar(PycRef var) { m_var = std::move(var); } + +private: + PycRef m_expr; + PycRef m_var; // optional value +}; + + class ASTWithBlock : public ASTBlock { public: ASTWithBlock(int end) @@ -644,9 +725,15 @@ class ASTComprehension : public ASTNode { public: typedef std::list> generator_t; - ASTComprehension(PycRef result) - : ASTNode(NODE_COMPREHENSION), m_result(std::move(result)) { } + enum CompType + { + COMP_LIST, COMP_DICT + }; + ASTComprehension(CompType comptype, PycRef result) + : ASTNode(NODE_COMPREHENSION), m_comptype(comptype), m_result(std::move(result)) {} + + CompType comptype() const { return m_comptype; } PycRef result() const { return m_result; } generator_t generators() const { return m_generators; } @@ -655,11 +742,31 @@ class ASTComprehension : public ASTNode { } private: + CompType m_comptype; PycRef m_result; generator_t m_generators; }; +class ASTListComprehension : public ASTComprehension +{ +public: + ASTListComprehension(PycRef result) + : ASTComprehension(COMP_LIST, std::move(result)) {} +}; + +class ASTDictComprehension : public ASTComprehension +{ +public: + ASTDictComprehension(PycRef key, PycRef value) + : ASTComprehension(COMP_DICT, std::move(value)), m_key(std::move(key)) {} + + PycRef key() const { return m_key; } + +private: + PycRef m_key; +}; + class ASTLoadBuildClass : public ASTNode { public: ASTLoadBuildClass(PycRef obj) @@ -757,4 +864,26 @@ class ASTTernary : public ASTNode PycRef m_else_expr; }; +class ASTCallIntrinsic1: public ASTNode +{ +public: + enum Function { + INTRINSIC_1_INVALID, INTRINSIC_PRINT, INTRINSIC_IMPORT_STAR, + INTRINSIC_STOPITERATION_ERROR, INTRINSIC_ASYNC_GEN_WRAP, + INTRINSIC_UNARY_POSITIVE, INTRINSIC_LIST_TO_TUPLE, INTRINSIC_TYPEVAR, + INTRINSIC_PARAMSPEC, INTRINSIC_TYPEVARTUPLE, + INTRINSIC_SUBSCRIPT_GENERIC, INTRINSIC_TYPEALIAS, + }; +}; + +class ASTCallIntrinsic2: public ASTNode +{ +public: + enum Function { + INTRINSIC_2_INVALID, INTRINSIC_PREP_RERAISE_STAR, + INTRINSIC_TYPEVAR_WITH_BOUND, INTRINSIC_TYPEVAR_WITH_CONSTRAINTS, + INTRINSIC_SET_FUNCTION_TYPE_PARAMS, INTRINSIC_SET_TYPEPARAM_DEFAULT, + }; +}; + #endif diff --git a/ASTree.cpp b/ASTree.cpp index 6635808e1..97abc6650 100644 --- a/ASTree.cpp +++ b/ASTree.cpp @@ -6,6 +6,7 @@ #include "FastStack.h" #include "pyc_numeric.h" #include "bytecode.h" +#include // This must be a triple quote (''' or """), to handle interpolated string literals containing the opposite quote style. // E.g. f'''{"interpolated "123' literal"}''' -> valid. @@ -40,6 +41,21 @@ static PycRef StackPopTop(FastStack& stack) return node; } +static FastStack &StackHistPopTop(FastStack &stack, stackhist_t &stack_hist) +{ + if (!stack_hist.empty()){ + stack = stack_hist.top(); + stack_hist.pop(); + } + return stack; +} + +static void StackPopIfNotEmpty(stackhist_t &stack) +{ + if (!stack.empty()) + stack.pop(); +} + /* compiler generates very, VERY similar byte code for if/else statement block and if-expression * statement * if a: b = 1 @@ -90,6 +106,7 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) int curpos = 0; int pos = 0; int unpack = 0; + int goto_addr = 0; bool else_pop = false; bool need_try = false; bool variable_annotations = false; @@ -111,12 +128,17 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) curpos = pos; bc_next(source, mod, opcode, operand, pos); + if (goto_addr != 0 && goto_addr >= pos){ + continue; + } + if (need_try && opcode != Pyc::SETUP_EXCEPT_A) { need_try = false; /* Store the current stack for the except/finally statement(s) */ stack_hist.push(stack); PycRef tryblock = new ASTBlock(ASTBlock::BLK_TRY, curblock->end(), true); + tryblock->setEnd(blocks.top()->end()); blocks.push(tryblock); curblock = blocks.top(); } else if (else_pop @@ -521,7 +543,16 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) stack.pop(); } - stack.push(new ASTCall(func, pparamList, kwparamList)); + if (func->type() == ASTNode::NODE_ASSERT){ + if (pparamList.size() > 1){ + fprintf(stderr, "Assert can only have one message\n"); + } else { + func.cast()->setMsg(pparamList.front()); + stack.push(func); + } + } else { + stack.push(new ASTCall(func, pparamList, kwparamList)); + } } break; case Pyc::CALL_FUNCTION_VAR_A: @@ -915,7 +946,7 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) if (mod->verCompare(3, 10) >= 0) end *= sizeof(uint16_t); // // BPO-27129 end += pos; - comprehension = strcmp(code->name()->value(), "") == 0; + comprehension = strcmp(code->name()->value(), "") == 0 || strcmp(code->name()->value(), "") == 0; } else { PycRef top = blocks.top(); end = top->end(); // block end position from SETUP_LOOP @@ -1040,6 +1071,7 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) case Pyc::JUMP_IF_TRUE_A: case Pyc::JUMP_IF_FALSE_OR_POP_A: case Pyc::JUMP_IF_TRUE_OR_POP_A: + case Pyc::JUMP_IF_NOT_EXC_MATCH_A: case Pyc::POP_JUMP_IF_FALSE_A: case Pyc::POP_JUMP_IF_TRUE_A: case Pyc::POP_JUMP_FORWARD_IF_FALSE_A: @@ -1048,9 +1080,18 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) case Pyc::INSTRUMENTED_POP_JUMP_IF_TRUE_A: { PycRef cond = stack.top(); - PycRef ifblk; + PycRef ifblk; int popped = ASTCondBlock::UNINITED; + if (opcode == Pyc::JUMP_IF_NOT_EXC_MATCH_A) + { + stack.pop(); + PycRef comp = new ASTCompare(stack.top(), cond, ASTCompare::CMP_EXCEPTION); + stack.pop(); + cond = comp.cast(); + popped = ASTCondBlock::POPPED; + } + if (opcode == Pyc::POP_JUMP_IF_FALSE_A || opcode == Pyc::POP_JUMP_IF_TRUE_A || opcode == Pyc::POP_JUMP_FORWARD_IF_FALSE_A @@ -1093,15 +1134,49 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) if (cond.type() == ASTNode::NODE_COMPARE && cond.cast()->op() == ASTCompare::CMP_EXCEPTION) { - if (curblock->blktype() == ASTBlock::BLK_EXCEPT - && curblock.cast()->cond() == NULL) { + if ((curblock->blktype() == ASTBlock::BLK_EXCEPT && curblock.cast()->expr() == NULL) + || (curblock->blktype() == ASTBlock::BLK_FINALLY && curblock->size() == 0)){ blocks.pop(); curblock = blocks.top(); - stack_hist.pop(); + StackPopIfNotEmpty(stack_hist); + if (mod->verCompare(3,9) >= 0){ + StackPopIfNotEmpty(stack_hist); + } + } + if (mod->verCompare(3, 9) >= 0){ + blocks.pop(); + if (!blocks.empty() && blocks.top()->blktype() != ASTBlock::BLK_MAIN && blocks.top()->end() < pos) + { + stack = StackHistPopTop(stack, stack_hist); + + PycRef tmp = curblock; + curblock = blocks.top(); + + if (tmp->blktype() != ASTBlock::BLK_ELSE && tmp->nodes().size() > 0){ + curblock->append(tmp.cast()); + } + stack = StackHistPopTop(stack, stack_hist); + + tmp = curblock; + blocks.pop(); + curblock = blocks.top(); + + if (!(tmp->blktype() == ASTBlock::BLK_ELSE && tmp->nodes().size() == 0)){ + curblock->append(tmp.cast()); + } + } else { + blocks.push(curblock); + } + if (curblock->blktype() == ASTBlock::BLK_CONTAINER) { + curblock.cast()->setExcept(pos); + curblock.cast()->setFinally(0); + curblock.cast()->setEnd(offs); + } + stack_hist.push(stack); } - ifblk = new ASTCondBlock(ASTBlock::BLK_EXCEPT, offs, cond.cast()->right(), false); + ifblk = new ASTExceptBlock(offs, cond.cast()->right()); } else if (curblock->blktype() == ASTBlock::BLK_ELSE && curblock->size() == 0) { /* Collapse into elif statement */ @@ -1224,7 +1299,7 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) if (curblock->blktype() == ASTBlock::BLK_CONTAINER) { PycRef cont = curblock.cast(); if (cont->hasExcept() && pos < cont->except()) { - PycRef except = new ASTCondBlock(ASTBlock::BLK_EXCEPT, 0, NULL, false); + PycRef except = new ASTExceptBlock(0); except->init(); blocks.push(except); curblock = blocks.top(); @@ -1264,7 +1339,7 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) if (push) { stack_hist.push(stack); } - PycRef next = new ASTCondBlock(ASTBlock::BLK_EXCEPT, blocks.top()->end(), NULL, false); + PycRef next = new ASTExceptBlock(blocks.top()->end()); next->init(); blocks.push(next.cast()); @@ -1293,13 +1368,24 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) if (mod->verCompare(3, 10) >= 0) offs *= sizeof(uint16_t); // // BPO-27129 + if (mod->verCompare(3, 9) >= 0){ + if (curblock->blktype() == ASTBlock::BLK_FINALLY){ + blocks.pop(); + curblock = blocks.top(); + StackPopIfNotEmpty(stack_hist); + } + if (curblock->blktype() == ASTBlock::BLK_EXCEPT || curblock->blktype() == ASTBlock::BLK_ELSE){ + break; + } + } + if (curblock->blktype() == ASTBlock::BLK_CONTAINER) { PycRef cont = curblock.cast(); - if (cont->hasExcept()) { + if (cont->hasExcept() && cont->except() >= pos) { stack_hist.push(stack); curblock->setEnd(pos+offs); - PycRef except = new ASTCondBlock(ASTBlock::BLK_EXCEPT, pos+offs, NULL, false); + PycRef except = new ASTExceptBlock(pos + offs); except->init(); blocks.push(except); curblock = blocks.top(); @@ -1349,7 +1435,7 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) if (push) { stack_hist.push(stack); } - PycRef next = new ASTCondBlock(ASTBlock::BLK_EXCEPT, pos+offs, NULL, false); + PycRef next = new ASTExceptBlock(pos + offs); next->init(); blocks.push(next.cast()); @@ -1370,8 +1456,7 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) } else if (prev->blktype() == ASTBlock::BLK_TRY && prev->end() < pos+offs) { /* Need to add an except/finally block */ - stack = stack_hist.top(); - stack.pop(); + stack = StackHistPopTop(stack, stack_hist); if (blocks.top()->blktype() == ASTBlock::BLK_CONTAINER) { PycRef cont = blocks.top().cast(); @@ -1380,7 +1465,7 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) stack_hist.push(stack); } - PycRef except = new ASTCondBlock(ASTBlock::BLK_EXCEPT, pos+offs, NULL, false); + PycRef except = new ASTExceptBlock(pos + offs); except->init(); blocks.push(except); } @@ -1410,15 +1495,42 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) PycRef list = stack.top(); - if (curblock->blktype() == ASTBlock::BLK_FOR + if ((curblock->blktype() == ASTBlock::BLK_FOR) && curblock.cast()->isComprehension()) { stack.pop(); - stack.push(new ASTComprehension(value)); + stack.push(new ASTListComprehension(value)); } else { stack.push(new ASTSubscr(list, value)); /* Total hack */ } } break; + + case Pyc::MAP_ADD_A: + { + PycRef value; + PycRef key; + if (mod->verCompare(3, 8) >= 0) { + value = stack.top(); + stack.pop(); + key = stack.top(); + stack.pop(); + } else { + key = stack.top(); + stack.pop(); + value = stack.top(); + stack.pop(); + } + if (curblock->blktype() == ASTBlock::BLK_FOR + && curblock.cast()->isComprehension()){ + stack.pop(); + stack.push(new ASTDictComprehension(key, value)); + } else { + PycRef map = stack.top().cast(); + map->add(key, value); + } + + } + break; case Pyc::SET_UPDATE_A: { PycRef rhs = stack.top(); @@ -1448,29 +1560,134 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) break; case Pyc::LIST_EXTEND_A: { + if (operand != 1) { + fprintf(stderr, "LIST_EXTEND operand list is not at the top of the stack\n"); + break; + } + PycRef rhs = stack.top(); stack.pop(); PycRef lhs = stack.top().cast(); stack.pop(); - if (rhs.type() != ASTNode::NODE_OBJECT) { - fprintf(stderr, "Unsupported argument found for LIST_EXTEND\n"); + switch(rhs.type()){ + case ASTNode::NODE_OBJECT: { + + // I've only ever seen this be a SMALL_TUPLE, but let's be careful... + PycRef obj = rhs.cast()->object(); + if (obj->type() != PycObject::TYPE_TUPLE && obj->type() != PycObject::TYPE_SMALL_TUPLE) { + fprintf(stderr, "Unsupported argument type found for LIST_EXTEND\n"); + break; + } + + ASTList::value_t result = lhs->values(); + for (const auto& it : obj.cast()->values()) { + result.push_back(new ASTObject(it)); + } + + stack.push(new ASTList(result)); + } + break; + case ASTNode::NODE_NAME: + case ASTNode::NODE_CALL: + case ASTNode::NODE_BINARY: + case ASTNode::NODE_SUBSCR: { + ASTList::value_t result = lhs->values(); + + // rhs is a variable, so to extend the list + // we need to unpack rhs + PycRef unpacked_ref = rhs; + unpacked_ref.setUnpacked(true); + + result.push_back(unpacked_ref); + stack.push(new ASTList(result)); + } + break; + default: + fprintf(stderr, "Unsupported argument %i found for LIST_EXTEND\n", rhs.type()); break; } - - // I've only ever seen this be a SMALL_TUPLE, but let's be careful... - PycRef obj = rhs.cast()->object(); - if (obj->type() != PycObject::TYPE_TUPLE && obj->type() != PycObject::TYPE_SMALL_TUPLE) { - fprintf(stderr, "Unsupported argument type found for LIST_EXTEND\n"); + } + break; + case Pyc::LIST_TO_TUPLE: + { + if (stack.top().type() != ASTNode::NODE_LIST){ + fprintf(stderr, "Unexpected argument type %i\n", stack.top().type()); break; } - ASTList::value_t result = lhs->values(); - for (const auto& it : obj.cast()->values()) { - result.push_back(new ASTObject(it)); + PycRef list = stack.top().cast(); + stack.pop(); + ASTTuple::value_t values; + for (PycRef val : list->values()) + { + values.push_back(val); + } + stack.push(new ASTTuple(values)); + } + break; + case Pyc::DICT_UPDATE_A: + case Pyc::DICT_MERGE_A: + { + PycRef rhs = stack.top(); + stack.pop(); + PycRef map = stack.top().cast(); + + switch (rhs.type()){ + case ASTNode::NODE_MAP: + case ASTNode::NODE_NAME: + case ASTNode::NODE_CALL: + case ASTNode::NODE_SUBSCR: + case ASTNode::NODE_BINARY: + map->add_unpacked_value(rhs); + break; + case ASTNode::NODE_CONST_MAP:{ + PycRef const_map = rhs.cast(); + PycTuple::value_t keys = const_map->keys().cast()->object().cast()->values(); + ASTConstMap::values_t values = const_map->values(); + + for (const auto &key : keys) + { + // Values are pushed onto the stack in reverse order. + PycRef value = values.back(); + values.pop_back(); + + map->add(new ASTObject(key), value); + } + } + break; + default: + fprintf(stderr, "Unsupported argument %i found for DICT_MERGE\n", rhs.type()); + break; } + } + break; + case Pyc::LOAD_ASSERTION_ERROR: + { + PycRef assertion = new ASTAssert(); + switch (curblock->blktype()){ + case ASTBlock::BLK_IF: + case ASTBlock::BLK_ELIF:{ + if (stack_hist.size()) + { + stack = stack_hist.top(); + stack_hist.pop(); + } - stack.push(new ASTList(result)); + PycRef prev = curblock.cast(); + blocks.pop(); + curblock = blocks.top(); + assertion.cast()->setCond(prev->cond()); + } + break; + case ASTBlock::BLK_MAIN: + assertion.cast()->setCond(new ASTObject(new PycObject(PycObject::TYPE_FALSE))); + break; + default: + fprintf(stderr, "Unsupported block type %i found for LOAD_ASSERTION_ERROR\n", curblock->blktype()); + break; + } + stack.push(assertion); } break; case Pyc::LOAD_ATTR_A: @@ -1520,6 +1737,7 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) stack.push(new ASTName(code->getCellVar(mod, operand))); break; case Pyc::LOAD_FAST_A: + case Pyc::LOAD_FAST_CHECK_A: if (mod->verCompare(1, 3) < 0) stack.push(new ASTName(code->getName(operand))); else @@ -1589,15 +1807,17 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) break; case Pyc::NOP: break; + case Pyc::POP_EXCEPT: + case Pyc::RERAISE_A: case Pyc::POP_BLOCK: { - if (curblock->blktype() == ASTBlock::BLK_CONTAINER || - curblock->blktype() == ASTBlock::BLK_FINALLY) { - /* These should only be popped by an END_FINALLY */ - break; + if (mod->verCompare(3, 9) < 0) { + if (opcode == Pyc::POP_EXCEPT) { + break; + } } - if (curblock->blktype() == ASTBlock::BLK_WITH) { + if (curblock->blktype() == ASTBlock::BLK_WITH and mod->verCompare(3,9) <0) { // This should only be popped by a WITH_CLEANUP break; } @@ -1626,8 +1846,8 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) if (!blocks.empty()) curblock = blocks.top(); - if (!(tmp->blktype() == ASTBlock::BLK_ELSE - && tmp->nodes().size() == 0)) { + if (!((tmp->blktype() == ASTBlock::BLK_ELSE || tmp->blktype() == ASTBlock::BLK_FINALLY) + && tmp->nodes().size() == 0)){ curblock->append(tmp.cast()); } @@ -1659,15 +1879,17 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) if (curblock->blktype() == ASTBlock::BLK_CONTAINER) { PycRef cont = curblock.cast(); - if (tmp->blktype() == ASTBlock::BLK_ELSE && !cont->hasFinally()) { + if ((tmp->blktype() == ASTBlock::BLK_ELSE && !cont->hasFinally() && !cont->hasExcept()) + || tmp->blktype() == ASTBlock::BLK_FINALLY) { /* Pop the container */ blocks.pop(); curblock = blocks.top(); curblock->append(cont.cast()); - - } else if ((tmp->blktype() == ASTBlock::BLK_ELSE && cont->hasFinally()) - || (tmp->blktype() == ASTBlock::BLK_TRY && !cont->hasExcept())) { + } else if (opcode != Pyc::RERAISE_A + && (((tmp->blktype() == ASTBlock::BLK_ELSE && cont->hasFinally()) + || (tmp->blktype() == ASTBlock::BLK_TRY && !cont->hasExcept()) + || (tmp->blktype() == ASTBlock::BLK_EXCEPT)))) { /* Add the finally block */ stack_hist.push(stack); @@ -1675,9 +1897,21 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) PycRef final = new ASTBlock(ASTBlock::BLK_FINALLY, 0, true); blocks.push(final); curblock = blocks.top(); + } else if (opcode == Pyc::RERAISE_A){ + stack = StackHistPopTop(stack, stack_hist); + } + + if (mod->verCompare(3,9) >= 0){ + if (tmp->end() > curblock->end()){ + cont->setEnd(tmp->end()); + } + goto_addr = tmp->end(); } } + if (tmp->blktype() == ASTBlock::BLK_CONTAINER && opcode == Pyc::RERAISE_A) + stack = StackHistPopTop(stack, stack_hist); + if ((curblock->blktype() == ASTBlock::BLK_FOR || curblock->blktype() == ASTBlock::BLK_ASYNCFOR) && curblock->end() == pos) { blocks.pop(); @@ -1686,9 +1920,6 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) } } break; - case Pyc::POP_EXCEPT: - /* Do nothing. */ - break; case Pyc::END_FOR: { stack.pop(); @@ -1743,7 +1974,7 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) auto& pparams = value.cast()->pparams(); if (!pparams.empty()) { PycRef res = pparams.front(); - stack.push(new ASTComprehension(res)); + stack.push(new ASTListComprehension(res)); } } } @@ -1809,6 +2040,12 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) break; case Pyc::RAISE_VARARGS_A: { + if (operand == 1 and stack.top()->type() == ASTNode::NODE_ASSERT){ + curblock->append(stack.top()); + stack.pop(); + break; + } + ASTRaise::param_t paramList; for (int i = 0; i < operand; i++) { paramList.push_front(stack.top()); @@ -1817,16 +2054,24 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) curblock->append(new ASTRaise(paramList)); if ((curblock->blktype() == ASTBlock::BLK_IF - || curblock->blktype() == ASTBlock::BLK_ELSE) + || curblock->blktype() == ASTBlock::BLK_ELSE + || curblock->blktype() == ASTBlock::BLK_TRY + || curblock->blktype() == ASTBlock::BLK_EXCEPT) && stack_hist.size() && (mod->verCompare(2, 6) >= 0)) { stack = stack_hist.top(); stack_hist.pop(); PycRef prev = curblock; - blocks.pop(); - curblock = blocks.top(); - curblock->append(prev.cast()); + if (!blocks.empty()){ + blocks.pop(); + curblock = blocks.top(); + curblock->append(prev.cast()); + } + if (prev->end() > curblock->end()){ + curblock->setEnd(prev->end()); + } + goto_addr = prev->end(); } } break; @@ -1951,6 +2196,8 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) { if (curblock->blktype() == ASTBlock::BLK_CONTAINER) { curblock.cast()->setExcept(pos+operand); + if (mod->verCompare(3,9) >=0) + curblock.cast()->setFinally(0); } else { PycRef next = new ASTContainerBlock(0, pos+operand); blocks.push(next.cast()); @@ -1967,7 +2214,15 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) break; case Pyc::SETUP_FINALLY_A: { - PycRef next = new ASTContainerBlock(pos+operand); + int offs = operand; + if (mod->verCompare(3, 10) >= 0) + offs *= sizeof(uint16_t); + if (curblock->blktype() == ASTBlock::BLK_EXCEPT && curblock->size() == 0){ + break; + } + + PycRef next = new ASTContainerBlock(pos + offs); + next->setEnd(pos + offs); blocks.push(next.cast()); curblock = blocks.top(); @@ -2155,6 +2410,11 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) curblock.cast()->setVar(name); } else if (value.type() == ASTNode::NODE_CHAINSTORE) { append_to_chain_store(value, name, stack, curblock); + } else if (curblock->blktype() == ASTBlock::BLK_EXCEPT + && curblock->size() == 0 + && curblock.cast()->var() == NULL + && value->type() == ASTNode::NODE_INVALID) { + curblock.cast()->setVar(name); } else { curblock->append(new ASTStore(value, name)); } @@ -2209,12 +2469,38 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) PycRef name = new ASTName(code->getName(operand)); PycRef tup = stack.top(); - if (tup.type() == ASTNode::NODE_TUPLE) + switch (tup.type()){ + case ASTNode::Type::NODE_TUPLE: tup.cast()->add(name); - else - fputs("Something TERRIBLE happened!\n", stderr); - - if (--unpack <= 0) { + break; + case ASTNode::Type::NODE_UNPACKED_TUPLE: + tup.cast()->add(name); + break; + default: + fprintf(stderr, "Unsupported iterable type %i\n", tup->type()); + break; + } + unpack--; + + while (unpack > 0 + and tup->type() == ASTNode::Type::NODE_UNPACKED_TUPLE + and tup.cast()->isFull() + ) + { + PycRef val = tup; + stack.pop(); + if (stack.top()->type() == ASTNode::Type::NODE_UNPACKED_TUPLE + and not stack.top().cast()->isFull()) + { + stack.top().cast()->add(tup); + tup = stack.top(); + unpack--; + } else { + stack.push(tup); + break; + } + } + if (unpack <= 0){ stack.pop(); PycRef seq = stack.top(); stack.pop(); @@ -2260,9 +2546,16 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) && !curblock->inited()) { curblock.cast()->setExpr(value); curblock.cast()->setVar(name); + } else if (curblock->blktype() == ASTBlock::BLK_EXCEPT + && curblock->size() == 0 + && curblock.cast()->var() == NULL + && value->type() == ASTNode::NODE_INVALID) { + curblock.cast()->setVar(name); } else if (value.type() == ASTNode::NODE_CHAINSTORE) { append_to_chain_store(value, name, stack, curblock); - } else { + } + else + { curblock->append(new ASTStore(value, name)); if (value.type() == ASTNode::NODE_INVALID) @@ -2453,6 +2746,15 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) } } break; + case Pyc::UNPACK_EX_A: + { + uint8_t before = operand & 0xFF; + uint8_t after = (operand >> 8) & 0xFF; + + stack.push(new ASTUnpackedTuple(before, after)); + unpack += (after + before + 1); + } + break; case Pyc::YIELD_FROM: { PycRef dest = stack.top(); @@ -2584,16 +2886,161 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) stack.push(value); } break; + case Pyc::CALL_INTRINSIC_1_A: + { + PycRef arg = stack.top(); + stack.pop(); + + if (operand != ASTCallIntrinsic1::INTRINSIC_LIST_TO_TUPLE) { + fprintf(stderr, "Unimplemented function %i", operand); + break; + } + + if (arg.type() != ASTNode::NODE_LIST) { + fprintf(stderr, "Unexpected argument type %i\n", arg.type()); + break; + } + + PycRef list = arg.cast(); + ASTTuple::value_t values; + for (PycRef val : list->values()) { + values.push_back(val); + } + stack.push(new ASTTuple(values)); + } + break; + case Pyc::CALL_FUNCTION_EX_A: + { + int has_kwmap = operand & 1; + ASTCall::kwparam_t kwparamList; + ASTCall::pparam_t pparamList; + ASTCall *call = new ASTCall(); + + // callable, iterable object & kwmap object (if present) + + if (has_kwmap) { + PycRef kwmap_stack = stack.top(); + switch (kwmap_stack.type()){ + case ASTNode::NODE_KW_NAMES_MAP: + case ASTNode::NODE_MAP:{ + stack.pop(); + PycRef kwmap = kwmap_stack.cast(); + for (ASTMap::map_t::const_iterator it = kwmap->values().begin(); it != kwmap->values().end(); it++) + { + if (kwmap->is_unpacked(*it)) + { + kwparamList.push_back(call->genKwparamUnpacked(it->second)); + } + else + { + kwparamList.push_back(std::make_pair(it->first, it->second)); + } + } + break; + } + case ASTNode::NODE_CONST_MAP:{ + stack.pop(); + PycRef const_map = kwmap_stack.cast(); + PycTuple::value_t keys = const_map->keys().cast()->object().cast()->values(); + ASTConstMap::values_t values = const_map->values(); + + for (const auto &key : keys) + { + // Values are pushed onto the stack in reverse order. + PycRef value = values.back(); + values.pop_back(); + + kwparamList.push_back(std::make_pair(new ASTObject(key), value)); + } + break; + } + case ASTNode::NODE_OBJECT:{ + PycRef obj = kwmap_stack.cast()->object(); + if (obj.type() == PycObject::TYPE_DICT){ + for (const auto &it : obj.cast()->values()) + { + kwparamList.push_back(std::make_pair(new ASTObject(std::get<0>(it)), new ASTObject(std::get<1>(it)))); + } + } else { + fprintf(stderr, "Unsupported node object type %i\n", obj.type()); + } + break; + } + default: + fprintf(stderr, "Unexpected object type %i for kwparams in CALL_FUNCTION_EX\n", kwmap_stack.type()); + break; + } + } + + PycRef iterable = stack.top(); + stack.pop(); + + switch (iterable.type()) { + case ASTNode::NODE_LIST: + for (PycRef n : iterable.cast()->values()) + { + pparamList.push_back(n); + } + break; + case ASTNode::NODE_TUPLE: + for (PycRef n : iterable.cast()->values()) + { + pparamList.push_back(n); + } + break; + case ASTNode::NODE_SET: + for (PycRef n : iterable.cast()->values()) + { + pparamList.push_back(n); + } + break; + case ASTNode::NODE_OBJECT: + switch (iterable.cast()->object().type()) + { + case PycObject::TYPE_LIST: + case PycObject::TYPE_SET: + case PycObject::TYPE_TUPLE: + case PycObject::TYPE_SMALL_TUPLE: + for (const auto &it : iterable.cast()->object().cast()->values()) + { + pparamList.push_back(new ASTObject(it)); + } + break; + default: + fprintf(stderr, "Unsupported node object type %i\n", iterable.cast()->object().type()); + break; + } + break; + case ASTNode::NODE_SUBSCR: + case ASTNode::NODE_BINARY: + case ASTNode::NODE_NAME: + pparamList.push_back(iterable); + break; + default: + fprintf(stderr, "Unsupported param iterable type %i in CALL_FUNC_EX\n", iterable.type()); + break; + } + PycRef func = stack.top(); + stack.pop(); + + call->setFunc(func); + call->setKwparams(kwparamList); + call->setPparams(pparamList); + stack.push(call); + } + break; default: fprintf(stderr, "Unsupported opcode: %s (%d)\n", Pyc::OpcodeName(opcode), opcode); cleanBuild = false; return new ASTNodeList(defblock->nodes()); } - else_pop = ( (curblock->blktype() == ASTBlock::BLK_ELSE) + else_pop = ((curblock->blktype() == ASTBlock::BLK_ELSE) || (curblock->blktype() == ASTBlock::BLK_IF) - || (curblock->blktype() == ASTBlock::BLK_ELIF) ) - && (curblock->end() == pos); + || (curblock->blktype() == ASTBlock::BLK_ELIF) + || (curblock->blktype() == ASTBlock::BLK_FINALLY) + || (curblock->blktype() == ASTBlock::BLK_EXCEPT)) + && (curblock->end() == pos || goto_addr == curblock->end()); } if (stack_hist.size()) { @@ -2782,6 +3229,7 @@ void print_formatted_value(PycRef formatted_value, PycModule* static std::unordered_set node_seen; +// TODO: Handle m_unpack for node correctly here. void print_src(PycRef node, PycModule* mod, std::ostream& pyc_output) { if (node == NULL) { @@ -2796,6 +3244,9 @@ void print_src(PycRef node, PycModule* mod, std::ostream& pyc_output) } node_seen.insert((ASTNode *)node); + if (node.isUnpacked()) + pyc_output << "*"; + switch (node->type()) { case ASTNode::NODE_BINARY: case ASTNode::NODE_COMPARE: @@ -2828,7 +3279,9 @@ void print_src(PycRef node, PycModule* mod, std::ostream& pyc_output) for (const auto& param : call->kwparams()) { if (!first) pyc_output << ", "; - if (param.first.type() == ASTNode::NODE_NAME) { + if (call->isKwparamUnpacked(param)){ + pyc_output << "**"; + } else if (param.first.type() == ASTNode::NODE_NAME) { pyc_output << param.first.cast()->name()->value() << " = "; } else { PycRef str_name = param.first.cast()->object().cast(); @@ -2944,7 +3397,15 @@ void print_src(PycRef node, PycModule* mod, std::ostream& pyc_output) { PycRef comp = node.cast(); - pyc_output << "[ "; + if (comp->comptype() == ASTComprehension::COMP_DICT) { + pyc_output << "{ "; + print_src(comp.cast()->key(), mod, pyc_output); + pyc_output << ": "; + } else if (comp->comptype() == ASTComprehension::COMP_LIST) { + pyc_output << "[ "; + } else { + fprintf(stderr, "Unsupported comprehension type %d in NODE_COMPREHENSION\n", comp->comptype()); + } print_src(comp->result(), mod, pyc_output); for (const auto& gen : comp->generators()) { @@ -2957,7 +3418,14 @@ void print_src(PycRef node, PycModule* mod, std::ostream& pyc_output) print_src(gen->condition(), mod, pyc_output); } } - pyc_output << " ]"; + if (comp->comptype() == ASTComprehension::COMP_DICT) + { + pyc_output << "} "; + } else if (comp->comptype() == ASTComprehension::COMP_LIST){ + pyc_output << "] "; + } else { + fprintf(stderr, "Unsupported comprehension type %d in NODE_COMPREHENSION\n", comp->comptype()); + } } break; case ASTNode::NODE_MAP: @@ -2965,14 +3433,19 @@ void print_src(PycRef node, PycModule* mod, std::ostream& pyc_output) pyc_output << "{"; bool first = true; cur_indent++; - for (const auto& val : node.cast()->values()) { + PycRef map = node.cast(); + for (const auto& val : map->values()) { if (first) pyc_output << "\n"; else pyc_output << ",\n"; start_line(cur_indent, pyc_output); - print_src(val.first, mod, pyc_output); - pyc_output << ": "; + if (map->is_unpacked(val)){ + pyc_output << "**"; + } else { + print_src(val.first, mod, pyc_output); + pyc_output << ": "; + } print_src(val.second, mod, pyc_output); first = false; } @@ -3043,9 +3516,15 @@ void print_src(PycRef node, PycModule* mod, std::ostream& pyc_output) pyc_output << " in "; print_src(blk.cast()->iter(), mod, pyc_output); } else if (blk->blktype() == ASTBlock::BLK_EXCEPT && - blk.cast()->cond() != NULL) { + blk.cast()->expr() != NULL) { pyc_output << " "; - print_src(blk.cast()->cond(), mod, pyc_output); + print_src(blk.cast()->expr(), mod, pyc_output); + PycRef var = blk.try_cast()->var(); + if (var != NULL) + { + pyc_output << " as "; + print_src(var, mod, pyc_output); + } } else if (blk->blktype() == ASTBlock::BLK_WITH) { pyc_output << " "; print_src(blk.cast()->expr(), mod, pyc_output); @@ -3106,7 +3585,21 @@ void print_src(PycRef node, PycModule* mod, std::ostream& pyc_output) } } break; - case ASTNode::NODE_RETURN: + case ASTNode::NODE_ASSERT: + { + PycRef assertion = node.cast(); + pyc_output << "assert "; + if (assertion->cond()){ + print_src(assertion->cond(), mod, pyc_output); + if (assertion->msg()){ + pyc_output << ", "; + print_src(assertion->msg(), mod, pyc_output); + } + } + + } + break; + case ASTNode::NODE_RETURN: { PycRef ret = node.cast(); PycRef value = ret->value(); @@ -3395,10 +3888,11 @@ void print_src(PycRef node, PycModule* mod, std::ostream& pyc_output) } break; case ASTNode::NODE_TUPLE: + case ASTNode::NODE_UNPACKED_TUPLE: { PycRef tuple = node.cast(); ASTTuple::value_t values = tuple->values(); - if (tuple->requireParens()) + if (tuple->requireParens() or tuple.isUnpacked()) pyc_output << "("; bool first = true; for (const auto& val : values) { @@ -3409,7 +3903,7 @@ void print_src(PycRef node, PycModule* mod, std::ostream& pyc_output) } if (values.size() == 1) pyc_output << ','; - if (tuple->requireParens()) + if (tuple->requireParens() or tuple.isUnpacked()) pyc_output << ')'; } break; diff --git a/FastStack.h b/FastStack.h index b91ec71de..624cf0a98 100644 --- a/FastStack.h +++ b/FastStack.h @@ -61,6 +61,25 @@ class FastStack { return m_ptr == -1; } + void debug_print(PycModule* mod, std::ostream& pyc_output) + { + pyc_output << "---- STACK CONTENTS ----\n"; + if (empty()) { + pyc_output << "empty stack\n"; + } + else { + for (int i = m_ptr; i >= 0; i--) { + print_src(m_stack[i], mod, pyc_output); + if (i == m_ptr) { + pyc_output << " <- STACK TOP"; + } + pyc_output << "\n"; + } + } + pyc_output << "------------------------\n"; + } + + private: std::vector> m_stack; int m_ptr; diff --git a/pyc_object.h b/pyc_object.h index 085944496..56baad842 100644 --- a/pyc_object.h +++ b/pyc_object.h @@ -6,21 +6,21 @@ template class PycRef { public: - PycRef() noexcept : m_obj() { } + PycRef() noexcept : m_obj(), m_unpack(false) { } - PycRef(_Obj* obj) noexcept : m_obj(obj) + PycRef(_Obj* obj) noexcept : m_obj(obj), m_unpack(false) { if (m_obj) m_obj->addRef(); } - PycRef(const PycRef<_Obj>& obj) noexcept : m_obj(obj.m_obj) + PycRef(const PycRef<_Obj>& obj) noexcept : m_obj(obj.m_obj), m_unpack(obj.m_unpack) { if (m_obj) m_obj->addRef(); } - PycRef(PycRef<_Obj>&& obj) noexcept : m_obj(obj.m_obj) + PycRef(PycRef<_Obj>&& obj) noexcept : m_obj(obj.m_obj), m_unpack(obj.m_unpack) { obj.m_obj = nullptr; } @@ -38,6 +38,7 @@ class PycRef { if (m_obj) m_obj->delRef(); m_obj = obj; + m_unpack = false; return *this; } @@ -48,16 +49,20 @@ class PycRef { if (m_obj) m_obj->delRef(); m_obj = obj.m_obj; + m_unpack = obj.m_unpack; return *this; } PycRef<_Obj>& operator=(PycRef<_Obj>&& obj) noexcept { m_obj = obj.m_obj; + m_unpack = obj.m_unpack; obj.m_obj = nullptr; + obj.m_unpack = false; return *this; } + // TODO: Handle m_unpack for remaining operators bool operator==(_Obj* obj) const { return m_obj == obj; } bool operator==(const PycRef<_Obj>& obj) const { return m_obj == obj.m_obj; } bool operator!=(_Obj* obj) const { return m_obj != obj; } @@ -75,16 +80,28 @@ class PycRef { template PycRef<_Cast> cast() const { - _Cast* result = dynamic_cast<_Cast*>(m_obj); - if (!result) + _Cast* casted_obj = dynamic_cast<_Cast*>(m_obj); + if (!casted_obj) throw std::bad_cast(); + + PycRef<_Cast> result = casted_obj; + result.setUnpacked(m_unpack); + return result; } bool isIdent(const _Obj* obj) const { return m_obj == obj; } + bool isUnpacked() const { return m_unpack; } + void setUnpacked(bool unpack) { m_unpack = unpack; } + private: _Obj* m_obj; + + // References to an object can be either packed or unpacked. + // Usually unpacked references will be used with variables + // or lists but they may arise in other places as well. + bool m_unpack; }; diff --git a/tests/compiled/call_func_ex.3.10.pyc b/tests/compiled/call_func_ex.3.10.pyc new file mode 100644 index 000000000..8e32007e1 Binary files /dev/null and b/tests/compiled/call_func_ex.3.10.pyc differ diff --git a/tests/compiled/dict_merge.3.10.pyc b/tests/compiled/dict_merge.3.10.pyc new file mode 100644 index 000000000..dc96a773f Binary files /dev/null and b/tests/compiled/dict_merge.3.10.pyc differ diff --git a/tests/compiled/jump_if_not_exc_match.3.10.pyc b/tests/compiled/jump_if_not_exc_match.3.10.pyc new file mode 100644 index 000000000..9fa00723b Binary files /dev/null and b/tests/compiled/jump_if_not_exc_match.3.10.pyc differ diff --git a/tests/compiled/list_extend.3.9.pyc b/tests/compiled/list_extend_1.3.9.pyc similarity index 100% rename from tests/compiled/list_extend.3.9.pyc rename to tests/compiled/list_extend_1.3.9.pyc diff --git a/tests/compiled/list_extend_2.3.12.pyc b/tests/compiled/list_extend_2.3.12.pyc new file mode 100644 index 000000000..25c333a8e Binary files /dev/null and b/tests/compiled/list_extend_2.3.12.pyc differ diff --git a/tests/compiled/list_to_tuple.3.10.pyc b/tests/compiled/list_to_tuple.3.10.pyc new file mode 100644 index 000000000..8e3063223 Binary files /dev/null and b/tests/compiled/list_to_tuple.3.10.pyc differ diff --git a/tests/compiled/load_assert.3.10.pyc b/tests/compiled/load_assert.3.10.pyc new file mode 100644 index 000000000..36623d861 Binary files /dev/null and b/tests/compiled/load_assert.3.10.pyc differ diff --git a/tests/compiled/map_add.3.10.pyc b/tests/compiled/map_add.3.10.pyc new file mode 100644 index 000000000..ab8e63e7d Binary files /dev/null and b/tests/compiled/map_add.3.10.pyc differ diff --git a/tests/compiled/map_add.3.7.pyc b/tests/compiled/map_add.3.7.pyc new file mode 100644 index 000000000..67d8cd85f Binary files /dev/null and b/tests/compiled/map_add.3.7.pyc differ diff --git a/tests/compiled/reraise.3.10.pyc b/tests/compiled/reraise.3.10.pyc new file mode 100644 index 000000000..a056a7704 Binary files /dev/null and b/tests/compiled/reraise.3.10.pyc differ diff --git a/tests/compiled/test_unpack.3.12.pyc b/tests/compiled/test_unpack.3.12.pyc new file mode 100644 index 000000000..5eb9cecc5 Binary files /dev/null and b/tests/compiled/test_unpack.3.12.pyc differ diff --git a/tests/compiled/unpack_ex.3.10.pyc b/tests/compiled/unpack_ex.3.10.pyc new file mode 100644 index 000000000..b2e20f1b8 Binary files /dev/null and b/tests/compiled/unpack_ex.3.10.pyc differ diff --git a/tests/input/call_func_ex.py b/tests/input/call_func_ex.py new file mode 100644 index 000000000..f76e9ca13 --- /dev/null +++ b/tests/input/call_func_ex.py @@ -0,0 +1,22 @@ +# tests from CPython (lib/test/dtracedata/call_stack.py) +def function_5(dummy, dummy2, **dummy3): + if False: + return 7 + return 8 + + +def start(): + function_5(*(1, 2), **{"test": 42}) + + +start() + +# other tests to try intricated params +a = {"a": 1} +b = (1, 2) +c = {"c": 0} + +def f(arg1, arg2, arg3, **kwargs): + return 1 + +f(*a, *b, kwarg=0, **c) # type: ignore diff --git a/tests/input/dict_merge.py b/tests/input/dict_merge.py new file mode 100644 index 000000000..79a1479b5 --- /dev/null +++ b/tests/input/dict_merge.py @@ -0,0 +1,5 @@ +b = {"b": 1} +# unpack of dict +dict(**{"c": 2}, **b) +# add arguments in plus of unpack +dict(a="a", **{"c": 2}, **b) diff --git a/tests/input/jump_if_not_exc_match.py b/tests/input/jump_if_not_exc_match.py new file mode 100644 index 000000000..0d8cf826d --- /dev/null +++ b/tests/input/jump_if_not_exc_match.py @@ -0,0 +1,15 @@ +try: + a = 1/0 +except ZeroDivisionError: + a = 1 + + +try: + a = 2 / 0 +except (ZeroDivisionError, AssertionError) as v: + print(v) +except Exception: + print("there was an exception") +finally: + b = 0 + a = 2 \ No newline at end of file diff --git a/tests/input/list_extend.py b/tests/input/list_extend_1.py similarity index 100% rename from tests/input/list_extend.py rename to tests/input/list_extend_1.py diff --git a/tests/input/list_extend_2.py b/tests/input/list_extend_2.py new file mode 100644 index 000000000..1473f17de --- /dev/null +++ b/tests/input/list_extend_2.py @@ -0,0 +1,2 @@ +def get(l): + return [*l] diff --git a/tests/input/list_to_tuple.py b/tests/input/list_to_tuple.py new file mode 100644 index 000000000..f37982813 --- /dev/null +++ b/tests/input/list_to_tuple.py @@ -0,0 +1,2 @@ +my_list = [1, 2, 3] +my_tuple = (*my_list,) \ No newline at end of file diff --git a/tests/input/load_assert.py b/tests/input/load_assert.py new file mode 100644 index 000000000..d9cb53f85 --- /dev/null +++ b/tests/input/load_assert.py @@ -0,0 +1,5 @@ +# Some tests come from CPython tests +assert a +assert 1 == 1, "toto" +assert a > 0 and bb > 0 and ccc == 1000000, "error msg" +assert False \ No newline at end of file diff --git a/tests/input/map_add.py b/tests/input/map_add.py new file mode 100644 index 000000000..479068efb --- /dev/null +++ b/tests/input/map_add.py @@ -0,0 +1,28 @@ +# the compiler optimizes the dict creation using a MAP_ADD +d = { + 1: 1, + 2: 1, + 3: 1, + 4: 1, + 5: 1, + 6: 1, + 7: 1, + 8: 1, + 9: 1, + 10: 1, + 11: 1, + 12: 1, + 13: 1, + 14: 1, + 15: 1, + 16: 1, + 17: 1, +} + + +# the following dict comes from cpython tests (lib/test/test_compile.py) +# multiline_dict_comprehension +{x: 2 * x for x in [1, 2, 3] if (x > 0)} + +# complete example with multiple conditions does not work, related to POP_JUMP_IF_FALSE +# if (x > 0 and x < 100 and x != 50) diff --git a/tests/input/reraise.py b/tests/input/reraise.py new file mode 100644 index 000000000..26e668420 --- /dev/null +++ b/tests/input/reraise.py @@ -0,0 +1,17 @@ +# This tests come from CPython test suite, some of them are a little bit modified. +def intricate_try_except(): + try: + try: + a = 1 + except AssertionError as a: + print(a) + except TypeError: + raise + except Exception as e: + raise e + +def reraise_example(): + try: + 1/0 + except ZeroDivisionError: + raise \ No newline at end of file diff --git a/tests/input/test_unpack.py b/tests/input/test_unpack.py new file mode 100644 index 000000000..fdf2ebf9e --- /dev/null +++ b/tests/input/test_unpack.py @@ -0,0 +1,6 @@ +import struct + +def wtob(w): + return struct.pack('<'+'I'*len(w), *w) + +wtob([12,3]) diff --git a/tests/input/unpack_ex.py b/tests/input/unpack_ex.py new file mode 100644 index 000000000..e87860301 --- /dev/null +++ b/tests/input/unpack_ex.py @@ -0,0 +1,39 @@ +## Tests from CPython tests (Lib/test/test_unpack_ex.py) +# Unpack tuple +t = (1, 2, 3) +a, *b, c = (1, 2, 3) + +# Unpack list +l = [4, 5, 6] +a, *b = l +a, *b, c = [4,5,6] + +#Unpack implied tuple +*a, = 7, 8, 9 + +# Unpack nested implied tuple +[*[*a],b] = [[7, 8, 9]] # note for tests, another notation possible is: *(*a,),b = [[7, 8, 9]] +[*[*a]] = [[7, 8, 9]] # note for tests, another notation possible is: *(*a,), = [[7, 8, 9]] + +# Unpack string... fun! +a, *b = "one" + +# Unpack long sequence +a, b, c, *d, e, f, g = range(10) + +# Unpack short sequence +a, *b, c = (1, 2) + +# Unpack in for statement +for a, *b, c in [(1,2,3), (4,5,6,7)]: + print(a, b, c) + +# Unpack in list +[a, *b, c] = range(5) + +# Multiple targets +a, *b, c = *d, e = range(5) + +# Assignment unpacking +a, b, *c = range(5) +*a, b, c = a, b, *c \ No newline at end of file diff --git a/tests/tokenized/call_func_ex.txt b/tests/tokenized/call_func_ex.txt new file mode 100644 index 000000000..ad58b6d48 --- /dev/null +++ b/tests/tokenized/call_func_ex.txt @@ -0,0 +1,17 @@ +def function_5 ( dummy , dummy2 , ** dummy3 ) : + +return 8 + +def start ( ) : + +function_5 ( 1 , 2 , ** { 'test' : 42 } ) + +start ( ) +a = { 'a' : 1 } +b = ( 1 , 2 ) +c = { 'c' : 0 } +def f ( arg1 , arg2 , arg3 , ** kwargs ) : + +return 1 + +f ( * a , * b , kwarg = 0 , ** c ) diff --git a/tests/tokenized/dict_merge.txt b/tests/tokenized/dict_merge.txt new file mode 100644 index 000000000..93478713f --- /dev/null +++ b/tests/tokenized/dict_merge.txt @@ -0,0 +1,3 @@ +b = { 'b' : 1 } +dict ( ** { 'c' : 2 } , ** b ) +dict ( a = 'a' , ** { 'c' : 2 } , ** b ) diff --git a/tests/tokenized/jump_if_not_exc_match.txt b/tests/tokenized/jump_if_not_exc_match.txt new file mode 100644 index 000000000..e657b9ead --- /dev/null +++ b/tests/tokenized/jump_if_not_exc_match.txt @@ -0,0 +1,27 @@ +try : + +a = 1 / 0 + +except ZeroDivisionError : + +a = 1 + +try : + +try : + +a = 2 / 0 + +except ( ZeroDivisionError , AssertionError ) as v : + +print ( v ) + +except Exception : + +print ( 'there was an exception' ) + + +finally : + +b = 0 +a = 2 diff --git a/tests/tokenized/list_extend.txt b/tests/tokenized/list_extend_1.txt similarity index 100% rename from tests/tokenized/list_extend.txt rename to tests/tokenized/list_extend_1.txt diff --git a/tests/tokenized/list_extend_2.txt b/tests/tokenized/list_extend_2.txt new file mode 100644 index 000000000..31a437e0c --- /dev/null +++ b/tests/tokenized/list_extend_2.txt @@ -0,0 +1,3 @@ +def get ( l ) : + +return [ * l ] diff --git a/tests/tokenized/list_to_tuple.txt b/tests/tokenized/list_to_tuple.txt new file mode 100644 index 000000000..e42586e4c --- /dev/null +++ b/tests/tokenized/list_to_tuple.txt @@ -0,0 +1,2 @@ +my_list = [ 1 , 2 , 3 ] +my_tuple = ( * my_list , ) diff --git a/tests/tokenized/load_assert.txt b/tests/tokenized/load_assert.txt new file mode 100644 index 000000000..df3d8366b --- /dev/null +++ b/tests/tokenized/load_assert.txt @@ -0,0 +1,4 @@ +assert a +assert 1 == 1 , 'toto' +assert a > 0 and bb > 0 or ccc == 1000000 , 'error msg' +assert False diff --git a/tests/tokenized/reraise.txt b/tests/tokenized/reraise.txt new file mode 100644 index 000000000..cd513ea74 --- /dev/null +++ b/tests/tokenized/reraise.txt @@ -0,0 +1,31 @@ +def intricate_try_except ( ) : + +try : + +try : + +a = 1 + +except AssertionError as a : + +print ( a ) + +except TypeError : + +raise + + +except Exception as e : + +raise e + + +def reraise_example ( ) : + +try : + +1 / 0 + +except ZeroDivisionError : + +raise diff --git a/tests/tokenized/test_unpack.txt b/tests/tokenized/test_unpack.txt new file mode 100644 index 000000000..16c7af1f4 --- /dev/null +++ b/tests/tokenized/test_unpack.txt @@ -0,0 +1,6 @@ +import struct +def wtob ( w ) : + +return struct . pack ( '<' + 'I' * len ( w ) , * w ) + +wtob ( [ 12 , 3 ] ) diff --git a/tests/tokenized/unpack_ex.txt b/tests/tokenized/unpack_ex.txt new file mode 100644 index 000000000..01439b3e1 --- /dev/null +++ b/tests/tokenized/unpack_ex.txt @@ -0,0 +1,19 @@ +t = ( 1 , 2 , 3 ) +a , * b , c = ( 1 , 2 , 3 ) +l = [ 4 , 5 , 6 ] +a , * b = l +a , * b , c = [ 4 , 5 , 6 ] +* a , = ( 7 , 8 , 9 ) +* ( * a , ) , b = [ [ 7 , 8 , 9 ] ] +* ( * a , ) , = [ [ 7 , 8 , 9 ] ] +a , * b = 'one' +a , b , c , * d , e , f , g = range ( 10 ) +a , * b , c = ( 1 , 2 ) +for a , * b , c in ( ( 1 , 2 , 3 ) , ( 4 , 5 , 6 , 7 ) ) : + +print ( a , b , c ) + +a , * b , c = range ( 5 ) +a , * b , c = * d , e = range ( 5 ) +a , b , * c = range ( 5 ) +* a , b , c = ( a , b , * c )