diff --git a/ASTNode.h b/ASTNode.h index 98760dbf5..0f7ebc28b 100644 --- a/ASTNode.h +++ b/ASTNode.h @@ -18,7 +18,7 @@ class ASTNode { NODE_COMPREHENSION, NODE_LOADBUILDCLASS, NODE_AWAITABLE, NODE_FORMATTEDVALUE, NODE_JOINEDSTR, NODE_CONST_MAP, NODE_ANNOTATED_VAR, NODE_CHAINSTORE, NODE_TERNARY, - NODE_KW_NAMES_MAP, + NODE_KW_NAMES_MAP, NODE_CALL_INTRINSIC_1, NODE_CALL_INTRINSIC_2, // Empty node types NODE_LOCALS, @@ -757,4 +757,26 @@ class ASTTernary : public ASTNode PycRef m_else_expr; }; +class ASTCallIntrinsic1: public ASTNode +{ +public: + enum Function { + INTRINSIC_1_INVALID, INTRINSIC_PRINT, INTRINSIC_IMPORT_STAR, + INTRINSIC_STOPITERATION_ERROR, INTRINSIC_ASYNC_GEN_WRAP, + INTRINSIC_UNARY_POSITIVE, INTRINSIC_LIST_TO_TUPLE, INTRINSIC_TYPEVAR, + INTRINSIC_PARAMSPEC, INTRINSIC_TYPEVARTUPLE, + INTRINSIC_SUBSCRIPT_GENERIC, INTRINSIC_TYPEALIAS, + }; +}; + +class ASTCallIntrinsic2: public ASTNode +{ +public: + enum Function { + INTRINSIC_2_INVALID, INTRINSIC_PREP_RERAISE_STAR, + INTRINSIC_TYPEVAR_WITH_BOUND, INTRINSIC_TYPEVAR_WITH_CONSTRAINTS, + INTRINSIC_SET_FUNCTION_TYPE_PARAMS, INTRINSIC_SET_TYPEPARAM_DEFAULT, + }; +}; + #endif diff --git a/ASTree.cpp b/ASTree.cpp index 6635808e1..08d352c4d 100644 --- a/ASTree.cpp +++ b/ASTree.cpp @@ -6,6 +6,7 @@ #include "FastStack.h" #include "pyc_numeric.h" #include "bytecode.h" +#include // This must be a triple quote (''' or """), to handle interpolated string literals containing the opposite quote style. // E.g. f'''{"interpolated "123' literal"}''' -> valid. @@ -1448,29 +1449,46 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) break; case Pyc::LIST_EXTEND_A: { + if (operand != 1) { + fprintf(stderr, "LIST_EXTEND operand list is not at the top of the stack\n"); + break; + } + PycRef rhs = stack.top(); stack.pop(); PycRef lhs = stack.top().cast(); stack.pop(); - if (rhs.type() != ASTNode::NODE_OBJECT) { - fprintf(stderr, "Unsupported argument found for LIST_EXTEND\n"); - break; - } + if (rhs.type() == ASTNode::NODE_OBJECT) { - // I've only ever seen this be a SMALL_TUPLE, but let's be careful... - PycRef obj = rhs.cast()->object(); - if (obj->type() != PycObject::TYPE_TUPLE && obj->type() != PycObject::TYPE_SMALL_TUPLE) { - fprintf(stderr, "Unsupported argument type found for LIST_EXTEND\n"); - break; - } + // I've only ever seen this be a SMALL_TUPLE, but let's be careful... + PycRef obj = rhs.cast()->object(); + if (obj->type() != PycObject::TYPE_TUPLE && obj->type() != PycObject::TYPE_SMALL_TUPLE) { + fprintf(stderr, "Unsupported argument type found for LIST_EXTEND\n"); + break; + } - ASTList::value_t result = lhs->values(); - for (const auto& it : obj.cast()->values()) { - result.push_back(new ASTObject(it)); + ASTList::value_t result = lhs->values(); + for (const auto& it : obj.cast()->values()) { + result.push_back(new ASTObject(it)); + } + + stack.push(new ASTList(result)); } + else if (rhs.type() == ASTNode::NODE_NAME) { + ASTList::value_t result = lhs->values(); + + // rhs is a variable, so to extend the list + // we need to unpack rhs + PycRef unpacked_ref = rhs; + unpacked_ref.setUnpacked(true); - stack.push(new ASTList(result)); + result.push_back(unpacked_ref); + stack.push(new ASTList(result)); + } + else { + fprintf(stderr, "Unsupported argument %i found for LIST_EXTEND\n", rhs.type()); + } } break; case Pyc::LOAD_ATTR_A: @@ -1520,6 +1538,7 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) stack.push(new ASTName(code->getCellVar(mod, operand))); break; case Pyc::LOAD_FAST_A: + case Pyc::LOAD_FAST_CHECK_A: if (mod->verCompare(1, 3) < 0) stack.push(new ASTName(code->getName(operand))); else @@ -2584,6 +2603,77 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) stack.push(value); } break; + case Pyc::CALL_INTRINSIC_1_A: + { + PycRef arg = stack.top(); + stack.pop(); + + if (operand != ASTCallIntrinsic1::INTRINSIC_LIST_TO_TUPLE) { + fprintf(stderr, "Unimplemented function %i", operand); + break; + } + + if (arg.type() != ASTNode::NODE_LIST) { + fprintf(stderr, "Unexpected argument type %i\n", arg.type()); + break; + } + + PycRef list = arg.cast(); + ASTTuple::value_t values; + for (PycRef val : list->values()) { + values.push_back(val); + } + stack.push(new ASTTuple(values)); + } + break; + case Pyc::CALL_FUNCTION_EX_A: + { + int has_kwmap = operand & 1; + ASTCall::kwparam_t kwparamList; + ASTCall::pparam_t pparamList; + + // callable, iterable object & kwmap object (if present) + + if (has_kwmap) { + PycRef object_or_map = stack.top(); + if (object_or_map.type() == ASTNode::NODE_KW_NAMES_MAP) { + stack.pop(); + PycRef kwparams_map = object_or_map.cast(); + for (ASTKwNamesMap::map_t::const_iterator it = kwparams_map->values().begin(); it != kwparams_map->values().end(); it++) { + kwparamList.push_front(std::make_pair(it->first, it->second)); + } + } + else { + fprintf(stderr, "Unexpected object type %i\n", object_or_map.type()); + } + } + + PycRef iterable = stack.top(); + stack.pop(); + + // Not sure how to combine these two conditions + if (iterable.type() == ASTNode::NODE_LIST) { + PycRef list = iterable.cast(); + for (PycRef n: list->values()) { + pparamList.push_back(n); + } + } + else if (iterable.type() == ASTNode::NODE_TUPLE) { + PycRef tuple = iterable.cast(); + for (PycRef n: tuple->values()) { + pparamList.push_back(n); + } + } + else { + fprintf(stderr, "Unsupported iterable type %i\n", iterable.type()); + } + + PycRef func = stack.top(); + stack.pop(); + + stack.push(new ASTCall(func, pparamList, kwparamList)); + } + break; default: fprintf(stderr, "Unsupported opcode: %s (%d)\n", Pyc::OpcodeName(opcode), opcode); cleanBuild = false; @@ -2782,6 +2872,7 @@ void print_formatted_value(PycRef formatted_value, PycModule* static std::unordered_set node_seen; +// TODO: Handle m_unpack for node correctly here. void print_src(PycRef node, PycModule* mod, std::ostream& pyc_output) { if (node == NULL) { @@ -2906,6 +2997,10 @@ void print_src(PycRef node, PycModule* mod, std::ostream& pyc_output) break; case ASTNode::NODE_LIST: { + if (node.isUnpacked()) { + pyc_output << "*"; + } + pyc_output << "["; bool first = true; cur_indent++; @@ -2999,6 +3094,9 @@ void print_src(PycRef node, PycModule* mod, std::ostream& pyc_output) } break; case ASTNode::NODE_NAME: + if (node.isUnpacked()) { + pyc_output << "*"; + } pyc_output << node.cast()->name()->value(); break; case ASTNode::NODE_NODELIST: diff --git a/FastStack.h b/FastStack.h index b91ec71de..624cf0a98 100644 --- a/FastStack.h +++ b/FastStack.h @@ -61,6 +61,25 @@ class FastStack { return m_ptr == -1; } + void debug_print(PycModule* mod, std::ostream& pyc_output) + { + pyc_output << "---- STACK CONTENTS ----\n"; + if (empty()) { + pyc_output << "empty stack\n"; + } + else { + for (int i = m_ptr; i >= 0; i--) { + print_src(m_stack[i], mod, pyc_output); + if (i == m_ptr) { + pyc_output << " <- STACK TOP"; + } + pyc_output << "\n"; + } + } + pyc_output << "------------------------\n"; + } + + private: std::vector> m_stack; int m_ptr; diff --git a/pyc_object.h b/pyc_object.h index 085944496..56baad842 100644 --- a/pyc_object.h +++ b/pyc_object.h @@ -6,21 +6,21 @@ template class PycRef { public: - PycRef() noexcept : m_obj() { } + PycRef() noexcept : m_obj(), m_unpack(false) { } - PycRef(_Obj* obj) noexcept : m_obj(obj) + PycRef(_Obj* obj) noexcept : m_obj(obj), m_unpack(false) { if (m_obj) m_obj->addRef(); } - PycRef(const PycRef<_Obj>& obj) noexcept : m_obj(obj.m_obj) + PycRef(const PycRef<_Obj>& obj) noexcept : m_obj(obj.m_obj), m_unpack(obj.m_unpack) { if (m_obj) m_obj->addRef(); } - PycRef(PycRef<_Obj>&& obj) noexcept : m_obj(obj.m_obj) + PycRef(PycRef<_Obj>&& obj) noexcept : m_obj(obj.m_obj), m_unpack(obj.m_unpack) { obj.m_obj = nullptr; } @@ -38,6 +38,7 @@ class PycRef { if (m_obj) m_obj->delRef(); m_obj = obj; + m_unpack = false; return *this; } @@ -48,16 +49,20 @@ class PycRef { if (m_obj) m_obj->delRef(); m_obj = obj.m_obj; + m_unpack = obj.m_unpack; return *this; } PycRef<_Obj>& operator=(PycRef<_Obj>&& obj) noexcept { m_obj = obj.m_obj; + m_unpack = obj.m_unpack; obj.m_obj = nullptr; + obj.m_unpack = false; return *this; } + // TODO: Handle m_unpack for remaining operators bool operator==(_Obj* obj) const { return m_obj == obj; } bool operator==(const PycRef<_Obj>& obj) const { return m_obj == obj.m_obj; } bool operator!=(_Obj* obj) const { return m_obj != obj; } @@ -75,16 +80,28 @@ class PycRef { template PycRef<_Cast> cast() const { - _Cast* result = dynamic_cast<_Cast*>(m_obj); - if (!result) + _Cast* casted_obj = dynamic_cast<_Cast*>(m_obj); + if (!casted_obj) throw std::bad_cast(); + + PycRef<_Cast> result = casted_obj; + result.setUnpacked(m_unpack); + return result; } bool isIdent(const _Obj* obj) const { return m_obj == obj; } + bool isUnpacked() const { return m_unpack; } + void setUnpacked(bool unpack) { m_unpack = unpack; } + private: _Obj* m_obj; + + // References to an object can be either packed or unpacked. + // Usually unpacked references will be used with variables + // or lists but they may arise in other places as well. + bool m_unpack; }; diff --git a/tests/compiled/list_extend.3.9.pyc b/tests/compiled/list_extend_1.3.9.pyc similarity index 100% rename from tests/compiled/list_extend.3.9.pyc rename to tests/compiled/list_extend_1.3.9.pyc diff --git a/tests/compiled/list_extend_2.3.12.pyc b/tests/compiled/list_extend_2.3.12.pyc new file mode 100644 index 000000000..25c333a8e Binary files /dev/null and b/tests/compiled/list_extend_2.3.12.pyc differ diff --git a/tests/compiled/test_unpack.3.12.pyc b/tests/compiled/test_unpack.3.12.pyc new file mode 100644 index 000000000..5eb9cecc5 Binary files /dev/null and b/tests/compiled/test_unpack.3.12.pyc differ diff --git a/tests/input/list_extend.py b/tests/input/list_extend_1.py similarity index 100% rename from tests/input/list_extend.py rename to tests/input/list_extend_1.py diff --git a/tests/input/list_extend_2.py b/tests/input/list_extend_2.py new file mode 100644 index 000000000..1473f17de --- /dev/null +++ b/tests/input/list_extend_2.py @@ -0,0 +1,2 @@ +def get(l): + return [*l] diff --git a/tests/input/test_unpack.py b/tests/input/test_unpack.py new file mode 100644 index 000000000..fdf2ebf9e --- /dev/null +++ b/tests/input/test_unpack.py @@ -0,0 +1,6 @@ +import struct + +def wtob(w): + return struct.pack('<'+'I'*len(w), *w) + +wtob([12,3]) diff --git a/tests/tokenized/list_extend.txt b/tests/tokenized/list_extend_1.txt similarity index 100% rename from tests/tokenized/list_extend.txt rename to tests/tokenized/list_extend_1.txt diff --git a/tests/tokenized/list_extend_2.txt b/tests/tokenized/list_extend_2.txt new file mode 100644 index 000000000..31a437e0c --- /dev/null +++ b/tests/tokenized/list_extend_2.txt @@ -0,0 +1,3 @@ +def get ( l ) : + +return [ * l ] diff --git a/tests/tokenized/test_unpack.txt b/tests/tokenized/test_unpack.txt new file mode 100644 index 000000000..16c7af1f4 --- /dev/null +++ b/tests/tokenized/test_unpack.txt @@ -0,0 +1,6 @@ +import struct +def wtob ( w ) : + +return struct . pack ( '<' + 'I' * len ( w ) , * w ) + +wtob ( [ 12 , 3 ] )