generated_cases.c.h

// This file is generated by Tools/cases_generator/tier1_generator.py
// from:
//   Python/bytecodes.c
// Do not edit!

#ifdef TIER_TWO
    #error "This file is for Tier 1 only"
#endif
#define TIER_ONE 1


        TARGET(BINARY_OP) {
            frame->instr_ptr = next_instr;
            next_instr += 2;
            INSTRUCTION_STATS(BINARY_OP);
            PREDICTED(BINARY_OP);
            _Py_CODEUNIT *this_instr = next_instr - 2;
            (void)this_instr;
            _PyStackRef lhs;
            _PyStackRef rhs;
            _PyStackRef res;
            // _SPECIALIZE_BINARY_OP
            rhs = stack_pointer[-1];
            lhs = stack_pointer[-2];
            {
                uint16_t counter = read_u16(&this_instr[1].cache);
                (void)counter;
                #if ENABLE_SPECIALIZATION
                if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
                    next_instr = this_instr;
                    _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY);
                    DISPATCH_SAME_OPARG();
                }
                STAT_INC(BINARY_OP, deferred);
                ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
                #endif  /* ENABLE_SPECIALIZATION */
                assert(NB_ADD <= oparg);
                assert(oparg <= NB_INPLACE_XOR);
            }
            // _BINARY_OP
            {
                PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs);
                PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs);
                assert(_PyEval_BinaryOps[oparg]);
                PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o);
                PyStackRef_CLOSE(lhs);
                PyStackRef_CLOSE(rhs);
                if (res_o == NULL) goto pop_2_error;
                res = PyStackRef_FromPyObjectSteal(res_o);
            }
            stack_pointer[-2] = res;
            stack_pointer += -1;
            assert(WITHIN_STACK_BOUNDS());
            DISPATCH();
        }

        TARGET(BINARY_OP_ADD_FLOAT) {
            frame->instr_ptr = next_instr;
            next_instr += 2;
            INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT);
            static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
            _PyStackRef left;
            _PyStackRef right;
            _PyStackRef res;
            // _GUARD_BOTH_FLOAT
            right = stack_pointer[-1];
            left = stack_pointer[-2];
            {
                PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
                PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
                DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP);
                DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP);
            }
            /* Skip 1 cache entry */
            // _BINARY_OP_ADD_FLOAT
            {
                PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
                PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
                STAT_INC(BINARY_OP, hit);
                double dres =
                ((PyFloatObject *)left_o)->ob_fval +
                ((PyFloatObject *)right_o)->ob_fval;
                PyObject *res_o;
                DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o);
                res = PyStackRef_FromPyObjectSteal(res_o);
            }
            stack_pointer[-2] = res;
            stack_pointer += -1;
            assert(WITHIN_STACK_BOUNDS());
            DISPATCH();
        }

        TARGET(BINARY_OP_ADD_INT) {
            frame->instr_ptr = next_instr;
            next_instr += 2;
            INSTRUCTION_STATS(BINARY_OP_ADD_INT);
            static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
            _PyStackRef left;
            _PyStackRef right;
            _PyStackRef res;
            // _GUARD_BOTH_INT
            right = stack_pointer[-1];
            left = stack_pointer[-2];
            {
                PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
                PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
                DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP);
                DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP);
            }
            /* Skip 1 cache entry */
            // _BINARY_OP_ADD_INT
            {
                PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
                PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
                STAT_INC(BINARY_OP, hit);
                PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o);
                _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free);
                _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free);
                if (res_o == NULL) goto pop_2_error;
                res = PyStackRef_FromPyObjectSteal(res_o);
            }
            stack_pointer[-2] = res;
            stack_pointer += -1;
            assert(WITHIN_STACK_BOUNDS());
            DISPATCH();
        }

        TARGET(BINARY_OP_ADD_UNICODE) {
            frame->instr_ptr = next_instr;
            next_instr += 2;
            INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE);
            static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
            _PyStackRef left;
            _PyStackRef right;
            _PyStackRef res;
            // _GUARD_BOTH_UNICODE
            right = stack_pointer[-1];
            left = stack_pointer[-2];
            {
                PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
                PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
                DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP);
                DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP);
            }
            /* Skip 1 cache entry */
            // _BINARY_OP_ADD_UNICODE
            {
                PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
                PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
                STAT_INC(BINARY_OP, hit);
                PyObject *res_o = PyUnicode_Concat(left_o, right_o);
                _Py_DECREF_SPECIALIZED(left_o, _PyUnicode_ExactDealloc);
                _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc);
                if (res_o == NULL) goto pop_2_error;
                res = PyStackRef_FromPyObjectSteal(res_o);
            }
            stack_pointer[-2] = res;
            stack_pointer += -1;
            assert(WITHIN_STACK_BOUNDS());
            DISPATCH();
        }

        TARGET(BINARY_OP_INPLACE_ADD_UNICODE) {
            frame->instr_ptr = next_instr;
            next_instr += 2;
            INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE);
            static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
            _PyStackRef left;
            _PyStackRef right;
            // _GUARD_BOTH_UNICODE
            right = stack_pointer[-1];
            left = stack_pointer[-2];
            {
                PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
                PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
                DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP);
                DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP);
            }
            /* Skip 1 cache entry */
            // _BINARY_OP_INPLACE_ADD_UNICODE
            {
                PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
                PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
                int next_oparg;
                #if TIER_ONE
                assert(next_instr->op.code == STORE_FAST);
                next_oparg = next_instr->op.arg;
                #else
                next_oparg = CURRENT_OPERAND();
                #endif
                _PyStackRef *target_local = &GETLOCAL(next_oparg);
                DEOPT_IF(!PyStackRef_Is(*target_local, left), BINARY_OP);
                STAT_INC(BINARY_OP, hit);
                /* Handle `left = left + right` or `left += right` for str.
                 *
                 * When possible, extend `left` in place rather than
                 * allocating a new PyUnicodeObject. This attempts to avoid
                 * quadratic behavior when one neglects to use str.join().
                 *
                 * If `left` has only two references remaining (one from
                 * the stack, one in the locals), DECREFing `left` leaves
                 * only the locals reference, so PyUnicode_Append knows
                 * that the string is safe to mutate.
                 */
                assert(Py_REFCNT(left_o) >= 2);
                _Py_DECREF_NO_DEALLOC(left_o);
                PyObject *temp = PyStackRef_AsPyObjectBorrow(*target_local);
                PyUnicode_Append(&temp, right_o);
                *target_local = PyStackRef_FromPyObjectSteal(temp);
                _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc);
                if (PyStackRef_IsNull(*target_local)) goto pop_2_error;
                #if TIER_ONE
                // The STORE_FAST is already done. This is done here in tier one,
                // and during trace projection in tier two:
                assert(next_instr->op.code == STORE_FAST);
                SKIP_OVER(1);
                #endif
            }
            stack_pointer += -2;
            assert(WITHIN_STACK_BOUNDS());
            DISPATCH();
        }

        TARGET(BINARY_OP_MULTIPLY_FLOAT) {
            frame->instr_ptr = next_instr;
            next_instr += 2;
            INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT);
            static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
            _PyStackRef left;
            _PyStackRef right;
            _PyStackRef res;
            // _GUARD_BOTH_FLOAT
            right = stack_pointer[-1];
            left = stack_pointer[-2];
            {
                PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
                PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
                DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP);
                DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP);
            }
            /* Skip 1 cache entry */
            // _BINARY_OP_MULTIPLY_FLOAT
            {
                PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
                PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
                STAT_INC(BINARY_OP, hit);
                double dres =
                ((PyFloatObject *)left_o)->ob_fval *
                ((PyFloatObject *)right_o)->ob_fval;
                PyObject *res_o;
                DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o);
                res = PyStackRef_FromPyObjectSteal(res_o);
            }
            stack_pointer[-2] = res;
            stack_pointer += -1;
            assert(WITHIN_STACK_BOUNDS());
            DISPATCH();
        }

        TARGET(BINARY_OP_MULTIPLY_INT) {
            frame->instr_ptr = next_instr;
            next_instr += 2;
            INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT);
            static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
            _PyStackRef left;
            _PyStackRef right;
            _PyStackRef res;
            // _GUARD_BOTH_INT
            right = stack_pointer[-1];
            left = stack_pointer[-2];
            {
                PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
                PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
                DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP);
                DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP);
            }
            /* Skip 1 cache entry */
            // _BINARY_OP_MULTIPLY_INT
            {
                PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
                PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
                STAT_INC(BINARY_OP, hit);
                PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o);
                _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free);
                _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free);
                if (res_o == NULL) goto pop_2_error;
                res = PyStackRef_FromPyObjectSteal(res_o);
            }
            stack_pointer[-2] = res;
            stack_pointer += -1;
            assert(WITHIN_STACK_BOUNDS());
            DISPATCH();
        }

        TARGET(BINARY_OP_SUBTRACT_FLOAT) {
            frame->instr_ptr = next_instr;
            next_instr += 2;
            INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT);
            static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
            _PyStackRef left;
            _PyStackRef right;
            _PyStackRef res;
            // _GUARD_BOTH_FLOAT
            right = stack_pointer[-1];
            left = stack_pointer[-2];
            {
                PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
                PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
                DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP);
                DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP);
            }
            /* Skip 1 cache entry */
            // _BINARY_OP_SUBTRACT_FLOAT
            {
                PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
                PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
                STAT_INC(BINARY_OP, hit);
                double dres =
                ((PyFloatObject *)left_o)->ob_fval -
                ((PyFloatObject *)right_o)->ob_fval;
                PyObject *res_o;
                DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o);
                res = PyStackRef_FromPyObjectSteal(res_o);
            }
            stack_pointer[-2] = res;
            stack_pointer += -1;
            assert(WITHIN_STACK_BOUNDS());
            DISPATCH();
        }

        TARGET(BINARY_OP_SUBTRACT_INT) {
            frame->instr_ptr = next_instr;
            next_instr += 2;
            INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT);
            static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
            _PyStackRef left;
            _PyStackRef right;
            _PyStackRef res;
            // _GUARD_BOTH_INT
            right = stack_pointer[-1];
            left = stack_pointer[-2];
            {
                PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
                PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
                DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP);
                DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP);
            }
            /* Skip 1 cache entry */
            // _BINARY_OP_SUBTRACT_INT
            {
                PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
                PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
                STAT_INC(BINARY_OP, hit);
                PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o);
                _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free);
                _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free);;
                if (res_o == NULL) goto pop_2_error;
                res = PyStackRef_FromPyObjectSteal(res_o);
            }
            stack_pointer[-2] = res;
            stack_pointer += -1;
            assert(WITHIN_STACK_BOUNDS());
            DISPATCH();
        }

        TARGET(BINARY_SLICE) {
            frame->instr_ptr = next_instr;
            next_instr += 1;
            INSTRUCTION_STATS(BINARY_SLICE);
            _PyStackRef container;
            _PyStackRef start;
            _PyStackRef stop;
            _PyStackRef res;
            stop = stack_pointer[-1];
            start = stack_pointer[-2];
            container = stack_pointer[-3];
            PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start),
                PyStackRef_AsPyObjectSteal(stop));
            PyObject *res_o;
            // Can't use ERROR_IF() here, because we haven't
            // DECREF'ed container yet, and we still own slice.
            if (slice == NULL) {
                res_o = NULL;
            }
            else {
                res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice);
                Py_DECREF(slice);
            }
            PyStackRef_CLOSE(container);
            if (res_o == NULL) goto pop_3_error;
            res = PyStackRef_FromPyObjectSteal(res_o);
            stack_pointer[-3] = res;
            stack_pointer += -2;
            assert(WITHIN_STACK_BOUNDS());
            DISPATCH();
        }

        TARGET(BINARY_SUBSCR) {
            frame->instr_ptr = next_instr;
            next_instr += 2;
            INSTRUCTION_STATS(BINARY_SUBSCR);
            PREDICTED(BINARY_SUBSCR);
            _Py_CODEUNIT *this_instr = next_instr - 2;
            (void)this_instr;
            _PyStackRef container;
            _PyStackRef sub;
            _PyStackRef res;
            // _SPECIALIZE_BINARY_SUBSCR
            sub = stack_pointer[-1];
            container = stack_pointer[-2];
            {
                uint16_t counter = read_u16(&this_instr[1].cache);
                (void)counter;
                #if ENABLE_SPECIALIZATION
                if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
                    next_instr = this_instr;
                    _Py_Specialize_BinarySubscr(container, sub, next_instr);
                    DISPATCH_SAME_OPARG();
                }
                STAT_INC(BINARY_SUBSCR, deferred);
                ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
                #endif  /* ENABLE_SPECIALIZATION */
            }
            // _BINARY_SUBSCR
            {
                PyObject *container_o = PyStackRef_AsPyObjectBorrow(container);
                PyObject *sub_o = PyStackRef_AsPyObjectBorrow(sub);
                PyObject *res_o = PyObject_GetItem(container_o, sub_o);
                PyStackRef_CLOSE(container);
                PyStackRef_CLOSE(sub);
                if (res_o == NULL) goto pop_2_error;
                res = PyStackRef_FromPyObjectSteal(res_o);
            }
            stack_pointer[-2] = res;
            stack_pointer += -1;
            assert(WITHIN_STACK_BOUNDS());
            DISPATCH();
        }

        TARGET(BINARY_SUBSCR_DICT) {
            frame->instr_ptr = next_instr;
            next_instr += 2;
            INSTRUCTION_STATS(BINARY_SUBSCR_DICT);
            static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size");
            _PyStackRef dict_st;
            _PyStackRef sub_st;
            _PyStackRef res;
            /* Skip 1 cache entry */
            sub_st = stack_pointer[-1];
            dict_st = stack_pointer[-2];
            PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
            PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st);
            DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR);
            STAT_INC(BINARY_SUBSCR, hit);
            PyObject *res_o;
            int rc = PyDict_GetItemRef(dict, sub, &res_o);
            if (rc == 0) {
                _PyErr_SetKeyError(sub);
            }
            PyStackRef_CLOSE(dict_st);
            PyStackRef_CLOSE(sub_st);
            if (rc <= 0) goto pop_2_error;
            // not found or error
            res = PyStackRef_FromPyObjectSteal(res_o);
            stack_pointer[-2] = res;
            stack_pointer += -1;
            assert(WITHIN_STACK_BOUNDS());
            DISPATCH();
        }

        TARGET(BINARY_SUBSCR_GETITEM) {
            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
            next_instr += 2;
            INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM);
            static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size");
            _PyStackRef container_st;
            _PyStackRef sub_st;
            /* Skip 1 cache entry */
            sub_st = stack_pointer[-1];
            container_st = stack_pointer[-2];
            PyObject *container = PyStackRef_AsPyObjectBorrow(container_st);
            DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR);
            PyTypeObject *tp = Py_TYPE(container);
            DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR);
            PyHeapTypeObject *ht = (PyHeapTypeObject *)tp;
            PyObject *cached = ht->_spec_cache.getitem;
            DEOPT_IF(cached == NULL, BINARY_SUBSCR);
            assert(PyFunction_Check(cached));
            PyFunctionObject *getitem = (PyFunctionObject *)cached;
            uint32_t cached_version = ht->_spec_cache.getitem_version;
            DEOPT_IF(getitem->func_version != cached_version, BINARY_SUBSCR);
            PyCodeObject *code = (PyCodeObject *)getitem->func_code;
            assert(code->co_argcount == 2);
            DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR);
            STAT_INC(BINARY_SUBSCR, hit);
            Py_INCREF(getitem);
            _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2);
            STACK_SHRINK(2);
            new_frame->localsplus[0] = container_st;
            new_frame->localsplus[1] = sub_st;
            frame->return_offset = (uint16_t)(next_instr - this_instr);
            DISPATCH_INLINED(new_frame);
        }

        TARGET(BINARY_SUBSCR_LIST_INT) {
            frame->instr_ptr = next_instr;
            next_instr += 2;
            INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT);
            static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size");
            _PyStackRef list_st;
            _PyStackRef sub_st;
            _PyStackRef res;
            /* Skip 1 cache entry */
            sub_st = stack_pointer[-1];
            list_st = stack_pointer[-2];
            PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
            PyObject *list = PyStackRef_AsPyObjectBorrow(list_st);
            DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
            DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR);
            // Deopt unless 0 <= sub < PyList_Size(list)
            DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR);
            Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
            DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR);
            STAT_INC(BINARY_SUBSCR, hit);
            PyObject *res_o = PyList_GET_ITEM(list, index);
            assert(res_o != NULL);
            Py_INCREF(res_o);
            _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
            PyStackRef_CLOSE(list_st);
            res = PyStackRef_FromPyObjectSteal(res_o);
            stack_pointer[-2] = res;
            stack_pointer += -1;
            assert(WITHIN_STACK_BOUNDS());
            DISPATCH();
        }

        TARGET(BINARY_SUBSCR_STR_INT) {
            frame->instr_ptr = next_instr;
            next_instr += 2;
            INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT);
            static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size");
            _PyStackRef str_st;
            _PyStackRef sub_st;
            _PyStackRef res;
            /* Skip 1 cache entry */
            sub_st = stack_pointer[-1];
            str_st = stack_pointer[-2];
            PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
            PyObject *str = PyStackRef_AsPyObjectBorrow(str_st);
            DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
            DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR);
            DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR);
            Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
            DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR);
            // Specialize for reading an ASCII character from any string:
            Py_UCS4 c = PyUnicode_READ_CHAR(str, index);
            DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR);
            STAT_INC(BINARY_SUBSCR, hit);
            PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c];
            _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
            PyStackRef_CLOSE(str_st);
            res = PyStackRef_FromPyObjectSteal(res_o);
            stack_pointer[-2] = res;
            stack_pointer += -1;
            assert(WITHIN_STACK_BOUNDS());
            DISPATCH();
        }

        TARGET(BINARY_SUBSCR_TUPLE_INT) {
            frame->instr_ptr = next_instr;
            next_instr += 2;
            INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT);
            static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size");
            _PyStackRef tuple_st;
            _PyStackRef sub_st;
            _PyStackRef res;
            /* Skip 1 cache entry */
            sub_st = stack_pointer[-1];
            tuple_st = stack_pointer[-2];
            PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
            PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st);
            DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
            DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR);
            // Deopt unless 0 <= sub < PyTuple_Size(list)
            DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR);
            Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
            DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR);
            STAT_INC(BINARY_SUBSCR, hit);
            PyObject *res_o = PyTuple_GET_ITEM(tuple, index);
            assert(res_o != NULL);
            Py_INCREF(res_o);
            _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
            PyStackRef_CLOSE(tuple_st);
            res = PyStackRef_FromPyObjectSteal(res_o);
            stack_pointer[-2] = res;
            stack_pointer += -1;
            assert(WITHIN_STACK_BOUNDS());
            DISPATCH();
        }

        TARGET(BUILD_LIST) {
            frame->instr_ptr = next_instr;
            next_instr += 1;
            INSTRUCTION_STATS(BUILD_LIST);
            _PyStackRef *values;
            _PyStackRef list;
            values = &stack_pointer[-oparg];
            STACKREFS_TO_PYOBJECTS(values, oparg, values_o);
            if (CONVERSION_FAILED(values_o)) {
                for (int _i = oparg; --_i >= 0;) {
                    PyStackRef_CLOSE(values[_i]);
                }
                if (true) { stack_pointer += -oparg; goto error; }
            }
            PyObject *list_o = _PyList_FromArraySteal(values_o, oparg);
            STACKREFS_TO_PYOBJECTS_CLEANUP(values_o);
            if (list_o == NULL) { stack_pointer += -oparg; goto error; }
            list = PyStackRef_FromPyObjectSteal(list_o);
            stack_pointer[-oparg] = list;
            stack_pointer += 1 - oparg;
            assert(WITHIN_STACK_BOUNDS());
            DISPATCH();
        }

        TARGET(BUILD_MAP) {
            frame->instr_ptr = next_instr;
            next_instr += 1;
            INSTRUCTION_STATS(BUILD_MAP);
            _PyStackRef *values;
            _PyStackRef map;
            values = &stack_pointer[-oparg*2];
            STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o);
            if (CONVERSION_FAILED(values_o)) {
                for (int _i = oparg*2; --_i >= 0;) {
                    PyStackRef_CLOSE(values[_i]);
                }
                if (true) { stack_pointer += -oparg*2; goto error; }
            }
            PyObject *map_o = _PyDict_FromItems(
                values_o, 2,
                values_o+1, 2,
                oparg);
            STACKREFS_TO_PYOBJECTS_CLEANUP(values_o);
            for (int _i = oparg*2; --_i >= 0;) {
                PyStackRef_CLOSE(values[_i]);
            }
            if (map_o == NULL) { stack_pointer += -oparg*2; goto error; }
            map = PyStackRef_FromPyObjectSteal(map_o);
            stack_pointer[-oparg*2] = map;
            stack_pointer += 1 - oparg*2;
            assert(WITHIN_STACK_BOUNDS());
            DISPATCH();
        }

        TARGET(BUILD_SET) {
            frame->instr_ptr = next_instr;
            next_instr += 1;
            INSTRUCTION_STATS(BUILD_SET);
            _PyStackRef *values;
            _PyStackRef set;
            values = &stack_pointer[-oparg];
            PyObject *set_o = PySet_New(NULL);
            if (set_o == NULL) {
                for (int _i = oparg; --_i >= 0;) {
                    PyStackRef_CLOSE(values[_i]);
                }
                if (true) { stack_pointer += -oparg; goto error; }
            }
            int err = 0;
            for (int i = 0; i < oparg; i++) {
                PyObject *item = PyStackRef_AsPyObjectSteal(values[i]);
                if (err == 0) {
                    err = PySet_Add(set_o, item);
                }
                Py_DECREF(item);
            }
            if (err != 0) {
                Py_DECREF(set_o);
                if (true) { stack_pointer += -oparg; goto error; }
            }
            set = PyStackRef_FromPyObjectSteal(set_o);
            stack_pointer[-oparg] = set;
            stack_pointer += 1 - oparg;
            assert(WITHIN_STACK_BOUNDS());
            DISPATCH();
        }

        TARGET(BUILD_SLICE) {
            frame->instr_ptr = next_instr;
            next_instr += 1;
            INSTRUCTION_STATS(BUILD_SLICE);
            _PyStackRef start;
            _PyStackRef stop;
            _PyStackRef step = PyStackRef_NULL;
            _PyStackRef slice;
            if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; }
            stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)];
            start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)];
            PyObject *start_o = PyStackRef_AsPyObjectBorrow(start);
            PyObject *stop_o = PyStackRef_AsPyObjectBorrow(stop);
            PyObject *step_o = PyStackRef_AsPyObjectBorrow(step);
            PyObject *slice_o = PySlice_New(start_o, stop_o, step_o);
            PyStackRef_CLOSE(start);
            PyStackRef_CLOSE(stop);
            PyStackRef_XCLOSE(step);
            if (slice_o == NULL) { stack_pointer += -2 - ((oparg == 3) ? 1 : 0); goto error; }
            slice = PyStackRef_FromPyObjectSteal(slice_o);
            stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice;
            stack_pointer += -1 - ((oparg == 3) ? 1 : 0);
            assert(WITHIN_STACK_BOUNDS());
            DISPATCH();
        }

        TARGET(BUILD_STRING) {
            frame->instr_ptr = next_instr;
            next_instr += 1;
            INSTRUCTION_STATS(BUILD_STRING);
            _PyStackRef *pieces;
            _PyStackRef str;
            pieces = &stack_pointer[-oparg];
            STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o);
            if (CONVERSION_FAILED(pieces_o)) {
                for (int _i = oparg; --_i >= 0;) {
                    PyStackRef_CLOSE(pieces[_i]);
                }
                if (true) { stack_pointer += -oparg; goto error; }
            }
            PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg);
            STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o);
            for (int _i = oparg; --_i >= 0;) {
                PyStackRef_CLOSE(pieces[_i]);
            }
            if (str_o == NULL) { stack_pointer += -oparg; goto error; }
            str = PyStackRef_FromPyObjectSteal(str_o);
            stack_pointer[-oparg] = str;
            stack_pointer += 1 - oparg;
            assert(WITHIN_STACK_BOUNDS());
            DISPATCH();
        }

        TARGET(BUILD_TUPLE) {
            frame->instr_ptr = next_instr;
            next_instr += 1;
            INSTRUCTION_STATS(BUILD_TUPLE);
            _PyStackRef *values;
            _PyStackRef tup;
            values = &stack_pointer[-oparg];
            PyObject *tup_o = _PyTuple_FromStackRefSteal(values, oparg);
            if (tup_o == NULL) { stack_pointer += -oparg; goto error; }
            tup = PyStackRef_FromPyObjectSteal(tup_o);
            stack_pointer[-oparg] = tup;
            stack_pointer += 1 - oparg;
            assert(WITHIN_STACK_BOUNDS());
            DISPATCH();
        }

        TARGET(CACHE) {
            frame->instr_ptr = next_instr;
            next_instr += 1;
            INSTRUCTION_STATS(CACHE);
            assert(0 && "Executing a cache.");
            Py_FatalError("Executing a cache.");
            DISPATCH();
        }

        TARGET(CALL) {
            frame->instr_ptr = next_instr;
            next_instr += 4;
            INSTRUCTION_STATS(CALL);
            PREDICTED(CALL);
            _Py_CODEUNIT *this_instr = next_instr - 4;
            (void)this_instr;
            _PyStackRef callable;
            _PyStackRef self_or_null;
            _PyStackRef *args;
            _PyStackRef func;
            _PyStackRef maybe_self;
            _PyStackRef res;
            // _SPECIALIZE_CALL
            self_or_null = stack_pointer[-1 - oparg];
            callable = stack_pointer[-2 - oparg];
            {
                args = &stack_pointer[-oparg];
                uint16_t counter = read_u16(&this_instr[1].cache);
                (void)counter;
                #if ENABLE_SPECIALIZATION
                if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
                    next_instr = this_instr;
                    _Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null));
                    DISPATCH_SAME_OPARG();
                }
                STAT_INC(CALL, deferred);
                ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
                #endif  /* ENABLE_SPECIALIZATION */
            }
            /* Skip 2 cache entries */
            // _MAYBE_EXPAND_METHOD
            {
                if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) {
                    PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
                    PyObject *self = ((PyMethodObject *)callable_o)->im_self;
                    maybe_self = PyStackRef_FromPyObjectNew(self);
                    PyObject *method = ((PyMethodObject *)callable_o)->im_func;
                    func = PyStackRef_FromPyObjectNew(method);
                    /* Make sure that callable and all args are in memory */
                    args[-2] = func;
                    args[-1] = maybe_self;
                    PyStackRef_CLOSE(callable);
                }
                else {
                    func = callable;
                    maybe_self = self_or_null;
                }
            }
            // _DO_CALL
            self_or_null = maybe_self;
            callable = func;
            {
                PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
                // oparg counts all of the args, but *not* self:
                int total_args = oparg;
                if (!PyStackRef_IsNull(self_or_null)) {
                    args--;
                    total_args++;
                }
                // Check if the call can be inlined or not
                if (Py_TYPE(callable_o) == &PyFunction_Type &&
                    tstate->interp->eval_frame == NULL &&
                    ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall)
                {
                    int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags;
                    PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o));
                    _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit(
                        tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals,
                        args, total_args, NULL
                    );
                    // Manipulate stack directly since we leave using DISPATCH_INLINED().
                    STACK_SHRINK(oparg + 2);
                    // The frame has stolen all the arguments from the stack,
                    // so there is no need to clean them up.
                    if (new_frame == NULL) {
                        goto error;
                    }
                    frame->return_offset = (uint16_t)(next_instr - this_instr);
                    DISPATCH_INLINED(new_frame);
                }
                /* Callable is not a normal Python function */
                STACKREFS_TO_PYOBJECTS(args, total_args, args_o);
                if (CONVERSION_FAILED(args_o)) {
                    PyStackRef_CLOSE(callable);
                    PyStackRef_CLOSE(self_or_null);
                    for (int _i = oparg; --_i >= 0;) {
                        PyStackRef_CLOSE(args[_i]);
                    }
                    if (true) { stack_pointer += -2 - oparg; goto error; }
                }
                PyObject *res_o = PyObject_Vectorcall(
                    callable_o, args_o,
                    total_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
                    NULL);
                STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
                if (opcode == INSTRUMENTED_CALL) {
                    PyObject *arg = total_args == 0 ?
                    &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]);
                    if (res_o == NULL) {
                        _Py_call_instrumentation_exc2(
                            tstate, PY_MONITORING_EVENT_C_RAISE,
                            frame, this_instr, callable_o, arg);
                    }
                    else {
                        int err = _Py_call_instrumentation_2args(
                            tstate, PY_MONITORING_EVENT_C_RETURN,
                            frame, this_instr, callable_o, arg);
                        if (err < 0) {
                            Py_CLEAR(res_o);
                        }
                    }
                }
                assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
                PyStackRef_CLOSE(callable);
                for (int i = 0; i < total_args; i++) {
                    PyStackRef_CLOSE(args[i]);
                }
                if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; }
                res = PyStackRef_FromPyObjectSteal(res_o);
            }
            // _CHECK_PERIODIC
            {
            }
            stack_pointer[-2 - oparg] = res;
            stack_pointer += -1 - oparg;
            assert(WITHIN_STACK_BOUNDS());
            CHECK_EVAL_BREAKER();
            DISPATCH();
        }

        TARGET(CALL_ALLOC_AND_ENTER_INIT) {
            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
            next_instr += 4;
            INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT);
            static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
            _PyStackRef callable;
            _PyStackRef null;
            _PyStackRef *args;
            /* Skip 1 cache entry */
            /* Skip 2 cache entries */
            args = &stack_pointer[-oparg];
            null = stack_pointer[-1 - oparg];
            callable = stack_pointer[-2 - oparg];
            PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
            /* This instruction does the following:
             * 1. Creates the object (by calling ``object.__new__``)
             * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``)
             * 3. Pushes the frame for ``__init__`` to the frame stack
             * */
            _PyCallCache *cache = (_PyCallCache *)&this_instr[1];
            DEOPT_IF(!PyStackRef_IsNull(null), CALL);
            DEOPT_IF(!PyType_Check(callable_o), CALL);
            PyTypeObject *tp = (PyTypeObject *)callable_o;
            DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version), CALL);
            assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES);
            PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o;
            PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init;
            PyCodeObject *code = (PyCodeObject *)init->func_code;
            DEOPT_IF(code->co_argcount != oparg+1, CALL);
            DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL);
            STAT_INC(CALL, hit);
            PyObject *self = _PyType_NewManagedObject(tp);
            if (self == NULL) {
                goto error;
            }
            PyStackRef_CLOSE(callable);
            _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked(
                tstate, (PyCodeObject *)&_Py_InitCleanup, 1);
            assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK);
            /* Push self onto stack of shim */
            Py_INCREF(self);
            shim->localsplus[0] = PyStackRef_FromPyObjectSteal(self);
            Py_INCREF(init);
            _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1);
            /* Copy self followed by args to __init__ frame */
            init_frame->localsplus[0] = PyStackRef_FromPyObjectSteal(self);
            for (int i = 0; i < oparg; i++) {
                init_frame->localsplus[i+1] = args[i];
            }
            frame->return_offset = (uint16_t)(next_instr - this_instr);
            STACK_SHRINK(oparg+2);
            _PyFrame_SetStackPointer(frame, stack_pointer);
            /* Link frames */
            init_frame->previous = shim;
            shim->previous = frame;
            frame = tstate->current_frame = init_frame;
            CALL_STAT_INC(inlined_py_calls);
            /* Account for pushing the extra frame.
             * We don't check recursion depth here,
             * as it will be checked after start_frame */
            tstate->py_recursion_remaining--;
            goto start_frame;
        }

        TARGET(CALL_BOUND_METHOD_EXACT_ARGS) {
            _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
            next_instr += 4;
            INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS);
            static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
            _PyStackRef callable;
            _PyStackRef null;
            _PyStackRef func;
            _PyStackRef self;
            _PyStackRef self_or_null;
            _PyStackRef *args;
            _PyInterpreterFrame *new_frame;
            /* Skip 1 cache entry */
            // _CHECK_PEP_523
            {
                DEOPT_IF(tstate->interp->eval_frame, CALL);
            }
            // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS
            null = stack_pointer[-1 - oparg];
            callable = stack_pointer[-2 - oparg];
            {
                DEOPT_IF(!PyStackRef_IsNull(null), CALL);
                DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type, CALL);
            }
            // _INIT_CALL_BOUND_METHOD_EXACT_ARGS
            {
                PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
                STAT_INC(CALL, hit);
                self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self);
                func = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func);
                PyStackRef_CLOSE(callable);
            }
            // flush
            stack_pointer[-2 - oparg] = func;
            stack_pointer[-1 - oparg] = self;
            // _CHECK_FUNCTION_VERSION
            callable = stack_pointer[-2 - oparg];
            {
                uint32_t func_version = read_u32(&this_instr[2].cache);
                PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);