diff --git a/Include/internal/pycore_optimizer_types.h b/Include/internal/pycore_optimizer_types.h index 0a193268c4d618..1062c4253feba1 100644 --- a/Include/internal/pycore_optimizer_types.h +++ b/Include/internal/pycore_optimizer_types.h @@ -128,6 +128,8 @@ typedef struct _JitOptContext { JitOptRef *n_consumed; JitOptRef *limit; JitOptRef locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; + + struct _PyJitTracerState *tracer; } JitOptContext; diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index 518fd94a31ae5e..2b41d6ad848a16 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -59,6 +59,8 @@ typedef struct _PyJitTracerState { _PyJitTracerTranslatorState translator_state; JitOptContext opt_context; _PyUOpInstruction code_buffer[UOP_MAX_TRACE_LENGTH]; + _PyUOpInstruction out_buffer[UOP_MAX_TRACE_LENGTH]; + int out_len; } _PyJitTracerState; #endif diff --git a/Python/optimizer.c b/Python/optimizer.c index 79ac179d0b710a..0d7dd0446fe255 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1509,6 +1509,7 @@ uop_optimize( if (length <= 0) { return length; } + buffer = _tstate->jit_tracer_state->out_buffer; } assert(length < UOP_MAX_TRACE_LENGTH/2); assert(length >= 1); diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index e855df4977acf8..230c06afc4b0b3 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -143,6 +143,18 @@ incorrect_keys(PyObject *obj, uint32_t version) #define STACK_LEVEL() ((int)(stack_pointer - ctx->frame->stack)) #define STACK_SIZE() ((int)(ctx->frame->stack_len)) +static inline int +is_terminator_uop(const _PyUOpInstruction *uop) +{ + int opcode = uop->opcode; + return ( + opcode == _EXIT_TRACE || + opcode == _JUMP_TO_TOP || + opcode == _DYNAMIC_EXIT || + opcode == _DEOPT + ); +} + #define CURRENT_FRAME_IS_INIT_SHIM() (ctx->frame->code == ((PyCodeObject *)&_Py_InitCleanup)) #define GETLOCAL(idx) ((ctx->frame->locals[idx])) @@ -152,6 +164,22 @@ incorrect_keys(PyObject *obj, uint32_t version) (INST)->oparg = ARG; \ (INST)->operand0 = OPERAND; +#define ADD_OP(OP, ARG, OPERAND) add_op(ctx, this_instr, (OP), (ARG), (OPERAND)) + +static inline void +add_op(JitOptContext *ctx, _PyUOpInstruction *this_instr, + uint16_t opcode, uint16_t oparg, uintptr_t operand0) +{ + _PyUOpInstruction *out = &ctx->tracer->out_buffer[ctx->tracer->out_len]; + out->opcode = (opcode); + out->format = this_instr->format; + out->oparg = (oparg); + out->target = this_instr->target; + out->operand0 = (operand0); + out->operand1 = this_instr->operand1; + ctx->tracer->out_len++; +} + /* Shortened forms for convenience, used in optimizer_bytecodes.c */ #define sym_is_not_null _Py_uop_sym_is_not_null #define sym_is_const _Py_uop_sym_is_const @@ -219,7 +247,7 @@ optimize_to_bool( bool insert_mode) { if (sym_matches_type(value, &PyBool_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); *result_ptr = value; return 1; } @@ -229,7 +257,7 @@ optimize_to_bool( int opcode = insert_mode ? _INSERT_1_LOAD_CONST_INLINE_BORROW : _POP_TOP_LOAD_CONST_INLINE_BORROW; - REPLACE_OP(this_instr, opcode, 0, (uintptr_t)load); + ADD_OP(opcode, 0, (uintptr_t)load); *result_ptr = sym_new_const(ctx, load); return 1; } @@ -237,9 +265,9 @@ optimize_to_bool( } static void -eliminate_pop_guard(_PyUOpInstruction *this_instr, bool exit) +eliminate_pop_guard(_PyUOpInstruction *this_instr, JitOptContext *ctx, bool exit) { - REPLACE_OP(this_instr, _POP_TOP, 0, 0); + ADD_OP(_POP_TOP, 0, 0); if (exit) { REPLACE_OP((this_instr+1), _EXIT_TRACE, 0, 0); this_instr[1].target = this_instr->target; @@ -256,7 +284,7 @@ lookup_attr(JitOptContext *ctx, _PyBloomFilter *dependencies, _PyUOpInstruction PyObject *lookup = _PyType_Lookup(type, name); if (lookup) { int opcode = _Py_IsImmortal(lookup) ? immortal : mortal; - REPLACE_OP(this_instr, opcode, 0, (uintptr_t)lookup); + ADD_OP(opcode, 0, (uintptr_t)lookup); PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); _Py_BloomFilter_Add(dependencies, type); return sym_new_const(ctx, lookup); @@ -349,6 +377,8 @@ optimize_uops( JitOptContext *ctx = &tstate->jit_tracer_state->opt_context; uint32_t opcode = UINT16_MAX; + ctx->tracer = tstate->jit_tracer_state; + // Make sure that watchers are set up PyInterpreterState *interp = _PyInterpreterState_GET(); if (interp->dict_state.watchers[GLOBALS_WATCHER_ID] == NULL) { @@ -364,6 +394,7 @@ optimize_uops( frame->func = func; ctx->curr_frame_depth++; ctx->frame = frame; + ctx->tracer->out_len = 0; _PyUOpInstruction *this_instr = NULL; JitOptRef *stack_pointer = ctx->frame->stack_pointer; @@ -395,6 +426,10 @@ optimize_uops( DPRINTF(1, "\nUnknown opcode in abstract interpreter\n"); Py_UNREACHABLE(); } + // If no ADD_OP was called during this iteration, copy the original instruction + if (ctx->tracer->out_len == i) { + ctx->tracer->out_buffer[ctx->tracer->out_len++] = *this_instr; + } assert(ctx->frame != NULL); if (!CURRENT_FRAME_IS_INIT_SHIM()) { DPRINTF(3, " stack_level %d\n", STACK_LEVEL()); @@ -423,7 +458,21 @@ optimize_uops( /* Either reached the end or cannot optimize further, but there * would be no benefit in retrying later */ _Py_uop_abstractcontext_fini(ctx); - return trace_len; + // Check that the trace ends with a proper terminator + if (ctx->tracer->out_len > 0) { + _PyUOpInstruction *last_uop = &ctx->tracer->out_buffer[ctx->tracer->out_len - 1]; + if (!is_terminator_uop(last_uop)) { + // Copy remaining uops from original trace until we find a terminator + for (int i = ctx->tracer->out_len; i < trace_len; i++) { + ctx->tracer->out_buffer[ctx->tracer->out_len++] = trace[i]; + if (is_terminator_uop(&trace[i])) { + break; + } + } + } + } + + return ctx->tracer->out_len; error: DPRINTF(3, "\n"); @@ -595,7 +644,7 @@ _Py_uop_analyze_and_optimize( assert(length > 0); - length = remove_unneeded_uops(buffer, length); + length = remove_unneeded_uops(tstate->jit_tracer_state->out_buffer, length); assert(length > 0); OPT_STAT_INC(optimizer_successes); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index d8868dc020787d..75e28e3a2d000a 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -48,7 +48,7 @@ optimize_to_bool( bool insert_mode); extern void -eliminate_pop_guard(_PyUOpInstruction *this_instr, bool exit); +eliminate_pop_guard(_PyUOpInstruction *this_instr, JitOptContext *ctx, bool exit); extern PyCodeObject *get_code(_PyUOpInstruction *op); @@ -139,11 +139,11 @@ dummy_func(void) { op(_GUARD_TOS_INT, (value -- value)) { if (sym_is_compact_int(value)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } else { if (sym_get_type(value) == &PyLong_Type) { - REPLACE_OP(this_instr, _GUARD_TOS_OVERFLOWED, 0, 0); + ADD_OP(_GUARD_TOS_OVERFLOWED, 0, 0); } sym_set_compact_int(value); } @@ -151,11 +151,11 @@ dummy_func(void) { op(_GUARD_NOS_INT, (left, unused -- left, unused)) { if (sym_is_compact_int(left)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } else { if (sym_get_type(left) == &PyLong_Type) { - REPLACE_OP(this_instr, _GUARD_NOS_OVERFLOWED, 0, 0); + ADD_OP(_GUARD_NOS_OVERFLOWED, 0, 0); } sym_set_compact_int(left); } @@ -165,7 +165,7 @@ dummy_func(void) { PyObject *type = (PyObject *)_PyType_LookupByVersion(type_version); if (type) { if (type == sym_get_const(ctx, owner)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } else { sym_set_const(owner, type); @@ -176,7 +176,7 @@ dummy_func(void) { op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) { assert(type_version); if (sym_matches_type_version(owner, type_version)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } else { // add watcher so that whenever the type changes we invalidate this PyTypeObject *type = _PyType_LookupByVersion(type_version); @@ -198,14 +198,14 @@ dummy_func(void) { op(_GUARD_TOS_FLOAT, (value -- value)) { if (sym_matches_type(value, &PyFloat_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(value, &PyFloat_Type); } op(_GUARD_NOS_FLOAT, (left, unused -- left, unused)) { if (sym_matches_type(left, &PyFloat_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(left, &PyFloat_Type); } @@ -352,7 +352,7 @@ dummy_func(void) { assert(index >= 0); int tuple_length = sym_tuple_length(tuple_st); if (tuple_length != -1 && index < tuple_length) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } } } @@ -419,7 +419,7 @@ dummy_func(void) { op(_GUARD_NOS_UNICODE, (nos, unused -- nos, unused)) { if (sym_matches_type(nos, &PyUnicode_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(nos, &PyUnicode_Type); } @@ -430,7 +430,7 @@ dummy_func(void) { op(_GUARD_TOS_UNICODE, (value -- value)) { if (sym_matches_type(value, &PyUnicode_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(value, &PyUnicode_Type); } @@ -533,7 +533,7 @@ dummy_func(void) { op(_LOAD_CONST, (-- value)) { PyCodeObject *co = get_current_code_object(ctx); PyObject *val = PyTuple_GET_ITEM(co->co_consts, oparg); - REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); value = PyJitRef_Borrow(sym_new_const(ctx, val)); } @@ -541,7 +541,7 @@ dummy_func(void) { PyObject *val = PyLong_FromLong(oparg); assert(val); assert(_Py_IsImmortal(val)); - REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); value = PyJitRef_Borrow(sym_new_const(ctx, val)); } @@ -579,34 +579,34 @@ dummy_func(void) { if (PyJitRef_IsBorrowed(value) || sym_is_immortal(PyJitRef_Unwrap(value)) || sym_is_null(value)) { - REPLACE_OP(this_instr, _POP_TOP_NOP, 0, 0); + ADD_OP(_POP_TOP_NOP, 0, 0); } else if (typ == &PyLong_Type) { - REPLACE_OP(this_instr, _POP_TOP_INT, 0, 0); + ADD_OP(_POP_TOP_INT, 0, 0); } else if (typ == &PyFloat_Type) { - REPLACE_OP(this_instr, _POP_TOP_FLOAT, 0, 0); + ADD_OP(_POP_TOP_FLOAT, 0, 0); } else if (typ == &PyUnicode_Type) { - REPLACE_OP(this_instr, _POP_TOP_UNICODE, 0, 0); + ADD_OP(_POP_TOP_UNICODE, 0, 0); } } op(_POP_TOP_INT, (value --)) { if (PyJitRef_IsBorrowed(value)) { - REPLACE_OP(this_instr, _POP_TOP_NOP, 0, 0); + ADD_OP(_POP_TOP_NOP, 0, 0); } } op(_POP_TOP_FLOAT, (value --)) { if (PyJitRef_IsBorrowed(value)) { - REPLACE_OP(this_instr, _POP_TOP_NOP, 0, 0); + ADD_OP(_POP_TOP_NOP, 0, 0); } } op(_POP_TOP_UNICODE, (value --)) { if (PyJitRef_IsBorrowed(value)) { - REPLACE_OP(this_instr, _POP_TOP_NOP, 0, 0); + ADD_OP(_POP_TOP_NOP, 0, 0); } } @@ -659,11 +659,11 @@ dummy_func(void) { op (_PUSH_NULL_CONDITIONAL, ( -- null[oparg & 1])) { if (oparg & 1) { - REPLACE_OP(this_instr, _PUSH_NULL, 0, 0); + ADD_OP(_PUSH_NULL, 0, 0); null[0] = sym_new_null(ctx); } else { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } } @@ -758,8 +758,8 @@ dummy_func(void) { op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyFunction_Type)) { assert(PyFunction_Check(sym_get_const(ctx, callable))); - REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0, func_version); - this_instr->operand1 = (uintptr_t)sym_get_const(ctx, callable); + ADD_OP(_CHECK_FUNCTION_VERSION_INLINE, 0, func_version); + ctx->tracer->out_buffer[ctx->tracer->out_len - 1].operand1 = (uintptr_t)sym_get_const(ctx, callable); } sym_set_type(callable, &PyFunction_Type); } @@ -768,8 +768,8 @@ dummy_func(void) { if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyMethod_Type)) { PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx, callable); assert(PyMethod_Check(method)); - REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0, func_version); - this_instr->operand1 = (uintptr_t)method->im_func; + ADD_OP(_CHECK_FUNCTION_VERSION_INLINE, 0, func_version); + ctx->tracer->out_buffer[ctx->tracer->out_len - 1].operand1 = (uintptr_t)method->im_func; } sym_set_type(callable, &PyMethod_Type); } @@ -781,7 +781,7 @@ dummy_func(void) { PyFunctionObject *func = (PyFunctionObject *)sym_get_const(ctx, callable); PyCodeObject *co = (PyCodeObject *)func->func_code; if (co->co_argcount == oparg + !sym_is_null(self_or_null)) { - REPLACE_OP(this_instr, _NOP, 0 ,0); + ADD_OP(_NOP, 0 ,0); } } } @@ -1044,7 +1044,7 @@ dummy_func(void) { op(_ITER_CHECK_TUPLE, (iter, null_or_index -- iter, null_or_index)) { if (sym_matches_type(iter, &PyTuple_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(iter, &PyTuple_Type); } @@ -1057,7 +1057,7 @@ dummy_func(void) { PyObject* type = (PyObject *)sym_get_type(arg); if (type) { res = sym_new_const(ctx, type); - REPLACE_OP(this_instr, _SHUFFLE_2_LOAD_CONST_INLINE_BORROW, 0, + ADD_OP(_SHUFFLE_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)type); } else { @@ -1095,7 +1095,7 @@ dummy_func(void) { out = Py_True; } sym_set_const(res, out); - REPLACE_OP(this_instr, _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)out); + ADD_OP(_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)out); } } @@ -1103,7 +1103,7 @@ dummy_func(void) { if (sym_is_const(ctx, flag)) { PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); - eliminate_pop_guard(this_instr, value != Py_True); + eliminate_pop_guard(this_instr, ctx, value != Py_True); } sym_set_const(flag, Py_True); } @@ -1148,7 +1148,7 @@ dummy_func(void) { if (sym_is_const(ctx, flag)) { PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); - eliminate_pop_guard(this_instr, value != Py_False); + eliminate_pop_guard(this_instr, ctx, value != Py_False); } sym_set_const(flag, Py_False); } @@ -1157,11 +1157,11 @@ dummy_func(void) { if (sym_is_const(ctx, val)) { PyObject *value = sym_get_const(ctx, val); assert(value != NULL); - eliminate_pop_guard(this_instr, !Py_IsNone(value)); + eliminate_pop_guard(this_instr, ctx, !Py_IsNone(value)); } else if (sym_has_type(val)) { assert(!sym_matches_type(val, &_PyNone_Type)); - eliminate_pop_guard(this_instr, true); + eliminate_pop_guard(this_instr, ctx, true); } sym_set_const(val, Py_None); } @@ -1170,11 +1170,11 @@ dummy_func(void) { if (sym_is_const(ctx, val)) { PyObject *value = sym_get_const(ctx, val); assert(value != NULL); - eliminate_pop_guard(this_instr, Py_IsNone(value)); + eliminate_pop_guard(this_instr, ctx, Py_IsNone(value)); } else if (sym_has_type(val)) { assert(!sym_matches_type(val, &_PyNone_Type)); - eliminate_pop_guard(this_instr, false); + eliminate_pop_guard(this_instr, ctx, false); } } @@ -1182,7 +1182,7 @@ dummy_func(void) { /* Setting the eval frame function invalidates * all executors, so no need to check dynamically */ if (_PyInterpreterState_GET()->eval_frame == NULL) { - REPLACE_OP(this_instr, _NOP, 0 ,0); + ADD_OP(_NOP, 0 ,0); } } @@ -1210,7 +1210,7 @@ dummy_func(void) { } op(_REPLACE_WITH_TRUE, (value -- res, v)) { - REPLACE_OP(this_instr, _INSERT_1_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); + ADD_OP(_INSERT_1_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); res = sym_new_const(ctx, Py_True); v = value; } @@ -1265,42 +1265,42 @@ dummy_func(void) { op(_GUARD_TOS_LIST, (tos -- tos)) { if (sym_matches_type(tos, &PyList_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(tos, &PyList_Type); } op(_GUARD_NOS_LIST, (nos, unused -- nos, unused)) { if (sym_matches_type(nos, &PyList_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(nos, &PyList_Type); } op(_GUARD_TOS_TUPLE, (tos -- tos)) { if (sym_matches_type(tos, &PyTuple_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(tos, &PyTuple_Type); } op(_GUARD_NOS_TUPLE, (nos, unused -- nos, unused)) { if (sym_matches_type(nos, &PyTuple_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(nos, &PyTuple_Type); } op(_GUARD_TOS_DICT, (tos -- tos)) { if (sym_matches_type(tos, &PyDict_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(tos, &PyDict_Type); } op(_GUARD_NOS_DICT, (nos, unused -- nos, unused)) { if (sym_matches_type(nos, &PyDict_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(nos, &PyDict_Type); } @@ -1309,48 +1309,48 @@ dummy_func(void) { if (sym_matches_type(tos, &PySet_Type) || sym_matches_type(tos, &PyFrozenSet_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } } op(_GUARD_NOS_NULL, (null, unused -- null, unused)) { if (sym_is_null(null)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_null(null); } op(_GUARD_NOS_NOT_NULL, (nos, unused -- nos, unused)) { if (sym_is_not_null(nos)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_non_null(nos); } op(_GUARD_THIRD_NULL, (null, unused, unused -- null, unused, unused)) { if (sym_is_null(null)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_null(null); } op(_GUARD_CALLABLE_TYPE_1, (callable, unused, unused -- callable, unused, unused)) { if (sym_get_const(ctx, callable) == (PyObject *)&PyType_Type) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_const(callable, (PyObject *)&PyType_Type); } op(_GUARD_CALLABLE_TUPLE_1, (callable, unused, unused -- callable, unused, unused)) { if (sym_get_const(ctx, callable) == (PyObject *)&PyTuple_Type) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_const(callable, (PyObject *)&PyTuple_Type); } op(_GUARD_CALLABLE_STR_1, (callable, unused, unused -- callable, unused, unused)) { if (sym_get_const(ctx, callable) == (PyObject *)&PyUnicode_Type) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_const(callable, (PyObject *)&PyUnicode_Type); } @@ -1364,7 +1364,7 @@ dummy_func(void) { goto error; } if (_Py_IsImmortal(temp)) { - REPLACE_OP(this_instr, _SHUFFLE_3_LOAD_CONST_INLINE_BORROW, + ADD_OP(_SHUFFLE_3_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); } res = sym_new_const(ctx, temp); @@ -1386,7 +1386,7 @@ dummy_func(void) { goto error; } if (_Py_IsImmortal(temp)) { - REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); } len = sym_new_const(ctx, temp); Py_DECREF(temp); @@ -1396,7 +1396,7 @@ dummy_func(void) { op(_GUARD_CALLABLE_LEN, (callable, unused, unused -- callable, unused, unused)) { PyObject *len = _PyInterpreterState_GET()->callable_cache.len; if (sym_get_const(ctx, callable) == len) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_const(callable, len); } @@ -1404,7 +1404,7 @@ dummy_func(void) { op(_GUARD_CALLABLE_ISINSTANCE, (callable, unused, unused, unused -- callable, unused, unused, unused)) { PyObject *isinstance = _PyInterpreterState_GET()->callable_cache.isinstance; if (sym_get_const(ctx, callable) == isinstance) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_const(callable, isinstance); } @@ -1412,7 +1412,7 @@ dummy_func(void) { op(_GUARD_CALLABLE_LIST_APPEND, (callable, unused, unused -- callable, unused, unused)) { PyObject *list_append = _PyInterpreterState_GET()->callable_cache.list_append; if (sym_get_const(ctx, callable) == list_append) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_const(callable, list_append); } @@ -1448,7 +1448,7 @@ dummy_func(void) { ctx->frame->globals_watched = true; } if (ctx->frame->globals_checked_version == version) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } } } @@ -1509,7 +1509,7 @@ dummy_func(void) { ctx->frame->globals_watched = true; } if (ctx->frame->globals_checked_version != version && this_instr[-1].opcode == _NOP) { - REPLACE_OP(this_instr-1, _GUARD_GLOBALS_VERSION, 0, version); + REPLACE_OP(&ctx->tracer->out_buffer[ctx->tracer->out_len - 1], _GUARD_GLOBALS_VERSION, 0, version); ctx->frame->globals_checked_version = version; } if (ctx->frame->globals_checked_version == version) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 845d2f235aeaf8..8f721507b464de 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -76,7 +76,7 @@ JitOptRef value; PyCodeObject *co = get_current_code_object(ctx); PyObject *val = PyTuple_GET_ITEM(co->co_consts, oparg); - REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); value = PyJitRef_Borrow(sym_new_const(ctx, val)); CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; @@ -90,7 +90,7 @@ PyObject *val = PyLong_FromLong(oparg); assert(val); assert(_Py_IsImmortal(val)); - REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); value = PyJitRef_Borrow(sym_new_const(ctx, val)); CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; @@ -117,16 +117,16 @@ if (PyJitRef_IsBorrowed(value) || sym_is_immortal(PyJitRef_Unwrap(value)) || sym_is_null(value)) { - REPLACE_OP(this_instr, _POP_TOP_NOP, 0, 0); + ADD_OP(_POP_TOP_NOP, 0, 0); } else if (typ == &PyLong_Type) { - REPLACE_OP(this_instr, _POP_TOP_INT, 0, 0); + ADD_OP(_POP_TOP_INT, 0, 0); } else if (typ == &PyFloat_Type) { - REPLACE_OP(this_instr, _POP_TOP_FLOAT, 0, 0); + ADD_OP(_POP_TOP_FLOAT, 0, 0); } else if (typ == &PyUnicode_Type) { - REPLACE_OP(this_instr, _POP_TOP_UNICODE, 0, 0); + ADD_OP(_POP_TOP_UNICODE, 0, 0); } CHECK_STACK_BOUNDS(-1); stack_pointer += -1; @@ -145,7 +145,7 @@ JitOptRef value; value = stack_pointer[-1]; if (PyJitRef_IsBorrowed(value)) { - REPLACE_OP(this_instr, _POP_TOP_NOP, 0, 0); + ADD_OP(_POP_TOP_NOP, 0, 0); } CHECK_STACK_BOUNDS(-1); stack_pointer += -1; @@ -157,7 +157,7 @@ JitOptRef value; value = stack_pointer[-1]; if (PyJitRef_IsBorrowed(value)) { - REPLACE_OP(this_instr, _POP_TOP_NOP, 0, 0); + ADD_OP(_POP_TOP_NOP, 0, 0); } CHECK_STACK_BOUNDS(-1); stack_pointer += -1; @@ -169,7 +169,7 @@ JitOptRef value; value = stack_pointer[-1]; if (PyJitRef_IsBorrowed(value)) { - REPLACE_OP(this_instr, _POP_TOP_NOP, 0, 0); + ADD_OP(_POP_TOP_NOP, 0, 0); } CHECK_STACK_BOUNDS(-1); stack_pointer += -1; @@ -241,7 +241,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result - REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } stack_pointer[-1] = res; @@ -283,7 +283,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result - REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } stack_pointer[-1] = res; @@ -335,7 +335,7 @@ JitOptRef nos; nos = stack_pointer[-2]; if (sym_matches_type(nos, &PyList_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(nos, &PyList_Type); break; @@ -345,7 +345,7 @@ JitOptRef tos; tos = stack_pointer[-1]; if (sym_matches_type(tos, &PyList_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(tos, &PyList_Type); break; @@ -391,7 +391,7 @@ JitOptRef nos; nos = stack_pointer[-2]; if (sym_matches_type(nos, &PyUnicode_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(nos, &PyUnicode_Type); break; @@ -401,7 +401,7 @@ JitOptRef value; value = stack_pointer[-1]; if (sym_matches_type(value, &PyUnicode_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(value, &PyUnicode_Type); break; @@ -430,7 +430,7 @@ JitOptRef res; JitOptRef v; value = stack_pointer[-1]; - REPLACE_OP(this_instr, _INSERT_1_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); + ADD_OP(_INSERT_1_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); res = sym_new_const(ctx, Py_True); v = value; CHECK_STACK_BOUNDS(1); @@ -465,7 +465,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result - REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } stack_pointer[-1] = res; @@ -486,11 +486,11 @@ JitOptRef left; left = stack_pointer[-2]; if (sym_is_compact_int(left)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } else { if (sym_get_type(left) == &PyLong_Type) { - REPLACE_OP(this_instr, _GUARD_NOS_OVERFLOWED, 0, 0); + ADD_OP(_GUARD_NOS_OVERFLOWED, 0, 0); } sym_set_compact_int(left); } @@ -501,11 +501,11 @@ JitOptRef value; value = stack_pointer[-1]; if (sym_is_compact_int(value)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } else { if (sym_get_type(value) == &PyLong_Type) { - REPLACE_OP(this_instr, _GUARD_TOS_OVERFLOWED, 0, 0); + ADD_OP(_GUARD_TOS_OVERFLOWED, 0, 0); } sym_set_compact_int(value); } @@ -564,7 +564,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } CHECK_STACK_BOUNDS(1); @@ -628,7 +628,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } CHECK_STACK_BOUNDS(1); @@ -692,7 +692,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } CHECK_STACK_BOUNDS(1); @@ -716,7 +716,7 @@ JitOptRef left; left = stack_pointer[-2]; if (sym_matches_type(left, &PyFloat_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(left, &PyFloat_Type); break; @@ -726,7 +726,7 @@ JitOptRef value; value = stack_pointer[-1]; if (sym_matches_type(value, &PyFloat_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(value, &PyFloat_Type); break; @@ -957,7 +957,7 @@ JitOptRef nos; nos = stack_pointer[-2]; if (sym_matches_type(nos, &PyTuple_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(nos, &PyTuple_Type); break; @@ -967,7 +967,7 @@ JitOptRef tos; tos = stack_pointer[-1]; if (sym_matches_type(tos, &PyTuple_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(tos, &PyTuple_Type); break; @@ -985,7 +985,7 @@ assert(index >= 0); int tuple_length = sym_tuple_length(tuple_st); if (tuple_length != -1 && index < tuple_length) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } } break; @@ -1031,7 +1031,7 @@ JitOptRef nos; nos = stack_pointer[-2]; if (sym_matches_type(nos, &PyDict_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(nos, &PyDict_Type); break; @@ -1041,7 +1041,7 @@ JitOptRef tos; tos = stack_pointer[-1]; if (sym_matches_type(tos, &PyDict_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(tos, &PyDict_Type); break; @@ -1438,11 +1438,11 @@ JitOptRef *null; null = &stack_pointer[0]; if (oparg & 1) { - REPLACE_OP(this_instr, _PUSH_NULL, 0, 0); + ADD_OP(_PUSH_NULL, 0, 0); null[0] = sym_new_null(ctx); } else { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } CHECK_STACK_BOUNDS((oparg & 1)); stack_pointer += (oparg & 1); @@ -1467,7 +1467,7 @@ ctx->frame->globals_watched = true; } if (ctx->frame->globals_checked_version == version) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } } } @@ -1496,7 +1496,7 @@ ctx->frame->globals_watched = true; } if (ctx->frame->globals_checked_version != version && this_instr[-1].opcode == _NOP) { - REPLACE_OP(this_instr-1, _GUARD_GLOBALS_VERSION, 0, version); + REPLACE_OP(&ctx->tracer->out_buffer[ctx->tracer->out_len - 1], _GUARD_GLOBALS_VERSION, 0, version); ctx->frame->globals_checked_version = version; } if (ctx->frame->globals_checked_version == version) { @@ -1762,7 +1762,7 @@ uint32_t type_version = (uint32_t)this_instr->operand0; assert(type_version); if (sym_matches_type_version(owner, type_version)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } else { PyTypeObject *type = _PyType_LookupByVersion(type_version); if (type) { @@ -1876,7 +1876,7 @@ PyObject *type = (PyObject *)_PyType_LookupByVersion(type_version); if (type) { if (type == sym_get_const(ctx, owner)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } else { sym_set_const(owner, type); @@ -2005,7 +2005,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } CHECK_STACK_BOUNDS(-1); @@ -2067,7 +2067,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } CHECK_STACK_BOUNDS(1); @@ -2131,7 +2131,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } CHECK_STACK_BOUNDS(1); @@ -2193,7 +2193,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } CHECK_STACK_BOUNDS(1); @@ -2262,7 +2262,7 @@ PyObject *result = sym_get_const(ctx, b); if (_Py_IsImmortal(result)) { // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } CHECK_STACK_BOUNDS(-1); @@ -2285,7 +2285,7 @@ if (sym_matches_type(tos, &PySet_Type) || sym_matches_type(tos, &PyFrozenSet_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } break; } @@ -2375,7 +2375,7 @@ goto error; } if (_Py_IsImmortal(temp)) { - REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); } len = sym_new_const(ctx, temp); CHECK_STACK_BOUNDS(1); @@ -2500,7 +2500,7 @@ JitOptRef iter; iter = stack_pointer[-2]; if (sym_matches_type(iter, &PyTuple_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(iter, &PyTuple_Type); break; @@ -2754,8 +2754,8 @@ uint32_t func_version = (uint32_t)this_instr->operand0; if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyFunction_Type)) { assert(PyFunction_Check(sym_get_const(ctx, callable))); - REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0, func_version); - this_instr->operand1 = (uintptr_t)sym_get_const(ctx, callable); + ADD_OP(_CHECK_FUNCTION_VERSION_INLINE, 0, func_version); + ctx->tracer->out_buffer[ctx->tracer->out_len - 1].operand1 = (uintptr_t)sym_get_const(ctx, callable); } sym_set_type(callable, &PyFunction_Type); break; @@ -2772,8 +2772,8 @@ if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyMethod_Type)) { PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx, callable); assert(PyMethod_Check(method)); - REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0, func_version); - this_instr->operand1 = (uintptr_t)method->im_func; + ADD_OP(_CHECK_FUNCTION_VERSION_INLINE, 0, func_version); + ctx->tracer->out_buffer[ctx->tracer->out_len - 1].operand1 = (uintptr_t)method->im_func; } sym_set_type(callable, &PyMethod_Type); break; @@ -2821,7 +2821,7 @@ case _CHECK_PEP_523: { if (_PyInterpreterState_GET()->eval_frame == NULL) { - REPLACE_OP(this_instr, _NOP, 0 ,0); + ADD_OP(_NOP, 0 ,0); } break; } @@ -2837,7 +2837,7 @@ PyFunctionObject *func = (PyFunctionObject *)sym_get_const(ctx, callable); PyCodeObject *co = (PyCodeObject *)func->func_code; if (co->co_argcount == oparg + !sym_is_null(self_or_null)) { - REPLACE_OP(this_instr, _NOP, 0 ,0); + ADD_OP(_NOP, 0 ,0); } } } @@ -2915,7 +2915,7 @@ JitOptRef null; null = stack_pointer[-2]; if (sym_is_null(null)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_null(null); break; @@ -2925,7 +2925,7 @@ JitOptRef nos; nos = stack_pointer[-2]; if (sym_is_not_null(nos)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_non_null(nos); break; @@ -2935,7 +2935,7 @@ JitOptRef null; null = stack_pointer[-3]; if (sym_is_null(null)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_null(null); break; @@ -2945,7 +2945,7 @@ JitOptRef callable; callable = stack_pointer[-3]; if (sym_get_const(ctx, callable) == (PyObject *)&PyType_Type) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_const(callable, (PyObject *)&PyType_Type); break; @@ -2959,8 +2959,8 @@ PyObject* type = (PyObject *)sym_get_type(arg); if (type) { res = sym_new_const(ctx, type); - REPLACE_OP(this_instr, _SHUFFLE_2_LOAD_CONST_INLINE_BORROW, 0, - (uintptr_t)type); + ADD_OP(_SHUFFLE_2_LOAD_CONST_INLINE_BORROW, 0, + (uintptr_t)type); } else { res = sym_new_not_null(ctx); @@ -2978,7 +2978,7 @@ JitOptRef callable; callable = stack_pointer[-3]; if (sym_get_const(ctx, callable) == (PyObject *)&PyUnicode_Type) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_const(callable, (PyObject *)&PyUnicode_Type); break; @@ -3008,7 +3008,7 @@ JitOptRef callable; callable = stack_pointer[-3]; if (sym_get_const(ctx, callable) == (PyObject *)&PyTuple_Type) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_const(callable, (PyObject *)&PyTuple_Type); break; @@ -3150,7 +3150,7 @@ callable = stack_pointer[-3]; PyObject *len = _PyInterpreterState_GET()->callable_cache.len; if (sym_get_const(ctx, callable) == len) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_const(callable, len); break; @@ -3172,8 +3172,8 @@ goto error; } if (_Py_IsImmortal(temp)) { - REPLACE_OP(this_instr, _SHUFFLE_3_LOAD_CONST_INLINE_BORROW, - 0, (uintptr_t)temp); + ADD_OP(_SHUFFLE_3_LOAD_CONST_INLINE_BORROW, + 0, (uintptr_t)temp); } res = sym_new_const(ctx, temp); CHECK_STACK_BOUNDS(-2); @@ -3196,7 +3196,7 @@ callable = stack_pointer[-4]; PyObject *isinstance = _PyInterpreterState_GET()->callable_cache.isinstance; if (sym_get_const(ctx, callable) == isinstance) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_const(callable, isinstance); break; @@ -3217,7 +3217,7 @@ out = Py_True; } sym_set_const(res, out); - REPLACE_OP(this_instr, _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)out); + ADD_OP(_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)out); } CHECK_STACK_BOUNDS(-3); stack_pointer[-4] = res; @@ -3231,7 +3231,7 @@ callable = stack_pointer[-3]; PyObject *list_append = _PyInterpreterState_GET()->callable_cache.list_append; if (sym_get_const(ctx, callable) == list_append) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_const(callable, list_append); break; @@ -3525,7 +3525,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } CHECK_STACK_BOUNDS(-1); @@ -3612,7 +3612,7 @@ if (sym_is_const(ctx, flag)) { PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); - eliminate_pop_guard(this_instr, value != Py_True); + eliminate_pop_guard(this_instr, ctx, value != Py_True); } sym_set_const(flag, Py_True); CHECK_STACK_BOUNDS(-1); @@ -3627,7 +3627,7 @@ if (sym_is_const(ctx, flag)) { PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); - eliminate_pop_guard(this_instr, value != Py_False); + eliminate_pop_guard(this_instr, ctx, value != Py_False); } sym_set_const(flag, Py_False); CHECK_STACK_BOUNDS(-1); @@ -3642,11 +3642,11 @@ if (sym_is_const(ctx, val)) { PyObject *value = sym_get_const(ctx, val); assert(value != NULL); - eliminate_pop_guard(this_instr, !Py_IsNone(value)); + eliminate_pop_guard(this_instr, ctx, !Py_IsNone(value)); } else if (sym_has_type(val)) { assert(!sym_matches_type(val, &_PyNone_Type)); - eliminate_pop_guard(this_instr, true); + eliminate_pop_guard(this_instr, ctx, true); } sym_set_const(val, Py_None); CHECK_STACK_BOUNDS(-1); @@ -3661,11 +3661,11 @@ if (sym_is_const(ctx, val)) { PyObject *value = sym_get_const(ctx, val); assert(value != NULL); - eliminate_pop_guard(this_instr, Py_IsNone(value)); + eliminate_pop_guard(this_instr, ctx, Py_IsNone(value)); } else if (sym_has_type(val)) { assert(!sym_matches_type(val, &_PyNone_Type)); - eliminate_pop_guard(this_instr, false); + eliminate_pop_guard(this_instr, ctx, false); } CHECK_STACK_BOUNDS(-1); stack_pointer += -1; diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index 922ed803c93fb5..56560ba88e47b4 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -270,7 +270,7 @@ def replace_opcode_if_evaluates_pure( emitter.emit(f"PyObject *result = sym_get_const(ctx, {output_identifier.text});\n") emitter.emit(f"if (_Py_IsImmortal(result)) {{\n") emitter.emit(f"// Replace with {replacement_uop} since we have {input_desc} and an immortal result\n") - emitter.emit(f"REPLACE_OP(this_instr, {replacement_uop}, 0, (uintptr_t)result);\n") + emitter.emit(f"ADD_OP({replacement_uop}, 0, (uintptr_t)result);\n") emitter.emit("}\n") emitter.emit("}\n")