hailin
/
php-8.0.30-src
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
php-8.0.30-src/ext/opcache/jit/zend_jit_x86.dasc

16189 lines
455 KiB
Plaintext
Raw Blame History

/*
* +----------------------------------------------------------------------+
* | Zend JIT |
* +----------------------------------------------------------------------+
* | Copyright (c) The PHP Group |
* +----------------------------------------------------------------------+
* | This source file is subject to version 3.01 of the PHP license, |
* | that is bundled with this package in the file LICENSE, and is |
* | available through the world-wide-web at the following url: |
* | http://www.php.net/license/3_01.txt |
* | If you did not receive a copy of the PHP license and are unable to |
* | obtain it through the world-wide-web, please send a note to |
* | license@php.net so we can mail you a copy immediately. |
* +----------------------------------------------------------------------+
* | Authors: Dmitry Stogov <dmitry@php.net> |
* | Xinchen Hui <laruence@php.net> |
* +----------------------------------------------------------------------+
*/
|.if X64
|.arch x64
|.else
|.arch x86
|.endif
|.if X64WIN
|.define FP, r14
|.define IP, r15
|.define IPl, r15d
|.define RX, r15 // the same as VM IP reused as a general purpose reg
|.define CARG1, rcx // x64/POSIX C call arguments.
|.define CARG2, rdx
|.define CARG3, r8
|.define CARG4, r9
|.define CARG1d, ecx
|.define CARG2d, edx
|.define CARG3d, r8d
|.define CARG4d, r9d
|.define FCARG1a, CARG1 // Simulate x86 fastcall.
|.define FCARG2a, CARG2
|.define FCARG1d, CARG1d
|.define FCARG2d, CARG2d
|.define SPAD, 0x58 // padding for CPU stack alignment
|.define NR_SPAD, 0x58 // padding for CPU stack alignment
|.define T3, [r4+0x50] // Used to store old value of IP
|.define T2, [r4+0x48] // Used to store old value of FP
|.define T1, [r4+0x40]
|.define A6, [r4+0x28] // preallocated slot for 6-th argument
|.define A5, [r4+0x20] // preallocated slot for 5-th argument
|.elif X64
|.define FP, r14
|.define IP, r15
|.define IPl, r15d
|.define RX, r15 // the same as VM IP reused as a general purpose reg
|.define CARG1, rdi // x64/POSIX C call arguments.
|.define CARG2, rsi
|.define CARG3, rdx
|.define CARG4, rcx
|.define CARG5, r8
|.define CARG6, r9
|.define CARG1d, edi
|.define CARG2d, esi
|.define CARG3d, edx
|.define CARG4d, ecx
|.define CARG5d, r8d
|.define CARG6d, r9d
|.define FCARG1a, CARG1 // Simulate x86 fastcall.
|.define FCARG2a, CARG2
|.define FCARG1d, CARG1d
|.define FCARG2d, CARG2d
|.define SPAD, 0x18 // padding for CPU stack alignment
|.define NR_SPAD, 0x28 // padding for CPU stack alignment
|.define T3, [r4+0x20] // Used to store old value of IP (CALL VM only)
|.define T2, [r4+0x18] // Used to store old value of FP (CALL VM only)
|.define T1, [r4]
|.else
|.define FP, esi
|.define IP, edi
|.define IPl, edi
|.define RX, edi // the same as VM IP reused as a general purpose reg
|.define FCARG1a, ecx // x86 fastcall arguments.
|.define FCARG2a, edx
|.define FCARG1d, ecx
|.define FCARG2d, edx
|.define SPAD, 0x1c // padding for CPU stack alignment
|.define NR_SPAD, 0x1c // padding for CPU stack alignment
|.define T3, [r4+0x18] // Used to store old value of IP (CALL VM only)
|.define T2, [r4+0x14] // Used to store old value of FP (CALL VM only)
|.define T1, [r4]
|.define A4, [r4+0xC] // preallocated slots for arguments of "cdecl" functions (intersect with T1)
|.define A3, [r4+0x8]
|.define A2, [r4+0x4]
|.define A1, [r4]
|.endif
|.define HYBRID_SPAD, 16 // padding for stack alignment
#ifdef _WIN64
# define TMP_ZVAL_OFFSET 0x20
#else
# define TMP_ZVAL_OFFSET 0
#endif
#define DASM_ALIGNMENT 16
/* According to x86 and x86_64 ABI, CPU stack has to be 16 byte aligned to
* guarantee proper alignment of 128-bit SSE data allocated on stack.
* With broken alignment any execution of SSE code, including calls to
* memcpy() and others, may lead to crash.
*/
#include "Zend/zend_cpuinfo.h"
#include "jit/zend_jit_x86.h"
#ifdef HAVE_VALGRIND
# include <valgrind/valgrind.h>
#endif
/* The generated code may contain tautological comparisons, ignore them. */
#if defined(__clang__)
# pragma clang diagnostic push
# pragma clang diagnostic ignored "-Wtautological-compare"
# pragma clang diagnostic ignored "-Wstring-compare"
#endif
const char* zend_reg_name[] = {
#if defined(__x86_64__) || defined(_M_X64)
"rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi",
"r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15",
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7",
"xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15"
#else
"rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi",
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"
#endif
};
#ifdef HAVE_GCC_GLOBAL_REGS
# define GCC_GLOBAL_REGS 1
#else
# define GCC_GLOBAL_REGS 0
#endif
#if ZTS
static size_t tsrm_ls_cache_tcb_offset = 0;
static size_t tsrm_tls_index;
static size_t tsrm_tls_offset;
#endif
/* By default avoid JITing inline handlers if it does not seem profitable due to lack of
* type information. Disabling this option allows testing some JIT handlers in the
* presence of try/catch blocks, which prevent SSA construction. */
#ifndef PROFITABILITY_CHECKS
# define PROFITABILITY_CHECKS 1
#endif
|.type EX, zend_execute_data, FP
|.type OP, zend_op
|.type ZVAL, zval
|.actionlist dasm_actions
|.globals zend_lb
static void* dasm_labels[zend_lb_MAX];
|.section code, cold_code, jmp_table
#define IS_32BIT(addr) (((uintptr_t)(addr)) <= 0x7fffffff)
#define IS_SIGNED_32BIT(val) ((((intptr_t)(val)) <= 0x7fffffff) && (((intptr_t)(val)) >= (-2147483647 - 1)))
#define BP_JIT_IS 6
#define CAN_USE_AVX() (JIT_G(opt_flags) & allowed_opt_flags & ZEND_JIT_CPU_AVX)
|.macro ADD_HYBRID_SPAD
||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
| add r4, HYBRID_SPAD
||#endif
|.endmacro
|.macro SUB_HYBRID_SPAD
||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
| sub r4, HYBRID_SPAD
||#endif
|.endmacro
|.macro LOAD_ADDR, reg, addr
| .if X64
|| if (IS_SIGNED_32BIT(addr)) {
| mov reg, ((ptrdiff_t)addr) // 0x48 0xc7 0xc0 <imm-32-bit>
|| } else {
| mov64 reg, ((ptrdiff_t)addr) // 0x48 0xb8 <imm-64-bit>
|| }
| .else
| mov reg, ((ptrdiff_t)addr)
| .endif
|.endmacro
|.macro LOAD_TSRM_CACHE, reg
| .if X64WIN
| gs
| mov reg, aword [0x58]
| mov reg, aword [reg+tsrm_tls_index]
| mov reg, aword [reg+tsrm_tls_offset]
| .elif WIN
| fs
| mov reg, aword [0x2c]
| mov reg, aword [reg+tsrm_tls_index]
| mov reg, aword [reg+tsrm_tls_offset]
| .elif X64APPLE
| gs
|| if (tsrm_ls_cache_tcb_offset) {
| mov reg, aword [tsrm_ls_cache_tcb_offset]
|| } else {
| mov reg, aword [tsrm_tls_index]
| mov reg, aword [reg+tsrm_tls_offset]
|| }
| .elif X64
| fs
|| if (tsrm_ls_cache_tcb_offset) {
| mov reg, aword [tsrm_ls_cache_tcb_offset]
|| } else {
| mov reg, [0x8]
| mov reg, aword [reg+tsrm_tls_index]
| mov reg, aword [reg+tsrm_tls_offset]
|| }
| .else
| gs
|| if (tsrm_ls_cache_tcb_offset) {
| mov reg, aword [tsrm_ls_cache_tcb_offset]
|| } else {
| mov reg, [0x4]
| mov reg, aword [reg+tsrm_tls_index]
| mov reg, aword [reg+tsrm_tls_offset]
|| }
| .endif
|.endmacro
|.macro LOAD_ADDR_ZTS, reg, struct, field
| .if ZTS
| LOAD_TSRM_CACHE reg
| lea reg, aword [reg + (struct.._offset + offsetof(zend_..struct, field))]
| .else
| LOAD_ADDR reg, &struct.field
| .endif
|.endmacro
|.macro ADDR_OP1, addr_ins, addr, tmp_reg
| .if X64
|| if (IS_SIGNED_32BIT(addr)) {
| addr_ins ((ptrdiff_t)addr)
|| } else {
| mov64 tmp_reg, ((ptrdiff_t)addr)
| addr_ins tmp_reg
|| }
| .else
| addr_ins ((ptrdiff_t)addr)
| .endif
|.endmacro
|.macro ADDR_OP2_2, addr_ins, op1, addr, tmp_reg
| .if X64
|| if (IS_SIGNED_32BIT(addr)) {
| addr_ins op1, ((ptrdiff_t)addr)
|| } else {
| mov64 tmp_reg, ((ptrdiff_t)addr)
| addr_ins op1, tmp_reg
|| }
| .else
| addr_ins op1, ((ptrdiff_t)addr)
| .endif
|.endmacro
|.macro PUSH_ADDR, addr, tmp_reg
| ADDR_OP1 push, addr, tmp_reg
|.endmacro
|.macro PUSH_ADDR_ZTS, struct, field, tmp_reg
| .if ZTS
| LOAD_TSRM_CACHE tmp_reg
| lea tmp_reg, aword [tmp_reg + (struct.._offset + offsetof(zend_..struct, field))]
| push tmp_reg
| .else
| ADDR_OP1 push, &struct.field, tmp_reg
| .endif
|.endmacro
|.macro MEM_OP1, mem_ins, prefix, addr, tmp_reg
| .if X64
|| if (IS_SIGNED_32BIT(addr)) {
| mem_ins prefix [addr]
|| } else {
| mov64 tmp_reg, ((ptrdiff_t)addr)
| mem_ins prefix [tmp_reg]
|| }
| .else
| mem_ins prefix [addr]
| .endif
|.endmacro
|.macro MEM_OP2_1, mem_ins, prefix, addr, op2, tmp_reg
| .if X64
|| if (IS_SIGNED_32BIT(addr)) {
| mem_ins prefix [addr], op2
|| } else {
| mov64 tmp_reg, ((ptrdiff_t)addr)
| mem_ins prefix [tmp_reg], op2
|| }
| .else
| mem_ins prefix [addr], op2
| .endif
|.endmacro
|.macro MEM_OP2_2, mem_ins, op1, prefix, addr, tmp_reg
| .if X64
|| if (IS_SIGNED_32BIT(addr)) {
| mem_ins op1, prefix [addr]
|| } else {
| mov64 tmp_reg, ((ptrdiff_t)addr)
| mem_ins op1, prefix [tmp_reg]
|| }
| .else
| mem_ins op1, prefix [addr]
| .endif
|.endmacro
|.macro MEM_OP2_1_ZTS, mem_ins, prefix, struct, field, op2, tmp_reg
| .if ZTS
| LOAD_TSRM_CACHE tmp_reg
| mem_ins prefix [tmp_reg+(struct.._offset+offsetof(zend_..struct, field))], op2
| .else
| MEM_OP2_1 mem_ins, prefix, &struct.field, op2, tmp_reg
| .endif
|.endmacro
|.macro MEM_OP2_2_ZTS, mem_ins, op1, prefix, struct, field, tmp_reg
| .if ZTS
| LOAD_TSRM_CACHE tmp_reg
| mem_ins op1, prefix [tmp_reg+(struct.._offset+offsetof(zend_..struct, field))]
| .else
| MEM_OP2_2 mem_ins, op1, prefix, &struct.field, tmp_reg
| .endif
|.endmacro
|.macro MEM_OP3_3, mem_ins, op1, op2, prefix, addr, tmp_reg
| .if X64
|| if (IS_SIGNED_32BIT(addr)) {
| mem_ins op1, op2, prefix [addr]
|| } else {
| mov64 tmp_reg, ((ptrdiff_t)addr)
| mem_ins op1, op2, prefix [tmp_reg]
|| }
| .else
| mem_ins op1, op2, prefix [addr]
| .endif
|.endmacro
|.macro LOAD_BASE_ADDR, reg, base, offset
|| if (offset) {
| lea reg, qword [Ra(base)+offset]
|| } else {
| mov reg, Ra(base)
|| }
|.endmacro
|.macro PUSH_BASE_ADDR, base, offset, tmp_reg
|| if (offset) {
| lea tmp_reg, qword [Ra(base)+offset]
| push tmp_reg
|| } else {
| push Ra(base)
|| }
|.endmacro
|.macro EXT_CALL, func, tmp_reg
| .if X64
|| if (IS_32BIT(dasm_end) && IS_32BIT(func)) {
| call qword &func
|| } else {
| LOAD_ADDR tmp_reg, func
| call tmp_reg
|| }
| .else
| call dword &func
| .endif
|.endmacro
|.macro EXT_JMP, func, tmp_reg
| .if X64
|| if (IS_32BIT(dasm_end) && IS_32BIT(func)) {
| jmp qword &func
|| } else {
| LOAD_ADDR tmp_reg, func
| jmp tmp_reg
|| }
| .else
| jmp dword &func
| .endif
|.endmacro
|.macro SAVE_IP
|| if (GCC_GLOBAL_REGS) {
| mov aword EX->opline, IP
|| }
|.endmacro
|.macro LOAD_IP
|| if (GCC_GLOBAL_REGS) {
| mov IP, aword EX->opline
|| }
|.endmacro
|.macro LOAD_IP_ADDR, addr
|| if (GCC_GLOBAL_REGS) {
| LOAD_ADDR IP, addr
|| } else {
| ADDR_OP2_2 mov, aword EX->opline, addr, RX
|| }
|.endmacro
|.macro LOAD_IP_ADDR_ZTS, struct, field
| .if ZTS
|| if (GCC_GLOBAL_REGS) {
| LOAD_TSRM_CACHE IP
| mov IP, aword [IP + (struct.._offset + offsetof(zend_..struct, field))]
|| } else {
| LOAD_TSRM_CACHE RX
| lea RX, aword [RX + (struct.._offset + offsetof(zend_..struct, field))]
| mov aword EX->opline, RX
|| }
| .else
| LOAD_IP_ADDR &struct.field
| .endif
|.endmacro
|.macro GET_IP, reg
|| if (GCC_GLOBAL_REGS) {
| mov reg, IP
|| } else {
| mov reg, aword EX->opline
|| }
|.endmacro
|.macro ADD_IP, val
|| if (GCC_GLOBAL_REGS) {
| add IP, val
|| } else {
| add aword EX->opline, val
|| }
|.endmacro
|.macro JMP_IP
|| if (GCC_GLOBAL_REGS) {
| jmp aword [IP]
|| } else {
| mov r0, aword EX:FCARG1a->opline
| jmp aword [r0]
|| }
|.endmacro
/* In 64-bit build we compare only low 32-bits.
* x86_64 cmp instruction doesn't support immediate 64-bit operand, and full
* comparison would require an additional load of 64-bit address into register.
* This is not a problem at all, while JIT buffer size is less than 4GB.
*/
|.macro CMP_IP, addr
|| if (GCC_GLOBAL_REGS) {
| cmp IPl, addr
|| } else {
| cmp dword EX->opline, addr
|| }
|.endmacro
|.macro LOAD_ZVAL_ADDR, reg, addr
|| if (Z_MODE(addr) == IS_CONST_ZVAL) {
| LOAD_ADDR reg, Z_ZV(addr)
|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
| LOAD_BASE_ADDR reg, Z_REG(addr), Z_OFFSET(addr)
|| } else {
|| ZEND_UNREACHABLE();
|| }
|.endmacro
|.macro PUSH_ZVAL_ADDR, addr, tmp_reg
|| if (Z_MODE(addr) == IS_CONST_ZVAL) {
| PUSH_ADDR Z_ZV(addr), tmp_reg
|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
| PUSH_BASE_ADDR Z_REG(addr), Z_OFFSET(addr), tmp_reg
|| } else {
|| ZEND_UNREACHABLE();
|| }
|.endmacro
|.macro GET_Z_TYPE_INFO, reg, zv
| mov reg, dword [zv+offsetof(zval,u1.type_info)]
|.endmacro
|.macro SET_Z_TYPE_INFO, zv, type
| mov dword [zv+offsetof(zval,u1.type_info)], type
|.endmacro
|.macro GET_ZVAL_TYPE, reg, addr
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| mov reg, byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval,u1.v.type)]
|.endmacro
|.macro GET_ZVAL_TYPE_INFO, reg, addr
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| mov reg, dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval,u1.type_info)]
|.endmacro
|.macro SET_ZVAL_TYPE_INFO, addr, type
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| mov dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval,u1.type_info)], type
|.endmacro
|.macro GET_Z_PTR, reg, zv
| mov reg, aword [zv]
|.endmacro
|.macro SET_Z_PTR, zv, val
| mov aword [zv], val
|.endmacro
|.macro GET_Z_W2, reg, zv
| mov reg, dword [zv+4]
|.endmacro
|.macro SET_Z_W2, zv, reg
| mov dword [zv+4], reg
|.endmacro
|.macro GET_ZVAL_PTR, reg, addr
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| mov reg, aword [Ra(Z_REG(addr))+Z_OFFSET(addr)]
|.endmacro
|.macro SET_ZVAL_PTR, addr, val
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| mov aword [Ra(Z_REG(addr))+Z_OFFSET(addr)], val
|.endmacro
|.macro GET_ZVAL_W2, reg, addr
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| mov reg, dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+4]
|.endmacro
|.macro SET_ZVAL_W2, addr, val
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| mov dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+4], val
|.endmacro
|.macro UNDEF_OPLINE_RESULT
| mov r0, EX->opline
| mov eax, dword OP:r0->result.var
| SET_Z_TYPE_INFO FP + r0, IS_UNDEF
|.endmacro
|.macro UNDEF_OPLINE_RESULT_IF_USED
| test byte OP:RX->result_type, (IS_TMP_VAR|IS_VAR)
| jz >1
| mov eax, dword OP:RX->result.var
| SET_Z_TYPE_INFO FP + r0, IS_UNDEF
|1:
|.endmacro
|.macro SSE_AVX_INS, sse_ins, avx_ins, op1, op2
|| if (CAN_USE_AVX()) {
| avx_ins op1, op2
|| } else {
| sse_ins op1, op2
|| }
|.endmacro
|.macro SSE_OP, sse_ins, reg, addr, tmp_reg
|| if (Z_MODE(addr) == IS_CONST_ZVAL) {
| MEM_OP2_2 sse_ins, xmm(reg-ZREG_XMM0), qword, Z_ZV(addr), tmp_reg
|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
| sse_ins xmm(reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)]
|| } else if (Z_MODE(addr) == IS_REG) {
| sse_ins xmm(reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0)
|| } else {
|| ZEND_UNREACHABLE();
|| }
|.endmacro
|.macro SSE_AVX_OP, sse_ins, avx_ins, reg, addr
|| if (Z_MODE(addr) == IS_CONST_ZVAL) {
| .if X64
|| if (IS_SIGNED_32BIT(Z_ZV(addr))) {
| SSE_AVX_INS sse_ins, avx_ins, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)]
|| } else {
| LOAD_ADDR r0, Z_ZV(addr)
| SSE_AVX_INS sse_ins, avx_ins, xmm(reg-ZREG_XMM0), qword [r0]
|| }
| .else
| SSE_AVX_INS sse_ins, avx_ins, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)]
| .endif
|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
| SSE_AVX_INS sse_ins, avx_ins, xmm(reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)]
|| } else if (Z_MODE(addr) == IS_REG) {
| SSE_AVX_INS sse_ins, avx_ins, xmm(reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0)
|| } else {
|| ZEND_UNREACHABLE();
|| }
|.endmacro
|.macro SSE_GET_LONG, reg, lval, tmp_reg
|| if (lval == 0) {
|| if (CAN_USE_AVX()) {
| vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0)
|| } else {
| xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0)
|| }
|| } else {
|.if X64
|| if (!IS_SIGNED_32BIT(lval)) {
| mov64 Ra(tmp_reg), lval
|| } else {
| mov Ra(tmp_reg), lval
|| }
|.else
| mov Ra(tmp_reg), lval
|.endif
|| if (CAN_USE_AVX()) {
| vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0)
| vcvtsi2sd, xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), Ra(tmp_reg)
|| } else {
| xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0)
| cvtsi2sd, xmm(reg-ZREG_XMM0), Ra(tmp_reg)
|| }
|| }
|.endmacro
|.macro SSE_GET_ZVAL_LVAL, reg, addr, tmp_reg
|| if (Z_MODE(addr) == IS_CONST_ZVAL) {
| SSE_GET_LONG reg, Z_LVAL_P(Z_ZV(addr)), tmp_reg
|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
|| if (CAN_USE_AVX()) {
| vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0)
| vcvtsi2sd xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)]
|| } else {
| xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0)
| cvtsi2sd xmm(reg-ZREG_XMM0), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)]
|| }
|| } else if (Z_MODE(addr) == IS_REG) {
|| if (CAN_USE_AVX()) {
| vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0)
| vcvtsi2sd xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), Ra(Z_REG(addr))
|| } else {
| xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0)
| cvtsi2sd xmm(reg-ZREG_XMM0), Ra(Z_REG(addr))
|| }
|| } else {
|| ZEND_UNREACHABLE();
|| }
|.endmacro
|.macro SSE_GET_ZVAL_DVAL, reg, addr
|| if (Z_MODE(addr) != IS_REG || reg != Z_REG(addr)) {
|| if (Z_MODE(addr) == IS_CONST_ZVAL) {
| .if X64
|| if (IS_SIGNED_32BIT(Z_ZV(addr))) {
| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)]
|| } else {
| LOAD_ADDR r0, Z_ZV(addr)
| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [r0]
|| }
| .else
| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)]
| .endif
|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)]
|| } else if (Z_MODE(addr) == IS_REG) {
| SSE_AVX_INS movaps, vmovaps, xmm(reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0)
|| } else {
|| ZEND_UNREACHABLE();
|| }
|| }
|.endmacro
|.macro SSE_MATH, opcode, reg, addr, tmp_reg
|| switch (opcode) {
|| case ZEND_ADD:
| SSE_OP addsd, reg, addr, tmp_reg
|| break;
|| case ZEND_SUB:
| SSE_OP subsd, reg, addr, tmp_reg
|| break;
|| case ZEND_MUL:
| SSE_OP mulsd, reg, addr, tmp_reg
|| break;
|| case ZEND_DIV:
| SSE_OP divsd, reg, addr, tmp_reg
|| break;
|| }
|.endmacro
|.macro SSE_MATH_REG, opcode, dst_reg, src_reg
|| switch (opcode) {
|| case ZEND_ADD:
| addsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0)
|| break;
|| case ZEND_SUB:
| subsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0)
|| break;
|| case ZEND_MUL:
| mulsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0)
|| break;
|| case ZEND_DIV:
| divsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0)
|| break;
|| }
|.endmacro
|.macro SSE_SET_ZVAL_DVAL, addr, reg
|| if (Z_MODE(addr) == IS_REG) {
|| if (reg != Z_REG(addr)) {
| SSE_AVX_INS movaps, vmovaps, xmm(Z_REG(addr)-ZREG_XMM0), xmm(reg-ZREG_XMM0)
|| }
|| } else {
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| SSE_AVX_INS movsd, vmovsd, qword [Ra(Z_REG(addr))+Z_OFFSET(addr)], xmm(reg-ZREG_XMM0)
|| }
|.endmacro
|.macro AVX_OP, avx_ins, reg, op1_reg, addr, tmp_reg
|| if (Z_MODE(addr) == IS_CONST_ZVAL) {
| MEM_OP3_3 avx_ins, xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword, Z_ZV(addr), tmp_reg
|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)]
|| } else if (Z_MODE(addr) == IS_REG) {
| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0)
|| } else {
|| ZEND_UNREACHABLE();
|| }
|.endmacro
|.macro AVX_MATH, opcode, reg, op1_reg, addr, tmp_reg
|| switch (opcode) {
|| case ZEND_ADD:
| AVX_OP vaddsd, reg, op1_reg, addr, tmp_reg
|| break;
|| case ZEND_SUB:
| AVX_OP vsubsd, reg, op1_reg, addr, tmp_reg
|| break;
|| case ZEND_MUL:
| AVX_OP vmulsd, reg, op1_reg, addr, tmp_reg
|| break;
|| case ZEND_DIV:
| AVX_OP vdivsd, reg, op1_reg, addr, tmp_reg
|| break;
|| }
|.endmacro
|.macro AVX_MATH_REG, opcode, dst_reg, op1_reg, src_reg
|| switch (opcode) {
|| case ZEND_ADD:
| vaddsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0)
|| break;
|| case ZEND_SUB:
| vsubsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0)
|| break;
|| case ZEND_MUL:
| vmulsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0)
|| break;
|| case ZEND_DIV:
| vdivsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0)
|| break;
|| }
|.endmacro
|.macro LONG_OP, long_ins, reg, addr, tmp_reg
|| if (Z_MODE(addr) == IS_CONST_ZVAL) {
| .if X64
|| if (!IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(addr)))) {
| mov64 tmp_reg, Z_LVAL_P(Z_ZV(addr))
| long_ins Ra(reg), tmp_reg
|| } else {
| long_ins Ra(reg), Z_LVAL_P(Z_ZV(addr))
|| }
| .else
| long_ins Ra(reg), Z_LVAL_P(Z_ZV(addr))
| .endif
|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
| long_ins Ra(reg), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)]
|| } else if (Z_MODE(addr) == IS_REG) {
| long_ins Ra(reg), Ra(Z_REG(addr))
|| } else {
|| ZEND_UNREACHABLE();
|| }
|.endmacro
|.macro LONG_OP_WITH_32BIT_CONST, long_ins, op1_addr, lval
|| if (Z_MODE(op1_addr) == IS_MEM_ZVAL) {
| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], lval
|| } else if (Z_MODE(op1_addr) == IS_REG) {
| long_ins Ra(Z_REG(op1_addr)), lval
|| } else {
|| ZEND_UNREACHABLE();
|| }
|.endmacro
|.macro LONG_OP_WITH_CONST, long_ins, op1_addr, lval
|| if (Z_MODE(op1_addr) == IS_MEM_ZVAL) {
| .if X64
|| if (!IS_SIGNED_32BIT(lval)) {
| mov64 r0, lval
| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], r0
|| } else {
| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], lval
|| }
| .else
| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], lval
| .endif
|| } else if (Z_MODE(op1_addr) == IS_REG) {
| .if X64
|| if (!IS_SIGNED_32BIT(lval)) {
| mov64 r0, lval
| long_ins Ra(Z_REG(op1_addr)), r0
|| } else {
| long_ins Ra(Z_REG(op1_addr)), lval
|| }
| .else
| long_ins Ra(Z_REG(op1_addr)), lval
| .endif
|| } else {
|| ZEND_UNREACHABLE();
|| }
|.endmacro
|.macro GET_ZVAL_LVAL, reg, addr
|| if (Z_MODE(addr) == IS_CONST_ZVAL) {
|| if (Z_LVAL_P(Z_ZV(addr)) == 0) {
| xor Ra(reg), Ra(reg)
|| } else {
| .if X64
|| if (!IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(addr)))) {
| mov64 Ra(reg), Z_LVAL_P(Z_ZV(addr))
|| } else {
| mov Ra(reg), Z_LVAL_P(Z_ZV(addr))
|| }
| .else
| mov Ra(reg), Z_LVAL_P(Z_ZV(addr))
| .endif
|| }
|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
| mov Ra(reg), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)]
|| } else if (Z_MODE(addr) == IS_REG) {
|| if (reg != Z_REG(addr)) {
| mov Ra(reg), Ra(Z_REG(addr))
|| }
|| } else {
|| ZEND_UNREACHABLE();
|| }
|.endmacro
|.macro LONG_MATH, opcode, reg, addr, tmp_reg
|| switch (opcode) {
|| case ZEND_ADD:
| LONG_OP add, reg, addr, Ra(tmp_reg)
|| break;
|| case ZEND_SUB:
| LONG_OP sub, reg, addr, Ra(tmp_reg)
|| break;
|| case ZEND_MUL:
| LONG_OP imul, reg, addr, Ra(tmp_reg)
|| break;
|| case ZEND_BW_OR:
| LONG_OP or, reg, addr, Ra(tmp_reg)
|| break;
|| case ZEND_BW_AND:
| LONG_OP and, reg, addr, Ra(tmp_reg)
|| break;
|| case ZEND_BW_XOR:
| LONG_OP xor, reg, addr, Ra(tmp_reg)
|| break;
|| default:
|| ZEND_UNREACHABLE();
|| }
|.endmacro
|.macro LONG_MATH_REG, opcode, dst_reg, src_reg
|| switch (opcode) {
|| case ZEND_ADD:
| add dst_reg, src_reg
|| break;
|| case ZEND_SUB:
| sub dst_reg, src_reg
|| break;
|| case ZEND_MUL:
| imul dst_reg, src_reg
|| break;
|| case ZEND_BW_OR:
| or dst_reg, src_reg
|| break;
|| case ZEND_BW_AND:
| and dst_reg, src_reg
|| break;
|| case ZEND_BW_XOR:
| xor dst_reg, src_reg
|| break;
|| default:
|| ZEND_UNREACHABLE();
|| }
|.endmacro
|.macro SET_ZVAL_LVAL, addr, lval
|| if (Z_MODE(addr) == IS_REG) {
| mov Ra(Z_REG(addr)), lval
|| } else {
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| mov aword [Ra(Z_REG(addr))+Z_OFFSET(addr)], lval
|| }
|.endmacro
|.macro ZVAL_COPY_CONST, dst_addr, dst_info, dst_def_info, zv, tmp_reg
|| if (Z_TYPE_P(zv) > IS_TRUE) {
|| if (Z_TYPE_P(zv) == IS_DOUBLE) {
|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : ZREG_XMM0;
|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) {
|| if (CAN_USE_AVX()) {
| vxorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0)
|| } else {
| xorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0)
|| }
| .if X64
|| } else if (!IS_SIGNED_32BIT(zv)) {
| mov64 Ra(tmp_reg), ((uintptr_t)zv)
| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [Ra(tmp_reg)]
| .endif
|| } else {
| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [((uint32_t)(uintptr_t)zv)]
|| }
| SSE_SET_ZVAL_DVAL dst_addr, dst_reg
|| } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) {
|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : ZREG_XMM0;
| SSE_GET_LONG dst_reg, Z_LVAL_P(zv), ZREG_R0
| SSE_SET_ZVAL_DVAL dst_addr, dst_reg
|| } else if (Z_LVAL_P(zv) == 0 && Z_MODE(dst_addr) == IS_REG) {
| xor Ra(Z_REG(dst_addr)), Ra(Z_REG(dst_addr))
|| } else {
| .if X64
|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) {
|| if (Z_MODE(dst_addr) == IS_REG) {
| mov64 Ra(Z_REG(dst_addr)), ((uintptr_t)Z_LVAL_P(zv))
|| } else {
| mov64 Ra(tmp_reg), ((uintptr_t)Z_LVAL_P(zv))
| SET_ZVAL_LVAL dst_addr, Ra(tmp_reg)
|| }
|| } else {
| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv)
|| }
| .else
| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv)
| .endif
|| }
|| }
|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) {
|| if (dst_def_info == MAY_BE_DOUBLE) {
|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
| SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE
|| }
|| } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) {
| SET_ZVAL_TYPE_INFO dst_addr, Z_TYPE_INFO_P(zv)
|| }
|| }
|.endmacro
|.macro ZVAL_COPY_CONST_2, dst_addr, res_addr, dst_info, dst_def_info, zv, tmp_reg
|| if (Z_TYPE_P(zv) > IS_TRUE) {
|| if (Z_TYPE_P(zv) == IS_DOUBLE) {
|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ?
|| Z_REG(dst_addr) : ((Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_XMM0);
|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) {
|| if (CAN_USE_AVX()) {
| vxorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0)
|| } else {
| xorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0)
|| }
| .if X64
|| } else if (!IS_SIGNED_32BIT(zv)) {
| mov64 Ra(tmp_reg), ((uintptr_t)zv)
| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [Ra(tmp_reg)]
| .endif
|| } else {
| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [((uint32_t)(uintptr_t)zv)]
|| }
| SSE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0
| SSE_SET_ZVAL_DVAL res_addr, ZREG_XMM0
|| } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) {
|| if (Z_MODE(dst_addr) == IS_REG) {
| SSE_GET_LONG Z_REG(dst_addr), Z_LVAL_P(zv), ZREG_R0
| SSE_SET_ZVAL_DVAL res_addr, Z_REG(dst_addr)
|| } else if (Z_MODE(res_addr) == IS_REG) {
| SSE_GET_LONG Z_REG(res_addr), Z_LVAL_P(zv), ZREG_R0
| SSE_SET_ZVAL_DVAL dst_addr, Z_REG(res_addr)
|| } else {
| SSE_GET_LONG ZREG_XMM0, Z_LVAL_P(zv), ZREG_R0
| SSE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0
| SSE_SET_ZVAL_DVAL res_addr, ZREG_XMM0
|| }
|| } else if (Z_LVAL_P(zv) == 0 && (Z_MODE(dst_addr) == IS_REG || Z_MODE(res_addr) == IS_REG)) {
|| if (Z_MODE(dst_addr) == IS_REG) {
| xor Ra(Z_REG(dst_addr)), Ra(Z_REG(dst_addr))
| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr))
|| } else {
| xor Ra(Z_REG(res_addr)), Ra(Z_REG(res_addr))
| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr))
|| }
|| } else {
| .if X64
|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) {
|| if (Z_MODE(dst_addr) == IS_REG) {
| mov64 Ra(Z_REG(dst_addr)), ((uintptr_t)Z_LVAL_P(zv))
| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr))
|| } else if (Z_MODE(res_addr) == IS_REG) {
| mov64 Ra(Z_REG(res_addr)), ((uintptr_t)Z_LVAL_P(zv))
| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr))
|| } else {
| mov64 Ra(tmp_reg), ((uintptr_t)Z_LVAL_P(zv))
| SET_ZVAL_LVAL dst_addr, Ra(tmp_reg)
| SET_ZVAL_LVAL res_addr, Ra(tmp_reg)
|| }
|| } else if (Z_MODE(dst_addr) == IS_REG) {
| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv)
| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr))
|| } else if (Z_MODE(res_addr) == IS_REG) {
| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv)
| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr))
|| } else {
| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv)
| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv)
|| }
| .else
|| if (Z_MODE(dst_addr) == IS_REG) {
| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv)
| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr))
|| } else if (Z_MODE(res_addr) == IS_REG) {
| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv)
| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr))
|| } else {
| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv)
| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv)
|| }
| .endif
|| }
|| }
|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) {
|| if (dst_def_info == MAY_BE_DOUBLE) {
|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
| SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE
|| }
|| } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) {
| SET_ZVAL_TYPE_INFO dst_addr, Z_TYPE_INFO_P(zv)
|| }
|| }
|| if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
|| if (dst_def_info == MAY_BE_DOUBLE) {
| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE
|| } else {
| SET_ZVAL_TYPE_INFO res_addr, Z_TYPE_INFO_P(zv)
|| }
|| }
|.endmacro
/* the same as above, but "src" may overlap with "tmp_reg1" */
|.macro ZVAL_COPY_VALUE, dst_addr, dst_info, src_addr, src_info, tmp_reg1, tmp_reg2
| ZVAL_COPY_VALUE_V dst_addr, dst_info, src_addr, src_info, tmp_reg1, tmp_reg2
|| if ((src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) &&
|| !(src_info & MAY_BE_GUARD) &&
|| has_concrete_type(src_info & MAY_BE_ANY)) {
|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) {
|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD))) {
|| zend_uchar type = concrete_type(src_info);
| SET_ZVAL_TYPE_INFO dst_addr, type
|| }
|| }
|| } else {
| GET_ZVAL_TYPE_INFO Rd(tmp_reg1), src_addr
| SET_ZVAL_TYPE_INFO dst_addr, Rd(tmp_reg1)
|| }
|.endmacro
|.macro ZVAL_COPY_VALUE_V, dst_addr, dst_info, src_addr, src_info, tmp_reg1, tmp_reg2
|| if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) {
|| if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_LONG) {
|| if (Z_MODE(src_addr) == IS_REG) {
|| if (Z_MODE(dst_addr) != IS_REG || Z_REG(dst_addr) != Z_REG(src_addr)) {
| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(src_addr))
|| }
|| } else if (Z_MODE(dst_addr) == IS_REG) {
| GET_ZVAL_LVAL Z_REG(dst_addr), src_addr
|| } else {
| GET_ZVAL_LVAL tmp_reg2, src_addr
| SET_ZVAL_LVAL dst_addr, Ra(tmp_reg2)
|| }
|| } else if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) {
|| if (Z_MODE(src_addr) == IS_REG) {
| SSE_SET_ZVAL_DVAL dst_addr, Z_REG(src_addr)
|| } else if (Z_MODE(dst_addr) == IS_REG) {
| SSE_GET_ZVAL_DVAL Z_REG(dst_addr), src_addr
|| } else {
| SSE_GET_ZVAL_DVAL ZREG_XMM0, src_addr
| SSE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0
|| }
|| } else if (!(src_info & (MAY_BE_DOUBLE|MAY_BE_GUARD))) {
| GET_ZVAL_PTR Ra(tmp_reg2), src_addr
| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2)
|| } else {
| .if X64
| GET_ZVAL_PTR Ra(tmp_reg2), src_addr
| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2)
| .else
|| if ((tmp_reg1 == tmp_reg2 || tmp_reg1 == Z_REG(src_addr))) {
| GET_ZVAL_W2 Ra(tmp_reg2), src_addr
| SET_ZVAL_W2 dst_addr, Ra(tmp_reg2)
| GET_ZVAL_PTR Ra(tmp_reg2), src_addr
| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2)
|| } else {
| GET_ZVAL_PTR Ra(tmp_reg2), src_addr
| GET_ZVAL_W2 Ra(tmp_reg1), src_addr
| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2)
| SET_ZVAL_W2 dst_addr, Ra(tmp_reg1)
|| }
| .endif
|| }
|| }
|.endmacro
|.macro ZVAL_COPY_VALUE_2, dst_addr, dst_info, res_addr, src_addr, src_info, tmp_reg1, tmp_reg2
|| if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) {
|| if ((src_info & MAY_BE_ANY) == MAY_BE_LONG) {
|| if (Z_MODE(src_addr) == IS_REG) {
|| if (Z_MODE(dst_addr) != IS_REG || Z_REG(dst_addr) != Z_REG(src_addr)) {
| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(src_addr))
|| }
|| if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != Z_REG(src_addr)) {
| SET_ZVAL_LVAL res_addr, Ra(Z_REG(src_addr))
|| }
|| } else if (Z_MODE(dst_addr) == IS_REG) {
| GET_ZVAL_LVAL Z_REG(dst_addr), src_addr
|| if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != Z_REG(dst_addr)) {
| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr))
|| }
|| } else if (Z_MODE(res_addr) == IS_REG) {
| GET_ZVAL_LVAL Z_REG(res_addr), src_addr
| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr))
|| } else {
| GET_ZVAL_LVAL tmp_reg2, src_addr
| SET_ZVAL_LVAL dst_addr, Ra(tmp_reg2)
| SET_ZVAL_LVAL res_addr, Ra(tmp_reg2)
|| }
|| } else if ((src_info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
|| if (Z_MODE(src_addr) == IS_REG) {
| SSE_SET_ZVAL_DVAL dst_addr, Z_REG(src_addr)
| SSE_SET_ZVAL_DVAL res_addr, Z_REG(src_addr)
|| } else if (Z_MODE(dst_addr) == IS_REG) {
| SSE_GET_ZVAL_DVAL Z_REG(dst_addr), src_addr
| SSE_SET_ZVAL_DVAL res_addr, Z_REG(dst_addr)
|| } else if (Z_MODE(res_addr) == IS_REG) {
| SSE_GET_ZVAL_DVAL Z_REG(res_addr), src_addr
| SSE_SET_ZVAL_DVAL dst_addr, Z_REG(res_addr)
|| } else {
| SSE_GET_ZVAL_DVAL ZREG_XMM0, src_addr
| SSE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0
| SSE_SET_ZVAL_DVAL res_addr, ZREG_XMM0
|| }
|| } else if (!(src_info & MAY_BE_DOUBLE)) {
| GET_ZVAL_PTR Ra(tmp_reg2), src_addr
| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2)
| SET_ZVAL_PTR res_addr, Ra(tmp_reg2)
|| } else {
| .if X64
| GET_ZVAL_PTR Ra(tmp_reg2), src_addr
| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2)
| SET_ZVAL_PTR res_addr, Ra(tmp_reg2)
| .else
|| if (tmp_reg1 == tmp_reg2 || tmp_reg1 == Z_REG(src_addr)) {
| GET_ZVAL_W2 Ra(tmp_reg2), src_addr
| SET_ZVAL_W2 dst_addr, Ra(tmp_reg2)
| SET_ZVAL_W2 res_addr, Ra(tmp_reg2)
| GET_ZVAL_PTR Ra(tmp_reg2), src_addr
| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2)
| SET_ZVAL_PTR res_addr, Ra(tmp_reg2)
|| } else {
| GET_ZVAL_PTR Ra(tmp_reg2), src_addr
| GET_ZVAL_W2 Ra(tmp_reg1), src_addr
| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2)
| SET_ZVAL_PTR res_addr, Ra(tmp_reg2)
| SET_ZVAL_W2 dst_addr, Ra(tmp_reg1)
| SET_ZVAL_W2 res_addr, Ra(tmp_reg1)
|| }
| .endif
|| }
|| }
|| if ((src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) &&
|| has_concrete_type(src_info & MAY_BE_ANY)) {
|| zend_uchar type = concrete_type(src_info);
|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) {
|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF))) {
| SET_ZVAL_TYPE_INFO dst_addr, type
|| }
|| }
|| if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
| SET_ZVAL_TYPE_INFO res_addr, type
|| }
|| } else {
| GET_ZVAL_TYPE_INFO Rd(tmp_reg1), src_addr
| SET_ZVAL_TYPE_INFO dst_addr, Rd(tmp_reg1)
| SET_ZVAL_TYPE_INFO res_addr, Rd(tmp_reg1)
|| }
|.endmacro
|.macro IF_UNDEF, type_reg, label
| test type_reg, type_reg
| je label
|.endmacro
|.macro IF_TYPE, type, val, label
| cmp type, val
| je label
|.endmacro
|.macro IF_NOT_TYPE, type, val, label
| cmp type, val
| jne label
|.endmacro
|.macro IF_Z_TYPE, zv, val, label
| IF_TYPE byte [zv+offsetof(zval, u1.v.type)], val, label
|.endmacro
|.macro IF_NOT_Z_TYPE, zv, val, label
| IF_NOT_TYPE byte [zv+offsetof(zval, u1.v.type)], val, label
|.endmacro
|.macro CMP_ZVAL_TYPE, addr, val
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| cmp byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type)], val
|.endmacro
|.macro IF_ZVAL_TYPE, addr, val, label
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| IF_TYPE byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type)], val, label
|.endmacro
|.macro IF_NOT_ZVAL_TYPE, addr, val, label
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| IF_NOT_TYPE byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type)], val, label
|.endmacro
|.macro IF_FLAGS, type_flags, mask, label
| test type_flags, mask
| jnz label
|.endmacro
|.macro IF_NOT_FLAGS, type_flags, mask, label
| test type_flags, mask
| jz label
|.endmacro
|.macro IF_REFCOUNTED, type_flags, label
| IF_FLAGS type_flags, IS_TYPE_REFCOUNTED, label
|.endmacro
|.macro IF_NOT_REFCOUNTED, type_flags, label
| //IF_NOT_FLAGS type_flags, IS_TYPE_REFCOUNTED, label
| test type_flags, type_flags
| jz label
|.endmacro
|.macro IF_ZVAL_FLAGS, addr, mask, label
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| IF_FLAGS byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags)], mask, label
|.endmacro
|.macro IF_NOT_ZVAL_FLAGS, addr, mask, label
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| IF_NOT_FLAGS byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags)], mask, label
|.endmacro
|.macro IF_ZVAL_REFCOUNTED, addr, label
| IF_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, label
|.endmacro
|.macro IF_NOT_ZVAL_REFCOUNTED, addr, label
| IF_NOT_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, label
|.endmacro
|.macro IF_NOT_ZVAL_COLLECTABLE, addr, label
| IF_NOT_ZVAL_FLAGS addr, IS_TYPE_COLLECTABLE, label
|.endmacro
|.macro GC_ADDREF, zv
| add dword [zv], 1
|.endmacro
|.macro GC_DELREF, zv
| sub dword [zv], 1
|.endmacro
|.macro IF_GC_MAY_NOT_LEAK, ptr, label
| test dword [ptr+4],(GC_INFO_MASK | (GC_NOT_COLLECTABLE << GC_FLAGS_SHIFT))
| jne label
|.endmacro
|.macro ADDREF_CONST, zv, tmp_reg
| .if X64
|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) {
| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv))
| add dword [tmp_reg], 1
|| } else {
| add dword [Z_LVAL_P(zv)], 1
|| }
| .else
| add dword [Z_LVAL_P(zv)], 1
| .endif
|.endmacro
|.macro ADDREF_CONST_2, zv, tmp_reg
| .if X64
|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) {
| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv))
| add dword [tmp_reg], 2
|| } else {
| add dword [Z_LVAL_P(zv)], 2
|| }
| .else
| add dword [Z_LVAL_P(zv)], 2
| .endif
|.endmacro
|.macro TRY_ADDREF, val_info, type_flags_reg, value_ptr_reg
|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
|| if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| IF_NOT_REFCOUNTED type_flags_reg, >1
|| }
| GC_ADDREF value_ptr_reg
|1:
|| }
|.endmacro
|.macro TRY_ADDREF_2, val_info, type_flags_reg, value_ptr_reg
|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
|| if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| IF_NOT_REFCOUNTED type_flags_reg, >1
|| }
| add dword [value_ptr_reg], 2
|1:
|| }
|.endmacro
|.macro ZVAL_DEREF, reg, info
|| if (info & MAY_BE_REF) {
| IF_NOT_Z_TYPE, reg, IS_REFERENCE, >1
| GET_Z_PTR reg, reg
| add reg, offsetof(zend_reference, val)
|1:
|| }
|.endmacro
|.macro SET_EX_OPLINE, op, tmp_reg
|| if (op == last_valid_opline) {
|| zend_jit_use_last_valid_opline();
| SAVE_IP
|| } else {
| ADDR_OP2_2 mov, aword EX->opline, op, tmp_reg
|| if (!GCC_GLOBAL_REGS) {
|| zend_jit_reset_last_valid_opline();
|| }
|| }
|.endmacro
// zval should be in FCARG1a
|.macro ZVAL_DTOR_FUNC, var_info, opline // arg1 must be in FCARG1a
|| do {
|| if (!((var_info) & MAY_BE_GUARD)
|| && has_concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
|| zend_uchar type = concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE));
|| if (type == IS_STRING && !ZEND_DEBUG) {
| EXT_CALL _efree, r0
|| break;
|| } else if (type == IS_ARRAY) {
|| if ((var_info) & (MAY_BE_ARRAY_KEY_STRING|MAY_BE_ARRAY_OF_STRING|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF)) {
|| if (opline && ((var_info) & (MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF))) {
| SET_EX_OPLINE opline, r0
|| }
| EXT_CALL zend_array_destroy, r0
|| } else {
| EXT_CALL zend_jit_array_free, r0
|| }
|| break;
|| } else if (type == IS_OBJECT) {
|| if (opline) {
| SET_EX_OPLINE opline, r0
|| }
| EXT_CALL zend_objects_store_del, r0
|| break;
|| }
|| }
|| if (opline) {
| SET_EX_OPLINE opline, r0
|| }
| EXT_CALL rc_dtor_func, r0
|| } while(0);
|.endmacro
|.macro ZVAL_PTR_DTOR, addr, op_info, gc, cold, opline
|| if ((op_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF|MAY_BE_GUARD)) {
|| if ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT|MAY_BE_GUARD)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| // if (Z_REFCOUNTED_P(cv)) {
|| if (cold) {
| IF_ZVAL_REFCOUNTED addr, >1
|.cold_code
|1:
|| } else {
| IF_NOT_ZVAL_REFCOUNTED addr, >4
|| }
|| }
| // if (!Z_DELREF_P(cv)) {
| GET_ZVAL_PTR FCARG1a, addr
| GC_DELREF FCARG1a
|| if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_1(op_info)) {
|| if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_N(op_info)) {
|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) {
| jnz >3
|| } else {
| jnz >4
|| }
|| }
| // zval_dtor_func(r);
| ZVAL_DTOR_FUNC op_info, opline
|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) {
| jmp >4
|| }
|3:
|| }
|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) {
|| if ((op_info) & (MAY_BE_REF|MAY_BE_GUARD)) {
|| zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, offsetof(zend_reference, val));
| IF_NOT_ZVAL_TYPE addr, IS_REFERENCE, >1
| IF_NOT_ZVAL_COLLECTABLE ref_addr, >4
| GET_ZVAL_PTR FCARG1a, ref_addr
|1:
|| }
| IF_GC_MAY_NOT_LEAK FCARG1a, >4
| // gc_possible_root(Z_COUNTED_P(z))
| EXT_CALL gc_possible_root, r0
|| }
|| if (cold && ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT|MAY_BE_GUARD)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) != 0) {
| jmp >4
|.code
|| }
|4:
|| }
|.endmacro
|.macro FREE_OP, op_type, op, op_info, cold, opline
|| if (op_type & (IS_VAR|IS_TMP_VAR)) {
| ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var), op_info, 0, cold, opline
|| }
|.endmacro
|.macro SEPARATE_ARRAY, addr, op_info, cold
|| if (RC_MAY_BE_N(op_info)) {
|| if (Z_REG(addr) != ZREG_FP) {
| GET_ZVAL_LVAL ZREG_R0, addr
|| if (RC_MAY_BE_1(op_info)) {
| cmp dword [r0], 1 // if (GC_REFCOUNT() > 1)
| jbe >2
|| }
|| if (Z_REG(addr) != ZREG_FCARG1a || Z_OFFSET(addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, addr
|| }
| EXT_CALL zend_jit_zval_array_dup, r0
|2:
| mov FCARG1a, r0
|| } else {
| GET_ZVAL_LVAL ZREG_FCARG1a, addr
|| if (RC_MAY_BE_1(op_info)) {
| cmp dword [FCARG1a], 1 // if (GC_REFCOUNT() > 1)
|| if (cold) {
| ja >1
|.cold_code
|1:
|| } else {
| jbe >2
|| }
|| }
| IF_NOT_ZVAL_REFCOUNTED addr, >1
| GC_DELREF FCARG1a
|1:
| EXT_CALL zend_array_dup, r0
| SET_ZVAL_PTR addr, r0
| SET_ZVAL_TYPE_INFO addr, IS_ARRAY_EX
| mov FCARG1a, r0
|| if (RC_MAY_BE_1(op_info)) {
|| if (cold) {
| jmp >2
|.code
|| }
|| }
|2:
|| }
|| } else {
| GET_ZVAL_LVAL ZREG_FCARG1a, addr
|| }
|.endmacro
|.macro EFREE_REG_REFERENCE
||#if ZEND_DEBUG
| xor FCARG2a, FCARG2a // filename
| .if X64WIN
| xor CARG3d, CARG3d // lineno
| xor CARG4, CARG4
| mov aword A5, 0
| EXT_CALL _efree, r0
| .elif X64
| xor CARG3d, CARG3d // lineno
| xor CARG4, CARG4
| xor CARG5, CARG5
| EXT_CALL _efree, r0
| .else
| sub r4, 4
| push 0
| push 0
| push 0 // lineno
| EXT_CALL _efree, r0
| add r4, 4
| .endif
||#else
||#ifdef HAVE_BUILTIN_CONSTANT_P
| EXT_CALL _efree_32, r0
||#else
| EXT_CALL _efree, r0
||#endif
||#endif
|.endmacro
|.macro EFREE_REFERENCE, ptr
| mov FCARG1a, ptr
| EFREE_REG_REFERENCE
|.endmacro
|.macro EMALLOC, size, op_array, opline
||#if ZEND_DEBUG
|| const char *filename = op_array->filename ? op_array->filename->val : NULL;
| mov FCARG1a, size
| LOAD_ADDR FCARG2a, filename
| .if X64WIN
| mov CARG3d, opline->lineno
| xor CARG4, CARG4
| mov aword A5, 0
| EXT_CALL _emalloc, r0
| .elif X64
| mov CARG3d, opline->lineno
| xor CARG4, CARG4
| xor CARG5, CARG5
| EXT_CALL _emalloc, r0
| .else
| sub r4, 4
| push 0
| push 0
| push opline->lineno
| EXT_CALL _emalloc, r0
| add r4, 4
| .endif
||#else
||#ifdef HAVE_BUILTIN_CONSTANT_P
|| if (size > 24 && size <= 32) {
| EXT_CALL _emalloc_32, r0
|| } else {
| mov FCARG1a, size
| EXT_CALL _emalloc, r0
|| }
||#else
| mov FCARG1a, size
| EXT_CALL _emalloc, r0
||#endif
||#endif
|.endmacro
|.macro OBJ_RELEASE, reg, exit_label
| GC_DELREF Ra(reg)
| jne >1
| // zend_objects_store_del(obj);
|| if (reg != ZREG_FCARG1a) {
| mov FCARG1a, Ra(reg)
|| }
| EXT_CALL zend_objects_store_del, r0
| jmp exit_label
|1:
| IF_GC_MAY_NOT_LEAK Ra(reg), >1
| // gc_possible_root(obj)
|| if (reg != ZREG_FCARG1a) {
| mov FCARG1a, Ra(reg)
|| }
| EXT_CALL gc_possible_root, r0
|1:
|.endmacro
|.macro UNDEFINED_OFFSET, opline
|| if (opline == last_valid_opline) {
|| zend_jit_use_last_valid_opline();
| call ->undefined_offset_ex
|| } else {
| SET_EX_OPLINE opline, r0
| call ->undefined_offset
|| }
|.endmacro
|.macro UNDEFINED_INDEX, opline
|| if (opline == last_valid_opline) {
|| zend_jit_use_last_valid_opline();
| call ->undefined_index_ex
|| } else {
| SET_EX_OPLINE opline, r0
| call ->undefined_index
|| }
|.endmacro
|.macro CANNOT_ADD_ELEMENT, opline
|| if (opline == last_valid_opline) {
|| zend_jit_use_last_valid_opline();
| call ->cannot_add_element_ex
|| } else {
| SET_EX_OPLINE opline, r0
| call ->cannot_add_element
|| }
|.endmacro
static zend_bool reuse_ip = 0;
static zend_bool delayed_call_chain = 0;
static uint32_t delayed_call_level = 0;
static const zend_op *last_valid_opline = NULL;
static zend_bool use_last_vald_opline = 0;
static zend_bool track_last_valid_opline = 0;
static int jit_return_label = -1;
static uint32_t current_trace_num = 0;
static uint32_t allowed_opt_flags = 0;
static void zend_jit_track_last_valid_opline(void)
{
use_last_vald_opline = 0;
track_last_valid_opline = 1;
}
static void zend_jit_use_last_valid_opline(void)
{
if (track_last_valid_opline) {
use_last_vald_opline = 1;
track_last_valid_opline = 0;
}
}
static zend_bool zend_jit_trace_uses_initial_ip(void)
{
return use_last_vald_opline;
}
static void zend_jit_set_last_valid_opline(const zend_op *target_opline)
{
if (!reuse_ip) {
track_last_valid_opline = 0;
last_valid_opline = target_opline;
}
}
static void zend_jit_reset_last_valid_opline(void)
{
track_last_valid_opline = 0;
last_valid_opline = NULL;
}
static void zend_jit_start_reuse_ip(void)
{
zend_jit_reset_last_valid_opline();
reuse_ip = 1;
}
static int zend_jit_reuse_ip(dasm_State **Dst)
{
if (!reuse_ip) {
zend_jit_start_reuse_ip();
| // call = EX(call);
| mov RX, EX->call
}
return 1;
}
static void zend_jit_stop_reuse_ip(void)
{
reuse_ip = 0;
}
/* bit helpers */
/* from http://aggregate.org/MAGIC/ */
static uint32_t ones32(uint32_t x)
{
x -= ((x >> 1) & 0x55555555);
x = (((x >> 2) & 0x33333333) + (x & 0x33333333));
x = (((x >> 4) + x) & 0x0f0f0f0f);
x += (x >> 8);
x += (x >> 16);
return x & 0x0000003f;
}
static uint32_t floor_log2(uint32_t x)
{
ZEND_ASSERT(x != 0);
x |= (x >> 1);
x |= (x >> 2);
x |= (x >> 4);
x |= (x >> 8);
x |= (x >> 16);
return ones32(x) - 1;
}
static zend_bool is_power_of_two(uint32_t x)
{
return !(x & (x - 1)) && x != 0;
}
static zend_bool has_concrete_type(uint32_t value_type)
{
return is_power_of_two (value_type & (MAY_BE_ANY|MAY_BE_UNDEF));
}
static uint32_t concrete_type(uint32_t value_type)
{
return floor_log2(value_type & (MAY_BE_ANY|MAY_BE_UNDEF));
}
static inline zend_bool is_signed(double d)
{
return (((unsigned char*)&d)[sizeof(double)-1] & 0x80) != 0;
}
static int zend_jit_interrupt_handler_stub(dasm_State **Dst)
{
|->interrupt_handler:
| SAVE_IP
| //EG(vm_interrupt) = 0;
| MEM_OP2_1_ZTS mov, byte, executor_globals, vm_interrupt, 0, r0
| //if (EG(timed_out)) {
| MEM_OP2_1_ZTS cmp, byte, executor_globals, timed_out, 0, r0
| je >1
| //zend_timeout();
| EXT_CALL zend_timeout, r0
|1:
| //} else if (zend_interrupt_function) {
if (zend_interrupt_function) {
| //zend_interrupt_function(execute_data);
|.if X64
| mov CARG1, FP
| EXT_CALL zend_interrupt_function, r0
|.else
| mov aword A1, FP
| EXT_CALL zend_interrupt_function, r0
|.endif
| MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0
| je >1
| EXT_CALL zend_jit_exception_in_interrupt_handler_helper, r0
|1:
| //ZEND_VM_ENTER();
| //execute_data = EG(current_execute_data);
| MEM_OP2_2_ZTS mov, FP, aword, executor_globals, current_execute_data, r0
| LOAD_IP
}
| //ZEND_VM_CONTINUE()
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| ADD_HYBRID_SPAD
| JMP_IP
} else if (GCC_GLOBAL_REGS) {
| add r4, SPAD // stack alignment
| JMP_IP
} else {
| mov FP, aword T2 // restore FP
| mov RX, aword T3 // restore IP
| add r4, NR_SPAD // stack alignment
| mov r0, 1 // ZEND_VM_ENTER
| ret
}
return 1;
}
static int zend_jit_exception_handler_stub(dasm_State **Dst)
{
|->exception_handler:
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
const void *handler = zend_get_opcode_handler_func(EG(exception_op));
| ADD_HYBRID_SPAD
| EXT_CALL handler, r0
| JMP_IP
} else {
const void *handler = EG(exception_op)->handler;
if (GCC_GLOBAL_REGS) {
| add r4, SPAD // stack alignment
| EXT_JMP handler, r0
} else {
| mov FCARG1a, FP
| EXT_CALL handler, r0
| mov FP, aword T2 // restore FP
| mov RX, aword T3 // restore IP
| add r4, NR_SPAD // stack alignment
| test eax, eax
| jl >1
| mov r0, 1 // ZEND_VM_ENTER
|1:
| ret
}
}
return 1;
}
static int zend_jit_exception_handler_undef_stub(dasm_State **Dst)
{
|->exception_handler_undef:
| MEM_OP2_2_ZTS mov, r0, aword, executor_globals, opline_before_exception, r0
| test byte OP:r0->result_type, (IS_TMP_VAR|IS_VAR)
| jz >1
| mov eax, dword OP:r0->result.var
| SET_Z_TYPE_INFO FP + r0, IS_UNDEF
|1:
| jmp ->exception_handler
return 1;
}
static int zend_jit_exception_handler_free_op1_op2_stub(dasm_State **Dst)
{
|->exception_handler_free_op1_op2:
| UNDEF_OPLINE_RESULT_IF_USED
| test byte OP:RX->op1_type, (IS_TMP_VAR|IS_VAR)
| je >9
| mov eax, dword OP:RX->op1.var
| add r0, FP
| ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL
|9:
| test byte OP:RX->op2_type, (IS_TMP_VAR|IS_VAR)
| je >9
| mov eax, dword OP:RX->op2.var
| add r0, FP
| ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL
|9:
| jmp ->exception_handler
return 1;
}
static int zend_jit_exception_handler_free_op2_stub(dasm_State **Dst)
{
|->exception_handler_free_op2:
| MEM_OP2_2_ZTS mov, RX, aword, executor_globals, opline_before_exception, r0
| UNDEF_OPLINE_RESULT_IF_USED
| test byte OP:RX->op2_type, (IS_TMP_VAR|IS_VAR)
| je >9
| mov eax, dword OP:RX->op2.var
| add r0, FP
| ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL
|9:
| jmp ->exception_handler
return 1;
}
static int zend_jit_leave_function_stub(dasm_State **Dst)
{
|->leave_function_handler:
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| test FCARG1d, ZEND_CALL_TOP
| jnz >1
| EXT_CALL zend_jit_leave_nested_func_helper, r0
| ADD_HYBRID_SPAD
| JMP_IP
|1:
| EXT_CALL zend_jit_leave_top_func_helper, r0
| ADD_HYBRID_SPAD
| JMP_IP
} else {
if (GCC_GLOBAL_REGS) {
| add r4, SPAD
} else {
| mov FCARG2a, FP
| mov FP, aword T2 // restore FP
| mov RX, aword T3 // restore IP
| add r4, NR_SPAD
}
| test FCARG1d, ZEND_CALL_TOP
| jnz >1
| EXT_JMP zend_jit_leave_nested_func_helper, r0
|1:
| EXT_JMP zend_jit_leave_top_func_helper, r0
}
return 1;
}
static int zend_jit_leave_throw_stub(dasm_State **Dst)
{
|->leave_throw_handler:
| // if (opline->opcode != ZEND_HANDLE_EXCEPTION) {
if (GCC_GLOBAL_REGS) {
| cmp byte OP:IP->opcode, ZEND_HANDLE_EXCEPTION
| je >5
| // EG(opline_before_exception) = opline;
| MEM_OP2_1_ZTS mov, aword, executor_globals, opline_before_exception, IP, r0
|5:
| // opline = EG(exception_op);
| LOAD_IP_ADDR_ZTS executor_globals, exception_op
| // HANDLE_EXCEPTION()
| jmp ->exception_handler
} else {
| GET_IP FCARG1a
| cmp byte OP:FCARG1a->opcode, ZEND_HANDLE_EXCEPTION
| je >5
| // EG(opline_before_exception) = opline;
| MEM_OP2_1_ZTS mov, aword, executor_globals, opline_before_exception, FCARG1a, r0
|5:
| // opline = EG(exception_op);
| LOAD_IP_ADDR_ZTS executor_globals, exception_op
| mov FP, aword T2 // restore FP
| mov RX, aword T3 // restore IP
| add r4, NR_SPAD // stack alignment
| mov r0, 2 // ZEND_VM_LEAVE
| ret
}
return 1;
}
static int zend_jit_icall_throw_stub(dasm_State **Dst)
{
|->icall_throw_handler:
| // zend_rethrow_exception(zend_execute_data *execute_data)
| mov IP, aword EX->opline
| // if (EX(opline)->opcode != ZEND_HANDLE_EXCEPTION) {
| cmp byte OP:IP->opcode, ZEND_HANDLE_EXCEPTION
| je >1
| // EG(opline_before_exception) = opline;
| MEM_OP2_1_ZTS mov, aword, executor_globals, opline_before_exception, IP, r0
|1:
| // opline = EG(exception_op);
| LOAD_IP_ADDR_ZTS executor_globals, exception_op
|| if (GCC_GLOBAL_REGS) {
| mov aword EX->opline, IP
|| }
| // HANDLE_EXCEPTION()
| jmp ->exception_handler
return 1;
}
static int zend_jit_throw_cannot_pass_by_ref_stub(dasm_State **Dst)
{
|->throw_cannot_pass_by_ref:
| mov r0, EX->opline
| mov ecx, dword OP:r0->result.var
| SET_Z_TYPE_INFO RX+r1, IS_UNDEF
| // last EX(call) frame may be delayed
| cmp RX, EX->call
| je >1
| mov r1, EX->call
| mov EX:RX->prev_execute_data, r1
| mov EX->call, RX
|1:
| mov RX, r0
| mov FCARG1d, dword OP:r0->op2.num
| EXT_CALL zend_cannot_pass_by_reference, r0
| cmp byte OP:RX->op1_type, IS_TMP_VAR
| jne >9
| mov eax, dword OP:RX->op1.var
| add r0, FP
| ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL
|9:
| jmp ->exception_handler
return 1;
}
static int zend_jit_undefined_offset_ex_stub(dasm_State **Dst)
{
|->undefined_offset_ex:
| SAVE_IP
| jmp ->undefined_offset
return 1;
}
static int zend_jit_undefined_offset_stub(dasm_State **Dst)
{
|->undefined_offset:
|.if X64WIN
| sub r4, 0x28
|.elif X64
| sub r4, 8
|.else
| sub r4, 12
|.endif
| mov r0, EX->opline
| mov ecx, dword OP:r0->result.var
| cmp byte OP:r0->op2_type, IS_CONST
| SET_Z_TYPE_INFO FP + r1, IS_NULL
| jne >2
|.if X64
| movsxd r1, dword OP:r0->op2.constant
| add r0, r1
|.else
| mov r0, aword OP:r0->op2.zv
|.endif
| jmp >3
|2:
| mov eax, dword OP:r0->op2.var
| add r0, FP
|3:
|.if X64WIN
| mov CARG1, E_WARNING
| LOAD_ADDR CARG2, "Undefined array key " ZEND_LONG_FMT
| mov CARG3, aword [r0]
| EXT_CALL zend_error, r0
| add r4, 0x28 // stack alignment
|.elif X64
| mov CARG1, E_WARNING
| LOAD_ADDR CARG2, "Undefined array key " ZEND_LONG_FMT
| mov CARG3, aword [r0]
| EXT_CALL zend_error, r0
| add r4, 8 // stack alignment
|.else
| sub r4, 4
| push aword [r0]
| push "Undefined array key " ZEND_LONG_FMT
| push E_WARNING
| EXT_CALL zend_error, r0
| add r4, 28
|.endif
| ret
return 1;
}
static int zend_jit_undefined_index_ex_stub(dasm_State **Dst)
{
|->undefined_index_ex:
| SAVE_IP
| jmp ->undefined_index
return 1;
}
static int zend_jit_undefined_index_stub(dasm_State **Dst)
{
|->undefined_index:
|.if X64WIN
| sub r4, 0x28
|.elif X64
| sub r4, 8
|.else
| sub r4, 12
|.endif
| mov r0, EX->opline
| mov ecx, dword OP:r0->result.var
| cmp byte OP:r0->op2_type, IS_CONST
| SET_Z_TYPE_INFO FP + r1, IS_NULL
| jne >2
|.if X64
| movsxd r1, dword OP:r0->op2.constant
| add r0, r1
|.else
| mov r0, aword OP:r0->op2.zv
|.endif
| jmp >3
|2:
| mov eax, dword OP:r0->op2.var
| add r0, FP
|3:
|.if X64WIN
| mov CARG1, E_WARNING
| LOAD_ADDR CARG2, "Undefined array key \"%s\""
| mov CARG3, aword [r0]
| add CARG3, offsetof(zend_string, val)
| EXT_CALL zend_error, r0
| add r4, 0x28
|.elif X64
| mov CARG1, E_WARNING
| LOAD_ADDR CARG2, "Undefined array key \"%s\""
| mov CARG3, aword [r0]
| add CARG3, offsetof(zend_string, val)
| EXT_CALL zend_error, r0
| add r4, 8
|.else
| sub r4, 4
| mov r0, aword [r0]
| add r0, offsetof(zend_string, val)
| push r0
| push "Undefined array key \"%s\""
| push E_WARNING
| EXT_CALL zend_error, r0
| add r4, 28
|.endif
| ret
return 1;
}
static int zend_jit_cannot_add_element_ex_stub(dasm_State **Dst)
{
|->cannot_add_element_ex:
| SAVE_IP
| jmp ->cannot_add_element
return 1;
}
static int zend_jit_cannot_add_element_stub(dasm_State **Dst)
{
|->cannot_add_element:
|.if X64WIN
| sub r4, 0x28
|.elif X64
| sub r4, 8
|.else
| sub r4, 12
|.endif
| mov r0, EX->opline
| cmp byte OP:r0->result_type, IS_UNUSED
| jz >1
| mov eax, dword OP:r0->result.var
| SET_Z_TYPE_INFO FP + r0, IS_NULL
|1:
|.if X64WIN
| xor CARG1, CARG1
| LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied"
| EXT_CALL zend_throw_error, r0
| add r4, 0x28
|.elif X64
| xor CARG1, CARG1
| LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied"
| EXT_CALL zend_throw_error, r0
| add r4, 8
|.else
| sub r4, 8
| push "Cannot add element to the array as the next element is already occupied"
| push 0
| EXT_CALL zend_throw_error, r0
| add r4, 28
|.endif
| ret
return 1;
}
static int zend_jit_undefined_function_stub(dasm_State **Dst)
{
|->undefined_function:
| mov r0, aword EX->opline
|.if X64
| xor CARG1, CARG1
| LOAD_ADDR CARG2, "Call to undefined function %s()"
| movsxd CARG3, dword [r0 + offsetof(zend_op, op2.constant)]
| mov CARG3, aword [r0 + CARG3]
| add CARG3, offsetof(zend_string, val)
| EXT_CALL zend_throw_error, r0
|.else
| mov r0, aword [r0 + offsetof(zend_op, op2.zv)]
| mov r0, aword [r0]
| add r0, offsetof(zend_string, val)
| mov aword A3, r0
| mov aword A2, "Call to undefined function %s()"
| mov aword A1, 0
| EXT_CALL zend_throw_error, r0
|.endif
| jmp ->exception_handler
return 1;
}
static int zend_jit_negative_shift_stub(dasm_State **Dst)
{
|->negative_shift:
| mov RX, EX->opline
|.if X64
|.if WIN
| LOAD_ADDR CARG1, &zend_ce_arithmetic_error
| mov CARG1, aword [CARG1]
|.else
| LOAD_ADDR CARG1, zend_ce_arithmetic_error
|.endif
| LOAD_ADDR CARG2, "Bit shift by negative number"
| EXT_CALL zend_throw_error, r0
|.else
| sub r4, 8
| push "Bit shift by negative number"
|.if WIN
| LOAD_ADDR r0, &zend_ce_arithmetic_error
| push aword [r0]
|.else
| PUSH_ADDR zend_ce_arithmetic_error, r0
|.endif
| EXT_CALL zend_throw_error, r0
| add r4, 16
|.endif
| jmp ->exception_handler_free_op1_op2
return 1;
}
static int zend_jit_mod_by_zero_stub(dasm_State **Dst)
{
|->mod_by_zero:
| mov RX, EX->opline
|.if X64
|.if WIN
| LOAD_ADDR CARG1, &zend_ce_division_by_zero_error
| mov CARG1, aword [CARG1]
|.else
| LOAD_ADDR CARG1, zend_ce_division_by_zero_error
|.endif
| LOAD_ADDR CARG2, "Modulo by zero"
| EXT_CALL zend_throw_error, r0
|.else
| sub r4, 8
| push "Modulo by zero"
|.if WIN
| LOAD_ADDR r0, &zend_ce_division_by_zero_error
| push aword [r0]
|.else
| PUSH_ADDR zend_ce_division_by_zero_error, r0
|.endif
| EXT_CALL zend_throw_error, r0
| add r4, 16
|.endif
| jmp ->exception_handler_free_op1_op2
return 1;
}
static int zend_jit_invalid_this_stub(dasm_State **Dst)
{
|->invalid_this:
| UNDEF_OPLINE_RESULT
|.if X64
| xor CARG1, CARG1
| LOAD_ADDR CARG2, "Using $this when not in object context"
| EXT_CALL zend_throw_error, r0
|.else
| sub r4, 8
| push "Using $this when not in object context"
| push 0
| EXT_CALL zend_throw_error, r0
| add r4, 16
|.endif
| jmp ->exception_handler
return 1;
}
static int zend_jit_double_one_stub(dasm_State **Dst)
{
|->one:
|.dword 0, 0x3ff00000
return 1;
}
static int zend_jit_hybrid_runtime_jit_stub(dasm_State **Dst)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
return 1;
}
|->hybrid_runtime_jit:
| EXT_CALL zend_runtime_jit, r0
| JMP_IP
return 1;
}
static int zend_jit_hybrid_profile_jit_stub(dasm_State **Dst)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
return 1;
}
|->hybrid_profile_jit:
| // ++zend_jit_profile_counter;
| .if X64
| LOAD_ADDR r0, &zend_jit_profile_counter
| inc aword [r0]
| .else
| inc aword [&zend_jit_profile_counter]
| .endif
| // op_array = (zend_op_array*)EX(func);
| mov r0, EX->func
| // run_time_cache = EX(run_time_cache);
| mov r2, EX->run_time_cache
| // jit_extension = (const void*)ZEND_FUNC_INFO(op_array);
| mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])]
| // ++ZEND_COUNTER_INFO(op_array)
| inc aword [r2 + zend_jit_profile_counter_rid * sizeof(void*)]
| // return ((zend_vm_opcode_handler_t)jit_extension->orig_handler)()
| jmp aword [r0 + offsetof(zend_jit_op_array_extension, orig_handler)]
return 1;
}
static int zend_jit_hybrid_hot_code_stub(dasm_State **Dst)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
return 1;
}
|->hybrid_hot_code:
| mov word [r2], ZEND_JIT_COUNTER_INIT
| mov FCARG1a, FP
| GET_IP FCARG2a
| EXT_CALL zend_jit_hot_func, r0
| JMP_IP
return 1;
}
/*
* This code is based Mike Pall's "Hashed profile counters" idea, implemented
* in LuaJIT. The full description may be found in "LuaJIT 2.0 intellectual
* property disclosure and research opportunities" email
* at http://lua-users.org/lists/lua-l/2009-11/msg00089.html
*
* In addition we use a variation of Knuth's multiplicative hash function
* described at https://code.i-harness.com/en/q/a21ce
*
* uint64_t hash(uint64_t x) {
* x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9;
* x = (x ^ (x >> 27)) * 0x94d049bb133111eb;
* x = x ^ (x >> 31);
* return x;
* }
*
* uint_32_t hash(uint32_t x) {
* x = ((x >> 16) ^ x) * 0x45d9f3b;
* x = ((x >> 16) ^ x) * 0x45d9f3b;
* x = (x >> 16) ^ x;
* return x;
* }
*
*/
static int zend_jit_hybrid_hot_counter_stub(dasm_State **Dst, uint32_t cost)
{
| mov r0, EX->func
| mov r1, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])]
| mov r2, aword [r1 + offsetof(zend_jit_op_array_hot_extension, counter)]
| sub word [r2], cost
| jle ->hybrid_hot_code
| GET_IP r2
| sub r2, aword [r0 + offsetof(zend_op_array, opcodes)]
| // divide by sizeof(zend_op)
| .if X64
|| ZEND_ASSERT(sizeof(zend_op) == 32);
| sar r2, 2
| .else
|| ZEND_ASSERT(sizeof(zend_op) == 28);
| imul r2, 0xb6db6db7
| .endif
| .if X64
| jmp aword [r1+r2+offsetof(zend_jit_op_array_hot_extension, orig_handlers)]
| .else
| jmp aword [r1+r2+offsetof(zend_jit_op_array_hot_extension, orig_handlers)]
| .endif
return 1;
}
static int zend_jit_hybrid_func_hot_counter_stub(dasm_State **Dst)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) {
return 1;
}
|->hybrid_func_hot_counter:
return zend_jit_hybrid_hot_counter_stub(Dst,
((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func)));
}
static int zend_jit_hybrid_loop_hot_counter_stub(dasm_State **Dst)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) {
return 1;
}
|->hybrid_loop_hot_counter:
return zend_jit_hybrid_hot_counter_stub(Dst,
((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop)));
}
static int zend_jit_hybrid_hot_trace_stub(dasm_State **Dst)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
return 1;
}
|->hybrid_hot_trace:
| mov word [r2], ZEND_JIT_COUNTER_INIT
| mov FCARG1a, FP
| GET_IP FCARG2a
| EXT_CALL zend_jit_trace_hot_root, r0
| test eax, eax // TODO : remove this check at least for HYBRID VM ???
| jl >1
| MEM_OP2_2_ZTS mov, FP, aword, executor_globals, current_execute_data, r0
| LOAD_IP
| JMP_IP
|1:
| EXT_JMP zend_jit_halt_op->handler, r0
return 1;
}
static int zend_jit_hybrid_trace_counter_stub(dasm_State **Dst, uint32_t cost)
{
| mov r0, EX->func
| mov r1, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])]
| mov r1, aword [r1 + offsetof(zend_jit_op_array_trace_extension, offset)]
| mov r2, aword [IP + r1 + offsetof(zend_op_trace_info, counter)]
| sub word [r2], cost
| jle ->hybrid_hot_trace
| jmp aword [IP + r1]
return 1;
}
static int zend_jit_hybrid_func_trace_counter_stub(dasm_State **Dst)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) {
return 1;
}
|->hybrid_func_trace_counter:
return zend_jit_hybrid_trace_counter_stub(Dst,
((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func)));
}
static int zend_jit_hybrid_ret_trace_counter_stub(dasm_State **Dst)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_return)) {
return 1;
}
|->hybrid_ret_trace_counter:
return zend_jit_hybrid_trace_counter_stub(Dst,
((ZEND_JIT_COUNTER_INIT + JIT_G(hot_return) - 1) / JIT_G(hot_return)));
}
static int zend_jit_hybrid_loop_trace_counter_stub(dasm_State **Dst)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) {
return 1;
}
|->hybrid_loop_trace_counter:
return zend_jit_hybrid_trace_counter_stub(Dst,
((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop)));
}
static int zend_jit_trace_halt_stub(dasm_State **Dst)
{
|->trace_halt:
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| ADD_HYBRID_SPAD
| EXT_JMP zend_jit_halt_op->handler, r0
} else if (GCC_GLOBAL_REGS) {
| add r4, SPAD // stack alignment
| xor IP, IP // PC must be zero
| ret
} else {
| mov FP, aword T2 // restore FP
| mov RX, aword T3 // restore IP
| add r4, NR_SPAD // stack alignment
| mov r0, -1 // ZEND_VM_RETURN
| ret
}
return 1;
}
static int zend_jit_trace_exit_stub(dasm_State **Dst)
{
|->trace_exit:
|
| // Save CPU registers
|.if X64
| sub r4, 16*8+16*8-8 /* CPU regs + SSE regs */
| mov aword [r4+15*8], r15
| mov aword [r4+11*8], r11
| mov aword [r4+10*8], r10
| mov aword [r4+9*8], r9
| mov aword [r4+8*8], r8
| mov aword [r4+7*8], rdi
| mov aword [r4+6*8], rsi
| mov aword [r4+2*8], rdx
| mov aword [r4+1*8], rcx
| mov aword [r4+0*8], rax
| mov FCARG1a, aword [r4+16*8+16*8-8] // exit_num = POP
| mov FCARG2a, r4
| movsd qword [r4+16*8+15*8], xmm15
| movsd qword [r4+16*8+14*8], xmm14
| movsd qword [r4+16*8+13*8], xmm13
| movsd qword [r4+16*8+12*8], xmm12
| movsd qword [r4+16*8+11*8], xmm11
| movsd qword [r4+16*8+10*8], xmm10
| movsd qword [r4+16*8+9*8], xmm9
| movsd qword [r4+16*8+8*8], xmm8
| movsd qword [r4+16*8+7*8], xmm7
| movsd qword [r4+16*8+6*8], xmm6
| movsd qword [r4+16*8+5*8], xmm5
| movsd qword [r4+16*8+4*8], xmm4
| movsd qword [r4+16*8+3*8], xmm3
| movsd qword [r4+16*8+2*8], xmm2
| movsd qword [r4+16*8+1*8], xmm1
| movsd qword [r4+16*8+0*8], xmm0
|.if X64WIN
| sub r4, 32 /* shadow space */
|.endif
|.else
| sub r4, 8*4+8*8-4 /* CPU regs + SSE regs */
| mov aword [r4+7*4], edi
| mov aword [r4+2*4], edx
| mov aword [r4+1*4], ecx
| mov aword [r4+0*4], eax
| mov FCARG1a, aword [r4+8*4+8*8-4] // exit_num = POP
| mov FCARG2a, r4
| movsd qword [r4+8*4+7*8], xmm7
| movsd qword [r4+8*4+6*8], xmm6
| movsd qword [r4+8*4+5*8], xmm5
| movsd qword [r4+8*4+4*8], xmm4
| movsd qword [r4+8*4+3*8], xmm3
| movsd qword [r4+8*4+2*8], xmm2
| movsd qword [r4+8*4+1*8], xmm1
| movsd qword [r4+8*4+0*8], xmm0
|.endif
|
| // EX(opline) = opline
| SAVE_IP
| // zend_jit_trace_exit(trace_num, exit_num)
| EXT_CALL zend_jit_trace_exit, r0
|.if X64WIN
| add r4, 16*8+16*8+32 /* CPU regs + SSE regs + shadow space */
|.elif X64
| add r4, 16*8+16*8 /* CPU regs + SSE regs */
|.else
| add r4, 8*4+8*8 /* CPU regs + SSE regs */
|.endif
| test eax, eax
| jne >1
| // execute_data = EG(current_execute_data)
| MEM_OP2_2_ZTS mov, FP, aword, executor_globals, current_execute_data, r0
| // opline = EX(opline)
| LOAD_IP
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| ADD_HYBRID_SPAD
| JMP_IP
} else if (GCC_GLOBAL_REGS) {
| add r4, SPAD // stack alignment
| JMP_IP
} else {
| mov FP, aword T2 // restore FP
| mov RX, aword T3 // restore IP
| add r4, NR_SPAD // stack alignment
| mov r0, 1 // ZEND_VM_ENTER
| ret
}
|1:
| jl ->trace_halt
| // execute_data = EG(current_execute_data)
| MEM_OP2_2_ZTS mov, FP, aword, executor_globals, current_execute_data, r0
| // opline = EX(opline)
| LOAD_IP
| // check for interrupt (try to avoid this ???)
| MEM_OP2_1_ZTS cmp, byte, executor_globals, vm_interrupt, 0, r0
| jne ->interrupt_handler
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| ADD_HYBRID_SPAD
| mov r0, EX->func
| mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])]
| mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)]
| jmp aword [IP + r0]
} else if (GCC_GLOBAL_REGS) {
| add r4, SPAD // stack alignment
| mov r0, EX->func
| mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])]
| mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)]
| jmp aword [IP + r0]
} else {
| mov IP, aword EX->opline
| mov FCARG1a, FP
| mov r0, EX->func
| mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])]
| mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)]
| call aword [IP + r0]
| test eax, eax
| jl ->trace_halt
| mov FP, aword T2 // restore FP
| mov RX, aword T3 // restore IP
| add r4, NR_SPAD // stack alignment
| mov r0, 1 // ZEND_VM_ENTER
| ret
}
return 1;
}
static int zend_jit_trace_escape_stub(dasm_State **Dst)
{
|->trace_escape:
|
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| ADD_HYBRID_SPAD
| JMP_IP
} else if (GCC_GLOBAL_REGS) {
| add r4, SPAD // stack alignment
| JMP_IP
} else {
| mov FP, aword T2 // restore FP
| mov RX, aword T3 // restore IP
| add r4, NR_SPAD // stack alignment
| mov r0, 1 // ZEND_VM_ENTER
| ret
}
return 1;
}
/* Keep 32 exit points in a single code block */
#define ZEND_JIT_EXIT_POINTS_SPACING 4 // push byte + short jmp = bytes
#define ZEND_JIT_EXIT_POINTS_PER_GROUP 32 // number of continuous exit points
static int zend_jit_trace_exit_group_stub(dasm_State **Dst, uint32_t n)
{
uint32_t i;
for (i = 0; i < ZEND_JIT_EXIT_POINTS_PER_GROUP - 1; i++) {
| push byte i
| .byte 0xeb, (4*(ZEND_JIT_EXIT_POINTS_PER_GROUP-i)-6) // jmp >1
}
| push byte i
|// 1:
| add aword [r4], n
| jmp ->trace_exit
return 1;
}
#ifdef CONTEXT_THREADED_JIT
static int zend_jit_context_threaded_call_stub(dasm_State **Dst)
{
|->context_threaded_call:
| pop r0
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| ADD_HYBRID_SPAD
| jmp aword [IP]
} else if (GCC_GLOBAL_REGS) {
| add r4, SPAD // stack alignment
| jmp aword [IP]
} else {
ZEND_UNREACHABLE();
// TODO: context threading can't work without GLOBAL REGS because we have to change
// the value of execute_data in execute_ex()
| mov FCARG1a, FP
| mov r0, aword [FP]
| mov FP, aword T2 // restore FP
| mov RX, aword T3 // restore IP
| add r4, NR_SPAD // stack alignment
| jmp aword [r0]
}
return 1;
}
#endif
static int zend_jit_assign_to_variable(dasm_State **Dst,
const zend_op *opline,
zend_jit_addr var_use_addr,
zend_jit_addr var_addr,
uint32_t var_info,
uint32_t var_def_info,
zend_uchar val_type,
zend_jit_addr val_addr,
uint32_t val_info,
zend_jit_addr res_addr,
zend_bool check_exception);
static int zend_jit_assign_const_stub(dasm_State **Dst)
{
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2a, 0);
uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN;
|->assign_const:
|.if X64WIN
| sub r4, 0x28
|.elif X64
| sub r4, 8
|.else
| sub r4, 12
|.endif
if (!zend_jit_assign_to_variable(
Dst, NULL,
var_addr, var_addr, -1, -1,
IS_CONST, val_addr, val_info,
0, 0)) {
return 0;
}
|.if X64WIN
| add r4, 0x28
|.elif X64
| add r4, 8
|.else
| add r4, 12
|.endif
| ret
return 1;
}
static int zend_jit_assign_tmp_stub(dasm_State **Dst)
{
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2a, 0);
uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN;
|->assign_tmp:
|.if X64WIN
| sub r4, 0x28
|.elif X64
| sub r4, 8
|.else
| sub r4, 12
|.endif
if (!zend_jit_assign_to_variable(
Dst, NULL,
var_addr, var_addr, -1, -1,
IS_TMP_VAR, val_addr, val_info,
0, 0)) {
return 0;
}
|.if X64WIN
| add r4, 0x28
|.elif X64
| add r4, 8
|.else
| add r4, 12
|.endif
| ret
return 1;
}
static int zend_jit_assign_var_stub(dasm_State **Dst)
{
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2a, 0);
uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF;
|->assign_var:
|.if X64WIN
| sub r4, 0x28
|.elif X64
| sub r4, 8
|.else
| sub r4, 12
|.endif
if (!zend_jit_assign_to_variable(
Dst, NULL,
var_addr, var_addr, -1, -1,
IS_VAR, val_addr, val_info,
0, 0)) {
return 0;
}
|.if X64WIN
| add r4, 0x28
|.elif X64
| add r4, 8
|.else
| add r4, 12
|.endif
| ret
return 1;
}
static int zend_jit_assign_cv_noref_stub(dasm_State **Dst)
{
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2a, 0);
uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN/*|MAY_BE_UNDEF*/;
|->assign_cv_noref:
|.if X64WIN
| sub r4, 0x28
|.elif X64
| sub r4, 8
|.else
| sub r4, 12
|.endif
if (!zend_jit_assign_to_variable(
Dst, NULL,
var_addr, var_addr, -1, -1,
IS_CV, val_addr, val_info,
0, 0)) {
return 0;
}
|.if X64WIN
| add r4, 0x28
|.elif X64
| add r4, 8
|.else
| add r4, 12
|.endif
| ret
return 1;
}
static int zend_jit_assign_cv_stub(dasm_State **Dst)
{
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2a, 0);
uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF/*|MAY_BE_UNDEF*/;
|->assign_cv:
|.if X64WIN
| sub r4, 0x28
|.elif X64
| sub r4, 8
|.else
| sub r4, 12
|.endif
if (!zend_jit_assign_to_variable(
Dst, NULL,
var_addr, var_addr, -1, -1,
IS_CV, val_addr, val_info,
0, 0)) {
return 0;
}
|.if X64WIN
| add r4, 0x28
|.elif X64
| add r4, 8
|.else
| add r4, 12
|.endif
| ret
return 1;
}
static const zend_jit_stub zend_jit_stubs[] = {
JIT_STUB(interrupt_handler),
JIT_STUB(exception_handler),
JIT_STUB(exception_handler_undef),
JIT_STUB(exception_handler_free_op1_op2),
JIT_STUB(exception_handler_free_op2),
JIT_STUB(leave_function),
JIT_STUB(leave_throw),
JIT_STUB(icall_throw),
JIT_STUB(throw_cannot_pass_by_ref),
JIT_STUB(undefined_offset),
JIT_STUB(undefined_index),
JIT_STUB(cannot_add_element),
JIT_STUB(undefined_offset_ex),
JIT_STUB(undefined_index_ex),
JIT_STUB(cannot_add_element_ex),
JIT_STUB(undefined_function),
JIT_STUB(negative_shift),
JIT_STUB(mod_by_zero),
JIT_STUB(invalid_this),
JIT_STUB(trace_halt),
JIT_STUB(trace_exit),
JIT_STUB(trace_escape),
JIT_STUB(hybrid_runtime_jit),
JIT_STUB(hybrid_profile_jit),
JIT_STUB(hybrid_hot_code),
JIT_STUB(hybrid_func_hot_counter),
JIT_STUB(hybrid_loop_hot_counter),
JIT_STUB(hybrid_hot_trace),
JIT_STUB(hybrid_func_trace_counter),
JIT_STUB(hybrid_ret_trace_counter),
JIT_STUB(hybrid_loop_trace_counter),
JIT_STUB(assign_const),
JIT_STUB(assign_tmp),
JIT_STUB(assign_var),
JIT_STUB(assign_cv_noref),
JIT_STUB(assign_cv),
JIT_STUB(double_one),
#ifdef CONTEXT_THREADED_JIT
JIT_STUB(context_threaded_call),
#endif
};
#if ZTS && defined(ZEND_WIN32)
extern uint32_t _tls_index;
extern char *_tls_start;
extern char *_tls_end;
#endif
static int zend_jit_setup(void)
{
if (!zend_cpu_supports_sse2()) {
zend_error(E_CORE_ERROR, "CPU doesn't support SSE2");
return FAILURE;
}
allowed_opt_flags = 0;
if (zend_cpu_supports_avx()) {
allowed_opt_flags |= ZEND_JIT_CPU_AVX;
}
#if ZTS
# ifdef _WIN64
tsrm_tls_index = _tls_index * sizeof(void*);
/* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */
/* Probably, it might be better solution */
do {
void ***tls_mem = ((void**)__readgsqword(0x58))[_tls_index];
void *val = _tsrm_ls_cache;
size_t offset = 0;
size_t size = (char*)&_tls_end - (char*)&_tls_start;
while (offset < size) {
if (*tls_mem == val) {
tsrm_tls_offset = offset;
break;
}
tls_mem++;
offset += sizeof(void*);
}
if (offset >= size) {
// TODO: error message ???
return FAILURE;
}
} while(0);
# elif ZEND_WIN32
tsrm_tls_index = _tls_index * sizeof(void*);
/* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */
/* Probably, it might be better solution */
do {
void ***tls_mem = ((void***)__readfsdword(0x2c))[_tls_index];
void *val = _tsrm_ls_cache;
size_t offset = 0;
size_t size = (char*)&_tls_end - (char*)&_tls_start;
while (offset < size) {
if (*tls_mem == val) {
tsrm_tls_offset = offset;
break;
}
tls_mem++;
offset += sizeof(void*);
}
if (offset >= size) {
// TODO: error message ???
return FAILURE;
}
} while(0);
# elif defined(__APPLE__) && defined(__x86_64__)
tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset();
if (tsrm_ls_cache_tcb_offset == 0) {
size_t *ti;
__asm__(
"leaq __tsrm_ls_cache(%%rip),%0"
: "=r" (ti));
tsrm_tls_offset = ti[2];
tsrm_tls_index = ti[1] * 8;
}
# elif defined(__GNUC__) && defined(__x86_64__)
tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset();
if (tsrm_ls_cache_tcb_offset == 0) {
#if defined(__has_attribute) && __has_attribute(tls_model) && !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__MUSL__)
size_t ret;
asm ("movq _tsrm_ls_cache@gottpoff(%%rip),%0"
: "=r" (ret));
tsrm_ls_cache_tcb_offset = ret;
#else
size_t *ti;
__asm__(
"leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n"
: "=a" (ti));
tsrm_tls_offset = ti[1];
tsrm_tls_index = ti[0] * 16;
#endif
}
# elif defined(__GNUC__) && defined(__i386__)
tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset();
if (tsrm_ls_cache_tcb_offset == 0) {
#if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__MUSL__)
size_t ret;
asm ("leal _tsrm_ls_cache@ntpoff,%0\n"
: "=a" (ret));
tsrm_ls_cache_tcb_offset = ret;
#else
size_t *ti, _ebx, _ecx, _edx;
__asm__(
"call 1f\n"
".subsection 1\n"
"1:\tmovl (%%esp), %%ebx\n\t"
"ret\n"
".previous\n\t"
"addl $_GLOBAL_OFFSET_TABLE_, %%ebx\n\t"
"leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n\t"
"call ___tls_get_addr@plt\n\t"
"leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n"
: "=a" (ti), "=&b" (_ebx), "=&c" (_ecx), "=&d" (_edx));
tsrm_tls_offset = ti[1];
tsrm_tls_index = ti[0] * 8;
#endif
}
# endif
#endif
return SUCCESS;
}
static ZEND_ATTRIBUTE_UNUSED int zend_jit_trap(dasm_State **Dst)
{
| int3
return 1;
}
static int zend_jit_align_func(dasm_State **Dst)
{
reuse_ip = 0;
delayed_call_chain = 0;
last_valid_opline = NULL;
use_last_vald_opline = 0;
track_last_valid_opline = 0;
jit_return_label = -1;
|.align 16
return 1;
}
static int zend_jit_prologue(dasm_State **Dst)
{
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| SUB_HYBRID_SPAD
} else if (GCC_GLOBAL_REGS) {
| sub r4, SPAD // stack alignment
} else {
| sub r4, NR_SPAD // stack alignment
| mov aword T2, FP // save FP
| mov aword T3, RX // save IP
| mov FP, FCARG1a
}
return 1;
}
static int zend_jit_label(dasm_State **Dst, unsigned int label)
{
|=>label:
return 1;
}
static int zend_jit_save_call_chain(dasm_State **Dst, uint32_t call_level)
{
| // call->prev_execute_data = EX(call);
if (call_level == 1) {
| mov aword EX:RX->prev_execute_data, 0
} else {
| mov r0, EX->call
| mov EX:RX->prev_execute_data, r0
}
| // EX(call) = call;
| mov EX->call, RX
delayed_call_chain = 0;
return 1;
}
static int zend_jit_set_ip(dasm_State **Dst, const zend_op *opline)
{
if (last_valid_opline == opline) {
zend_jit_use_last_valid_opline();
} else if (GCC_GLOBAL_REGS && last_valid_opline) {
zend_jit_use_last_valid_opline();
| ADD_IP (opline - last_valid_opline) * sizeof(zend_op);
} else {
| LOAD_IP_ADDR opline
}
zend_jit_set_last_valid_opline(opline);
return 1;
}
static int zend_jit_set_ip_ex(dasm_State **Dst, const zend_op *opline, bool set_ip_reg)
{
if (last_valid_opline == opline) {
zend_jit_use_last_valid_opline();
} else if (GCC_GLOBAL_REGS && last_valid_opline) {
zend_jit_use_last_valid_opline();
| ADD_IP (opline - last_valid_opline) * sizeof(zend_op);
} else if (!GCC_GLOBAL_REGS && set_ip_reg) {
| LOAD_ADDR RX, opline
| mov aword EX->opline, RX
} else {
| LOAD_IP_ADDR opline
}
zend_jit_set_last_valid_opline(opline);
return 1;
}
static int zend_jit_set_valid_ip(dasm_State **Dst, const zend_op *opline)
{
if (delayed_call_chain) {
if (!zend_jit_save_call_chain(Dst, delayed_call_level)) {
return 0;
}
}
if (!zend_jit_set_ip(Dst, opline)) {
return 0;
}
reuse_ip = 0;
return 1;
}
static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline, const void *exit_addr)
{
#if 0
if (!zend_jit_set_valid_ip(Dst, opline)) {
return 0;
}
| MEM_OP2_1_ZTS cmp, byte, executor_globals, vm_interrupt, 0, r0
| jne ->interrupt_handler
#else
| MEM_OP2_1_ZTS cmp, byte, executor_globals, vm_interrupt, 0, r0
if (exit_addr) {
| jne &exit_addr
} else if (last_valid_opline == opline) {
|| zend_jit_use_last_valid_opline();
| jne ->interrupt_handler
} else {
| jne >1
|.cold_code
|1:
| LOAD_IP_ADDR opline
| jmp ->interrupt_handler
|.code
}
#endif
return 1;
}
static int zend_jit_trace_end_loop(dasm_State **Dst, int loop_label, const void *timeout_exit_addr)
{
if (timeout_exit_addr) {
| MEM_OP2_1_ZTS cmp, byte, executor_globals, vm_interrupt, 0, r0
| je =>loop_label
| jmp &timeout_exit_addr
} else {
| jmp =>loop_label
}
return 1;
}
static int zend_jit_check_exception(dasm_State **Dst)
{
| MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0
| jne ->exception_handler
return 1;
}
static int zend_jit_check_exception_undef_result(dasm_State **Dst, const zend_op *opline)
{
if (opline->result_type & (IS_TMP_VAR|IS_VAR)) {
| MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0
| jne ->exception_handler_undef
return 1;
}
return zend_jit_check_exception(Dst);
}
static int zend_jit_trace_begin(dasm_State **Dst, uint32_t trace_num, zend_jit_trace_info *parent, uint32_t exit_num)
{
zend_regset regset = ZEND_REGSET_SCRATCH;
#if ZTS
if (1) {
#else
if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(jit_trace_num)))) {
#endif
/* assignment to EG(jit_trace_num) shouldn't clober CPU register used by deoptimizer */
if (parent) {
int i;
int parent_vars_count = parent->exit_info[exit_num].stack_size;
zend_jit_trace_stack *parent_stack =
parent->stack_map +
parent->exit_info[exit_num].stack_offset;
for (i = 0; i < parent_vars_count; i++) {
if (STACK_REG(parent_stack, i) != ZREG_NONE) {
if (STACK_REG(parent_stack, i) < ZREG_NUM) {
ZEND_REGSET_EXCL(regset, STACK_REG(parent_stack, i));
} else if (STACK_REG(parent_stack, i) == ZREG_ZVAL_COPY_R0) {
ZEND_REGSET_EXCL(regset, ZREG_R0);
}
}
}
}
}
if (parent && parent->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) {
ZEND_REGSET_EXCL(regset, ZREG_R0);
}
current_trace_num = trace_num;
| // EG(jit_trace_num) = trace_num;
if (regset == ZEND_REGSET_EMPTY) {
| push r0
| MEM_OP2_1_ZTS mov, dword, executor_globals, jit_trace_num, trace_num, r0
| pop r0
} else {
zend_reg tmp = ZEND_REGSET_FIRST(regset);
| MEM_OP2_1_ZTS mov, dword, executor_globals, jit_trace_num, trace_num, Ra(tmp)
(void)tmp;
}
return 1;
}
/* This taken from LuaJIT. Thanks to Mike Pall. */
static uint32_t _asm_x86_inslen(const uint8_t* p)
{
static const uint8_t map_op1[256] = {
0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x20,
0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51,
0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51,
0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51,
#if defined(__x86_64__) || defined(_M_X64)
0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x14,0x14,0x14,0x14,0x14,0x14,0x14,0x14,
#else
0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,
#endif
0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,
0x51,0x51,0x92,0x92,0x10,0x10,0x12,0x11,0x45,0x86,0x52,0x93,0x51,0x51,0x51,0x51,
0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,
0x93,0x86,0x93,0x93,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,
0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x47,0x51,0x51,0x51,0x51,0x51,
#if defined(__x86_64__) || defined(_M_X64)
0x59,0x59,0x59,0x59,0x51,0x51,0x51,0x51,0x52,0x45,0x51,0x51,0x51,0x51,0x51,0x51,
#else
0x55,0x55,0x55,0x55,0x51,0x51,0x51,0x51,0x52,0x45,0x51,0x51,0x51,0x51,0x51,0x51,
#endif
0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x05,0x05,0x05,0x05,0x05,0x05,0x05,0x05,
0x93,0x93,0x53,0x51,0x70,0x71,0x93,0x86,0x54,0x51,0x53,0x51,0x51,0x52,0x51,0x51,
0x92,0x92,0x92,0x92,0x52,0x52,0x51,0x51,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,
0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x45,0x45,0x47,0x52,0x51,0x51,0x51,0x51,
0x10,0x51,0x10,0x10,0x51,0x51,0x63,0x66,0x51,0x51,0x51,0x51,0x51,0x51,0x92,0x92
};
static const uint8_t map_op2[256] = {
0x93,0x93,0x93,0x93,0x52,0x52,0x52,0x52,0x52,0x52,0x51,0x52,0x51,0x93,0x52,0x94,
0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
0x53,0x53,0x53,0x53,0x53,0x53,0x53,0x53,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x34,0x51,0x35,0x51,0x51,0x51,0x51,0x51,
0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
0x53,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
0x94,0x54,0x54,0x54,0x93,0x93,0x93,0x52,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,
0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
0x52,0x52,0x52,0x93,0x94,0x93,0x51,0x51,0x52,0x52,0x52,0x93,0x94,0x93,0x93,0x93,
0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x94,0x93,0x93,0x93,0x93,0x93,
0x93,0x93,0x94,0x93,0x94,0x94,0x94,0x93,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,
0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x52
};
uint32_t result = 0;
uint32_t prefixes = 0;
uint32_t x = map_op1[*p];
for (;;) {
switch (x >> 4) {
case 0:
return result + x + (prefixes & 4);
case 1:
prefixes |= x;
x = map_op1[*++p];
result++;
break;
case 2:
x = map_op2[*++p];
break;
case 3:
p++;
goto mrm;
case 4:
result -= (prefixes & 2);
/* fallthrough */
case 5:
return result + (x & 15);
case 6: /* Group 3. */
if (p[1] & 0x38) {
x = 2;
} else if ((prefixes & 2) && (x == 0x66)) {
x = 4;
}
goto mrm;
case 7: /* VEX c4/c5. */
#if !defined(__x86_64__) && !defined(_M_X64)
if (p[1] < 0xc0) {
x = 2;
goto mrm;
}
#endif
if (x == 0x70) {
x = *++p & 0x1f;
result++;
if (x >= 2) {
p += 2;
result += 2;
goto mrm;
}
}
p++;
result++;
x = map_op2[*++p];
break;
case 8:
result -= (prefixes & 2);
/* fallthrough */
case 9:
mrm:
/* ModR/M and possibly SIB. */
result += (x & 15);
x = *++p;
switch (x >> 6) {
case 0:
if ((x & 7) == 5) {
return result + 4;
}
break;
case 1:
result++;
break;
case 2:
result += 4;
break;
case 3:
return result;
}
if ((x & 7) == 4) {
result++;
if (x < 0x40 && (p[1] & 7) == 5) {
result += 4;
}
}
return result;
}
}
}
typedef ZEND_SET_ALIGNED(1, uint16_t unaligned_uint16_t);
typedef ZEND_SET_ALIGNED(1, int32_t unaligned_int32_t);
static int zend_jit_patch(const void *code, size_t size, uint32_t jmp_table_size, const void *from_addr, const void *to_addr)
{
int ret = 0;
uint8_t *p, *end;
if (jmp_table_size) {
const void **jmp_slot = (const void **)((char*)code + size);
size -= jmp_table_size * sizeof(void*);
do {
jmp_slot--;
if (*jmp_slot == from_addr) {
*jmp_slot = to_addr;
ret++;
}
} while (--jmp_table_size);
}
p = (uint8_t*)code;
end = p + size - 5;
while (p < end) {
if ((*(unaligned_uint16_t*)p & 0xf0ff) == 0x800f && p + *(unaligned_int32_t*)(p+2) == (uint8_t*)from_addr - 6) {
*(unaligned_int32_t*)(p+2) = ((uint8_t*)to_addr - (p + 6));
ret++;
} else if (*p == 0xe9 && p + *(unaligned_int32_t*)(p+1) == (uint8_t*)from_addr - 5) {
*(unaligned_int32_t*)(p+1) = ((uint8_t*)to_addr - (p + 5));
ret++;
}
p += _asm_x86_inslen(p);
}
#ifdef HAVE_VALGRIND
VALGRIND_DISCARD_TRANSLATIONS(code, size);
#endif
return ret;
}
static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t jmp_table_size, uint32_t exit_num, const void *addr)
{
return zend_jit_patch(code, size, jmp_table_size, zend_jit_trace_get_exit_addr(exit_num), addr);
}
static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t, const void *timeout_exit_addr)
{
const void *link_addr;
size_t prologue_size;
/* Skip prologue. */
// TODO: don't hardcode this ???
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
prologue_size = 0;
#elif defined(__x86_64__) || defined(_M_X64)
// sub r4, HYBRID_SPAD
prologue_size = 4;
#else
// sub r4, HYBRID_SPAD
prologue_size = 3;
#endif
} else if (GCC_GLOBAL_REGS) {
// sub r4, SPAD // stack alignment
#if defined(__x86_64__) || defined(_M_X64)
prologue_size = 4;
#else
prologue_size = 3;
#endif
} else {
// sub r4, NR_SPAD // stack alignment
// mov aword T2, FP // save FP
// mov aword T3, RX // save IP
// mov FP, FCARG1a
#if defined(__x86_64__) || defined(_M_X64)
prologue_size = 17;
#else
prologue_size = 13;
#endif
}
link_addr = (const void*)((const char*)t->code_start + prologue_size);
if (timeout_exit_addr) {
/* Check timeout for links to LOOP */
| MEM_OP2_1_ZTS cmp, byte, executor_globals, vm_interrupt, 0, r0
| je &link_addr
| jmp &timeout_exit_addr
} else {
| jmp &link_addr
}
return 1;
}
static int zend_jit_trace_return(dasm_State **Dst, zend_bool original_handler, const zend_op *opline)
{
#if 0
| jmp ->trace_escape
#else
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| ADD_HYBRID_SPAD
if (!original_handler) {
| JMP_IP
} else {
| mov r0, EX->func
| mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])]
| mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)]
| jmp aword [IP + r0]
}
} else if (GCC_GLOBAL_REGS) {
| add r4, SPAD // stack alignment
if (!original_handler) {
| JMP_IP
} else {
| mov r0, EX->func
| mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])]
| mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)]
| jmp aword [IP + r0]
}
} else {
if (original_handler) {
| mov FCARG1a, FP
| mov r0, EX->func
| mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])]
| mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)]
| call aword [IP + r0]
}
| mov FP, aword T2 // restore FP
| mov RX, aword T3 // restore IP
| add r4, NR_SPAD // stack alignment
if (!original_handler || !opline ||
(opline->opcode != ZEND_RETURN
&& opline->opcode != ZEND_RETURN_BY_REF
&& opline->opcode != ZEND_GENERATOR_RETURN
&& opline->opcode != ZEND_GENERATOR_CREATE
&& opline->opcode != ZEND_YIELD
&& opline->opcode != ZEND_YIELD_FROM)) {
| mov r0, 2 // ZEND_VM_LEAVE
}
| ret
}
#endif
return 1;
}
static int zend_jit_type_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint8_t type)
{
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_NOT_Z_TYPE FP + var, type, &exit_addr
return 1;
}
static int zend_jit_packed_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint32_t op_info)
{
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| GET_ZVAL_LVAL ZREG_FCARG1a, ZEND_ADDR_MEM_ZVAL(ZREG_FP, var)
if (op_info & MAY_BE_ARRAY_PACKED) {
| test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED
| jz &exit_addr
} else {
| test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED
| jnz &exit_addr
}
return 1;
}
static int zend_jit_trace_handler(dasm_State **Dst, const zend_op_array *op_array, const zend_op *opline, int may_throw, zend_jit_trace_rec *trace)
{
zend_jit_op_array_trace_extension *jit_extension =
(zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
size_t offset = jit_extension->offset;
const void *handler =
(zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler;
if (!zend_jit_set_valid_ip(Dst, opline)) {
return 0;
}
if (!GCC_GLOBAL_REGS) {
| mov FCARG1a, FP
}
| EXT_CALL handler, r0
if (may_throw
&& opline->opcode != ZEND_RETURN
&& opline->opcode != ZEND_RETURN_BY_REF) {
| MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r1
| jne ->exception_handler
}
while (trace->op != ZEND_JIT_TRACE_VM && trace->op != ZEND_JIT_TRACE_END) {
trace++;
}
if (!GCC_GLOBAL_REGS
&& (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_RETURN)) {
if (opline->opcode == ZEND_RETURN ||
opline->opcode == ZEND_RETURN_BY_REF ||
opline->opcode == ZEND_DO_UCALL ||
opline->opcode == ZEND_DO_FCALL_BY_NAME ||
opline->opcode == ZEND_DO_FCALL ||
opline->opcode == ZEND_GENERATOR_CREATE) {
| MEM_OP2_2_ZTS mov, FP, aword, executor_globals, current_execute_data, r1
}
}
if (zend_jit_trace_may_exit(op_array, opline)) {
if (opline->opcode == ZEND_RETURN ||
opline->opcode == ZEND_RETURN_BY_REF ||
opline->opcode == ZEND_GENERATOR_CREATE) {
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
#if 0
/* this check should be handled by the following OPLINE guard or jmp [IP] */
| cmp IP, zend_jit_halt_op
| je ->trace_halt
#endif
} else if (GCC_GLOBAL_REGS) {
| test IP, IP
| je ->trace_halt
} else {
| test eax, eax
| jl ->trace_halt
}
} else if (opline->opcode == ZEND_EXIT ||
opline->opcode == ZEND_GENERATOR_RETURN ||
opline->opcode == ZEND_YIELD ||
opline->opcode == ZEND_YIELD_FROM) {
| jmp ->trace_halt
}
if (trace->op != ZEND_JIT_TRACE_END ||
(trace->stop != ZEND_JIT_TRACE_STOP_RETURN &&
trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) {
const zend_op *next_opline = trace->opline;
const zend_op *exit_opline = NULL;
uint32_t exit_point;
const void *exit_addr;
uint32_t old_info = 0;
uint32_t old_res_info = 0;
zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
if (zend_is_smart_branch(opline)) {
zend_bool exit_if_true = 0;
exit_opline = zend_jit_trace_get_exit_opline(trace, opline + 1, &exit_if_true);
} else {
switch (opline->opcode) {
case ZEND_JMPZ:
case ZEND_JMPNZ:
case ZEND_JMPZ_EX:
case ZEND_JMPNZ_EX:
case ZEND_JMP_SET:
case ZEND_COALESCE:
case ZEND_JMP_NULL:
case ZEND_FE_RESET_R:
case ZEND_FE_RESET_RW:
exit_opline = (trace->opline == opline + 1) ?
OP_JMP_ADDR(opline, opline->op2) :
opline + 1;
break;
case ZEND_JMPZNZ:
exit_opline = (trace->opline == OP_JMP_ADDR(opline, opline->op2)) ?
ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) :
OP_JMP_ADDR(opline, opline->op2);
break;
case ZEND_FE_FETCH_R:
case ZEND_FE_FETCH_RW:
exit_opline = (trace->opline == opline + 1) ?
ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) :
opline + 1;
break;
}
}
switch (opline->opcode) {
case ZEND_FE_FETCH_R:
case ZEND_FE_FETCH_RW:
if (opline->op2_type != IS_UNUSED) {
old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var));
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var), IS_UNKNOWN, 1);
}
break;
}
if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) {
old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
}
exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) {
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
}
switch (opline->opcode) {
case ZEND_FE_FETCH_R:
case ZEND_FE_FETCH_RW:
if (opline->op2_type != IS_UNUSED) {
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var), old_info);
}
break;
}
if (!exit_addr) {
return 0;
}
| CMP_IP next_opline
| jne &exit_addr
}
}
zend_jit_set_last_valid_opline(trace->opline);
return 1;
}
static int zend_jit_handler(dasm_State **Dst, const zend_op *opline, int may_throw)
{
const void *handler;
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
handler = zend_get_opcode_handler_func(opline);
} else {
handler = opline->handler;
}
if (!zend_jit_set_valid_ip(Dst, opline)) {
return 0;
}
if (!GCC_GLOBAL_REGS) {
| mov FCARG1a, FP
}
| EXT_CALL handler, r0
if (may_throw) {
zend_jit_check_exception(Dst);
}
/* Skip the following OP_DATA */
switch (opline->opcode) {
case ZEND_ASSIGN_DIM:
case ZEND_ASSIGN_OBJ:
case ZEND_ASSIGN_STATIC_PROP:
case ZEND_ASSIGN_DIM_OP:
case ZEND_ASSIGN_OBJ_OP:
case ZEND_ASSIGN_STATIC_PROP_OP:
case ZEND_ASSIGN_STATIC_PROP_REF:
case ZEND_ASSIGN_OBJ_REF:
zend_jit_set_last_valid_opline(opline + 2);
break;
default:
zend_jit_set_last_valid_opline(opline + 1);
break;
}
return 1;
}
static int zend_jit_tail_handler(dasm_State **Dst, const zend_op *opline)
{
if (!zend_jit_set_valid_ip(Dst, opline)) {
return 0;
}
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
if (opline->opcode == ZEND_DO_UCALL ||
opline->opcode == ZEND_DO_FCALL_BY_NAME ||
opline->opcode == ZEND_DO_FCALL ||
opline->opcode == ZEND_RETURN) {
/* Use inlined HYBRID VM handler */
const void *handler = opline->handler;
| ADD_HYBRID_SPAD
| EXT_JMP handler, r0
} else {
const void *handler = zend_get_opcode_handler_func(opline);
| EXT_CALL handler, r0
| ADD_HYBRID_SPAD
| JMP_IP
}
} else {
const void *handler = opline->handler;
if (GCC_GLOBAL_REGS) {
| add r4, SPAD // stack alignment
} else {
| mov FCARG1a, FP
| mov FP, aword T2 // restore FP
| mov RX, aword T3 // restore IP
| add r4, NR_SPAD // stack alignment
}
| EXT_JMP handler, r0
}
zend_jit_reset_last_valid_opline();
return 1;
}
static int zend_jit_trace_opline_guard(dasm_State **Dst, const zend_op *opline)
{
uint32_t exit_point = zend_jit_trace_get_exit_point(NULL, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| CMP_IP opline
| jne &exit_addr
zend_jit_set_last_valid_opline(opline);
return 1;
}
static int zend_jit_jmp(dasm_State **Dst, unsigned int target_label)
{
| jmp =>target_label
return 1;
}
static int zend_jit_cond_jmp(dasm_State **Dst, const zend_op *next_opline, unsigned int target_label)
{
| CMP_IP next_opline
| jne =>target_label
zend_jit_set_last_valid_opline(next_opline);
return 1;
}
#ifdef CONTEXT_THREADED_JIT
static int zend_jit_context_threaded_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block)
{
if (!zend_jit_handler(Dst, opline, 1)) return 0;
if (opline->opcode == ZEND_DO_UCALL) {
| call ->context_threaded_call
} else {
const zend_op *next_opline = opline + 1;
| CMP_IP next_opline
| je =>next_block
| call ->context_threaded_call
}
return 1;
}
#endif
static int zend_jit_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block)
{
#ifdef CONTEXT_THREADED_JIT
return zend_jit_context_threaded_call(Dst, opline, next_block);
#else
return zend_jit_tail_handler(Dst, opline);
#endif
}
static int zend_jit_spill_store(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info, zend_bool set_type)
{
ZEND_ASSERT(Z_MODE(src) == IS_REG);
ZEND_ASSERT(Z_MODE(dst) == IS_MEM_ZVAL);
if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
| SET_ZVAL_LVAL dst, Ra(Z_REG(src))
if (set_type) {
| SET_ZVAL_TYPE_INFO dst, IS_LONG
}
} else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
| SSE_SET_ZVAL_DVAL dst, Z_REG(src)
if (set_type) {
| SET_ZVAL_TYPE_INFO dst, IS_DOUBLE
}
} else {
ZEND_UNREACHABLE();
}
return 1;
}
static int zend_jit_load_reg(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info)
{
ZEND_ASSERT(Z_MODE(src) == IS_MEM_ZVAL);
ZEND_ASSERT(Z_MODE(dst) == IS_REG);
if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
| GET_ZVAL_LVAL Z_REG(dst), src
} else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
| SSE_GET_ZVAL_DVAL Z_REG(dst), src
} else {
ZEND_UNREACHABLE();
}
return 1;
}
static int zend_jit_store_var(dasm_State **Dst, uint32_t info, int var, zend_reg reg, zend_bool set_type)
{
zend_jit_addr src = ZEND_ADDR_REG(reg);
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
return zend_jit_spill_store(Dst, src, dst, info, set_type);
}
static int zend_jit_store_var_type(dasm_State **Dst, int var, uint8_t type)
{
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
| SET_ZVAL_TYPE_INFO dst, type
return 1;
}
static int zend_jit_store_var_if_necessary(dasm_State **Dst, int var, zend_jit_addr src, uint32_t info)
{
if (Z_MODE(src) == IS_REG && Z_STORE(src)) {
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
return zend_jit_spill_store(Dst, src, dst, info, 1);
}
return 1;
}
static int zend_jit_store_var_if_necessary_ex(dasm_State **Dst, int var, zend_jit_addr src, uint32_t info, zend_jit_addr old, uint32_t old_info)
{
if (Z_MODE(src) == IS_REG && Z_STORE(src)) {
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
zend_bool set_type = 1;
if ((info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) ==
(old_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF))) {
if (Z_MODE(old) != IS_REG || Z_LOAD(old) || Z_STORE(old)) {
set_type = 0;
}
}
return zend_jit_spill_store(Dst, src, dst, info, set_type);
}
return 1;
}
static int zend_jit_load_var(dasm_State **Dst, uint32_t info, int var, zend_reg reg)
{
zend_jit_addr src = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
zend_jit_addr dst = ZEND_ADDR_REG(reg);
return zend_jit_load_reg(Dst, src, dst, info);
}
static int zend_jit_invalidate_var_if_necessary(dasm_State **Dst, zend_uchar op_type, zend_jit_addr addr, znode_op op)
{
if ((op_type & (IS_TMP_VAR|IS_VAR)) && Z_MODE(addr) == IS_REG && !Z_LOAD(addr) && !Z_STORE(addr)) {
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var);
| SET_ZVAL_TYPE_INFO dst, IS_UNDEF
}
return 1;
}
static int zend_jit_update_regs(dasm_State **Dst, uint32_t var, zend_jit_addr src, zend_jit_addr dst, uint32_t info)
{
if (!zend_jit_same_addr(src, dst)) {
if (Z_MODE(src) == IS_REG) {
if (Z_MODE(dst) == IS_REG) {
if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
| mov Ra(Z_REG(dst)), Ra(Z_REG(src))
} else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
| SSE_AVX_INS movaps, vmovaps, xmm(Z_REG(dst)-ZREG_XMM0), xmm(Z_REG(src)-ZREG_XMM0)
} else {
ZEND_UNREACHABLE();
}
if (!Z_LOAD(src) && !Z_STORE(src) && Z_STORE(dst)) {
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
if (!zend_jit_spill_store(Dst, dst, var_addr, info,
JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
JIT_G(current_frame) == NULL ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN ||
(1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY)
)) {
return 0;
}
}
} else if (Z_MODE(dst) == IS_MEM_ZVAL) {
if (!Z_LOAD(src) && !Z_STORE(src)) {
if (!zend_jit_spill_store(Dst, src, dst, info,
JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
JIT_G(current_frame) == NULL ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN ||
(1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY)
)) {
return 0;
}
}
} else {
ZEND_UNREACHABLE();
}
} else if (Z_MODE(src) == IS_MEM_ZVAL) {
if (Z_MODE(dst) == IS_REG) {
if (!zend_jit_load_reg(Dst, src, dst, info)) {
return 0;
}
} else {
ZEND_UNREACHABLE();
}
} else {
ZEND_UNREACHABLE();
}
} else if (Z_MODE(dst) == IS_REG && Z_STORE(dst)) {
dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
if (!zend_jit_spill_store(Dst, src, dst, info,
JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
JIT_G(current_frame) == NULL ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN ||
(1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY)
)) {
return 0;
}
}
return 1;
}
static int zend_jit_escape_if_undef_r0(dasm_State **Dst, int var, uint32_t flags, const zend_op *opline)
{
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0);
| IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >1
if (flags & ZEND_JIT_EXIT_RESTORE_CALL) {
if (!zend_jit_save_call_chain(Dst, -1)) {
return 0;
}
}
ZEND_ASSERT(opline);
if ((opline-1)->opcode != ZEND_FETCH_CONSTANT
&& (opline-1)->opcode != ZEND_FETCH_LIST_R
&& ((opline-1)->op1_type & (IS_VAR|IS_TMP_VAR))
&& !(flags & ZEND_JIT_EXIT_FREE_OP1)) {
val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline-1)->op1.var);
| IF_NOT_ZVAL_REFCOUNTED val_addr, >2
| GET_ZVAL_PTR r0, val_addr
| GC_ADDREF r0
|2:
}
| LOAD_IP_ADDR (opline - 1)
| jmp ->trace_escape
|1:
return 1;
}
static int zend_jit_store_const(dasm_State **Dst, int var, zend_reg reg)
{
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
if (reg == ZREG_LONG_MIN_MINUS_1) {
|.if X64
| SET_ZVAL_LVAL dst, 0x00000000
| SET_ZVAL_W2 dst, 0xc3e00000
|.else
| SET_ZVAL_LVAL dst, 0x00200000
| SET_ZVAL_W2 dst, 0xc1e00000
|.endif
| SET_ZVAL_TYPE_INFO dst, IS_DOUBLE
} else if (reg == ZREG_LONG_MIN) {
|.if X64
| SET_ZVAL_LVAL dst, 0x00000000
| SET_ZVAL_W2 dst, 0x80000000
|.else
| SET_ZVAL_LVAL dst, ZEND_LONG_MIN
|.endif
| SET_ZVAL_TYPE_INFO dst, IS_LONG
} else if (reg == ZREG_LONG_MAX) {
|.if X64
| SET_ZVAL_LVAL dst, 0xffffffff
| SET_ZVAL_W2 dst, 0x7fffffff
|.else
| SET_ZVAL_LVAL dst, ZEND_LONG_MAX
|.endif
| SET_ZVAL_TYPE_INFO dst, IS_LONG
} else if (reg == ZREG_LONG_MAX_PLUS_1) {
|.if X64
| SET_ZVAL_LVAL dst, 0
| SET_ZVAL_W2 dst, 0x43e00000
|.else
| SET_ZVAL_LVAL dst, 0
| SET_ZVAL_W2 dst, 0x41e00000
|.endif
| SET_ZVAL_TYPE_INFO dst, IS_DOUBLE
} else if (reg == ZREG_NULL) {
| SET_ZVAL_TYPE_INFO dst, IS_NULL
} else if (reg == ZREG_ZVAL_TRY_ADDREF) {
| IF_NOT_ZVAL_REFCOUNTED dst, >1
| GET_ZVAL_PTR r1, dst
| GC_ADDREF r1
|1:
} else if (reg == ZREG_ZVAL_COPY_R0) {
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0);
| ZVAL_COPY_VALUE dst, -1, val_addr, -1, ZREG_R1, ZREG_R2
| TRY_ADDREF -1, ch, r2
} else {
ZEND_UNREACHABLE();
}
return 1;
}
static int zend_jit_free_trampoline(dasm_State **Dst)
{
| /// if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE))
| test dword [r0 + offsetof(zend_function, common.fn_flags)], ZEND_ACC_CALL_VIA_TRAMPOLINE
| jz >1
| mov FCARG1a, r0
| EXT_CALL zend_jit_free_trampoline_helper, r0
|1:
return 1;
}
static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op1_def_info, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw)
{
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_LONG)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2
}
if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) {
| ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1
}
if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, MAY_BE_LONG)) {
return 0;
}
if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
| LONG_OP_WITH_32BIT_CONST add, op1_def_addr, Z_L(1)
} else {
| LONG_OP_WITH_32BIT_CONST sub, op1_def_addr, Z_L(1)
}
if (may_overflow &&
(((op1_def_info & MAY_BE_GUARD) && (op1_def_info & MAY_BE_LONG)) ||
((opline->result_type != IS_UNUSED && (res_info & MAY_BE_GUARD) && (res_info & MAY_BE_LONG))))) {
int32_t exit_point;
const void *exit_addr;
zend_jit_trace_stack *stack;
uint32_t old_op1_info, old_res_info = 0;
stack = JIT_G(current_frame)->stack;
old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_DOUBLE, 0);
if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_LONG_MAX_PLUS_1);
} else {
SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_LONG_MIN_MINUS_1);
}
if (opline->result_type != IS_UNUSED) {
old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
if (opline->opcode == ZEND_PRE_INC) {
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0);
SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MAX_PLUS_1);
} else if (opline->opcode == ZEND_PRE_DEC) {
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0);
SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MIN_MINUS_1);
} else if (opline->opcode == ZEND_POST_INC) {
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0);
SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MAX);
} else if (opline->opcode == ZEND_POST_DEC) {
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0);
SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MIN);
}
}
exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
| jo &exit_addr
if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
opline->result_type != IS_UNUSED) {
| ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1
}
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info);
if (opline->result_type != IS_UNUSED) {
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
}
} else if (may_overflow) {
| jo >1
if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
opline->result_type != IS_UNUSED) {
| ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1
}
|.cold_code
|1:
if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
|.if X64
| mov64 rax, 0x43e0000000000000
| SET_ZVAL_LVAL op1_def_addr, rax
|.else
| SET_ZVAL_LVAL op1_def_addr, 0
| SET_ZVAL_W2 op1_def_addr, 0x41e00000
|.endif
} else {
|.if X64
| mov64 rax, 0xc3e0000000000000
| SET_ZVAL_LVAL op1_def_addr, rax
|.else
| SET_ZVAL_LVAL op1_def_addr, 0x00200000
| SET_ZVAL_W2 op1_def_addr, 0xc1e00000
|.endif
}
if (Z_MODE(op1_def_addr) == IS_MEM_ZVAL) {
| SET_ZVAL_TYPE_INFO op1_def_addr, IS_DOUBLE
}
if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
opline->result_type != IS_UNUSED) {
| ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_DOUBLE, ZREG_R0, ZREG_R1
}
| jmp >3
|.code
} else {
if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
opline->result_type != IS_UNUSED) {
| ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1
}
}
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
|.cold_code
|2:
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
| SET_EX_OPLINE opline, r0
if (op1_info & MAY_BE_UNDEF) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >2
| // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
| mov FCARG1d, opline->op1.var
| SET_ZVAL_TYPE_INFO op1_addr, IS_NULL
| EXT_CALL zend_jit_undefined_op_helper, r0
op1_info |= MAY_BE_NULL;
}
|2:
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
| // ZVAL_DEREF(var_ptr);
if (op1_info & MAY_BE_REF) {
| IF_NOT_Z_TYPE, FCARG1a, IS_REFERENCE, >2
| GET_Z_PTR FCARG1a, FCARG1a
| cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0
| jz >1
if (RETURN_VALUE_USED(opline)) {
| LOAD_ZVAL_ADDR FCARG2a, res_addr
} else {
| xor FCARG2a, FCARG2a
}
if (opline->opcode == ZEND_PRE_INC) {
| EXT_CALL zend_jit_pre_inc_typed_ref, r0
} else if (opline->opcode == ZEND_PRE_DEC) {
| EXT_CALL zend_jit_pre_dec_typed_ref, r0
} else if (opline->opcode == ZEND_POST_INC) {
| EXT_CALL zend_jit_post_inc_typed_ref, r0
} else if (opline->opcode == ZEND_POST_DEC) {
| EXT_CALL zend_jit_post_dec_typed_ref, r0
} else {
ZEND_UNREACHABLE();
}
zend_jit_check_exception(Dst);
| jmp >3
|1:
| lea FCARG1a, [FCARG1a + offsetof(zend_reference, val)]
|2:
}
if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) {
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
| ZVAL_COPY_VALUE res_addr, res_use_info, val_addr, op1_info, ZREG_R0, ZREG_R2
| TRY_ADDREF op1_info, ah, r2
}
if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
if (opline->opcode == ZEND_PRE_INC && opline->result_type != IS_UNUSED) {
| LOAD_ZVAL_ADDR FCARG2a, res_addr
| EXT_CALL zend_jit_pre_inc, r0
} else {
| EXT_CALL increment_function, r0
}
} else {
if (opline->opcode == ZEND_PRE_DEC && opline->result_type != IS_UNUSED) {
| LOAD_ZVAL_ADDR FCARG2a, res_addr
| EXT_CALL zend_jit_pre_dec, r0
} else {
| EXT_CALL decrement_function, r0
}
}
if (may_throw) {
zend_jit_check_exception(Dst);
}
} else {
zend_reg tmp_reg;
if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) {
| ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_DOUBLE, ZREG_R0, ZREG_R2
}
if (Z_MODE(op1_def_addr) == IS_REG) {
tmp_reg = Z_REG(op1_def_addr);
} else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) {
tmp_reg = Z_REG(op1_addr);
} else {
tmp_reg = ZREG_XMM0;
}
| SSE_GET_ZVAL_DVAL tmp_reg, op1_addr
if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
if (CAN_USE_AVX()) {
| vaddsd xmm(tmp_reg-ZREG_XMM0), xmm(tmp_reg-ZREG_XMM0), qword [->one]
} else {
| addsd xmm(tmp_reg-ZREG_XMM0), qword [->one]
}
} else {
if (CAN_USE_AVX()) {
| vsubsd xmm(tmp_reg-ZREG_XMM0), xmm(tmp_reg-ZREG_XMM0), qword [->one]
} else {
| subsd xmm(tmp_reg-ZREG_XMM0), qword [->one]
}
}
| SSE_SET_ZVAL_DVAL op1_def_addr, tmp_reg
if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
opline->result_type != IS_UNUSED) {
| ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, op1_def_info, ZREG_R0, ZREG_R1
| TRY_ADDREF op1_def_info, ah, r1
}
}
| jmp >3
|.code
}
|3:
if (!zend_jit_store_var_if_necessary_ex(Dst, opline->op1.var, op1_def_addr, op1_def_info, op1_addr, op1_info)) {
return 0;
}
if (opline->result_type != IS_UNUSED) {
if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) {
return 0;
}
}
return 1;
}
static int zend_jit_opline_uses_reg(const zend_op *opline, int8_t reg)
{
if ((opline+1)->opcode == ZEND_OP_DATA
&& ((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV))
&& JIT_G(current_frame)->stack[EX_VAR_TO_NUM((opline+1)->op1.var)].reg == reg) {
return 1;
}
return
((opline->result_type & (IS_VAR|IS_TMP_VAR|IS_CV)) &&
JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->result.var)].reg == reg) ||
((opline->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) &&
JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->op1.var)].reg == reg) ||
((opline->op2_type & (IS_VAR|IS_TMP_VAR|IS_CV)) &&
JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->op2.var)].reg == reg);
}
static int zend_jit_math_long_long(dasm_State **Dst,
const zend_op *opline,
zend_uchar opcode,
zend_jit_addr op1_addr,
zend_jit_addr op2_addr,
zend_jit_addr res_addr,
uint32_t res_info,
uint32_t res_use_info,
int may_overflow)
{
zend_bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
zend_reg result_reg;
zend_reg tmp_reg = ZREG_R0;
if (Z_MODE(res_addr) == IS_REG && (res_info & MAY_BE_LONG)) {
if (may_overflow && (res_info & MAY_BE_GUARD)
&& JIT_G(current_frame)
&& zend_jit_opline_uses_reg(opline, Z_REG(res_addr))) {
result_reg = ZREG_R0;
} else {
result_reg = Z_REG(res_addr);
}
} else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr) && !may_overflow) {
result_reg = Z_REG(op1_addr);
} else if (Z_REG(res_addr) != ZREG_R0) {
result_reg = ZREG_R0;
} else {
/* ASSIGN_DIM_OP */
result_reg = ZREG_FCARG1a;
tmp_reg = ZREG_FCARG1a;
}
if (opcode == ZEND_MUL &&
Z_MODE(op2_addr) == IS_CONST_ZVAL &&
Z_LVAL_P(Z_ZV(op2_addr)) == 2) {
if (Z_MODE(op1_addr) == IS_REG && !may_overflow) {
| lea Ra(result_reg), [Ra(Z_REG(op1_addr))+Ra(Z_REG(op1_addr))]
} else {
| GET_ZVAL_LVAL result_reg, op1_addr
| add Ra(result_reg), Ra(result_reg)
}
} else if (opcode == ZEND_MUL &&
Z_MODE(op2_addr) == IS_CONST_ZVAL &&
!may_overflow &&
Z_LVAL_P(Z_ZV(op2_addr)) > 0 &&
zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr)))) {
| GET_ZVAL_LVAL result_reg, op1_addr
| shl Ra(result_reg), zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr)))
} else if (opcode == ZEND_MUL &&
Z_MODE(op1_addr) == IS_CONST_ZVAL &&
Z_LVAL_P(Z_ZV(op1_addr)) == 2) {
if (Z_MODE(op2_addr) == IS_REG && !may_overflow) {
| lea Ra(result_reg), [Ra(Z_REG(op2_addr))+Ra(Z_REG(op2_addr))]
} else {
| GET_ZVAL_LVAL result_reg, op2_addr
| add Ra(result_reg), Ra(result_reg)
}
} else if (opcode == ZEND_MUL &&
Z_MODE(op1_addr) == IS_CONST_ZVAL &&
!may_overflow &&
Z_LVAL_P(Z_ZV(op1_addr)) > 0 &&
zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))) {
| GET_ZVAL_LVAL result_reg, op2_addr
| shl Ra(result_reg), zend_long_floor_log2(Z_LVAL_P(Z_ZV(op1_addr)))
} else if (opcode == ZEND_DIV &&
(Z_MODE(op2_addr) == IS_CONST_ZVAL &&
zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr))))) {
| GET_ZVAL_LVAL result_reg, op1_addr
| shr Ra(result_reg), zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr)))
} else if (opcode == ZEND_ADD &&
!may_overflow &&
Z_MODE(op1_addr) == IS_REG &&
Z_MODE(op2_addr) == IS_CONST_ZVAL &&
IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op2_addr)))) {
| lea Ra(result_reg), [Ra(Z_REG(op1_addr))+Z_LVAL_P(Z_ZV(op2_addr))]
} else if (opcode == ZEND_ADD &&
!may_overflow &&
Z_MODE(op2_addr) == IS_REG &&
Z_MODE(op1_addr) == IS_CONST_ZVAL &&
IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op1_addr)))) {
| lea Ra(result_reg), [Ra(Z_REG(op2_addr))+Z_LVAL_P(Z_ZV(op1_addr))]
} else if (opcode == ZEND_SUB &&
!may_overflow &&
Z_MODE(op1_addr) == IS_REG &&
Z_MODE(op2_addr) == IS_CONST_ZVAL &&
IS_SIGNED_32BIT(-Z_LVAL_P(Z_ZV(op2_addr)))) {
| lea Ra(result_reg), [Ra(Z_REG(op1_addr))-Z_LVAL_P(Z_ZV(op2_addr))]
} else {
| GET_ZVAL_LVAL result_reg, op1_addr
if ((opcode == ZEND_ADD || opcode == ZEND_SUB)
&& Z_MODE(op2_addr) == IS_CONST_ZVAL
&& Z_LVAL_P(Z_ZV(op2_addr)) == 0) {
/* +/- 0 */
may_overflow = 0;
} else if (same_ops && opcode != ZEND_DIV) {
| LONG_MATH_REG opcode, Ra(result_reg), Ra(result_reg)
} else {
zend_reg tmp_reg;
if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) {
tmp_reg = ZREG_R1;
} else if (result_reg != ZREG_R0) {
tmp_reg = ZREG_R0;
} else {
tmp_reg = ZREG_R1;
}
| LONG_MATH opcode, result_reg, op2_addr, tmp_reg
(void)tmp_reg;
}
}
if (may_overflow) {
if (res_info & MAY_BE_GUARD) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if ((res_info & MAY_BE_ANY) == MAY_BE_LONG) {
| jo &exit_addr
if (Z_MODE(res_addr) == IS_REG && result_reg != Z_REG(res_addr)) {
| mov Ra(Z_REG(res_addr)), Ra(result_reg)
}
} else if ((res_info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
| jno &exit_addr
} else {
ZEND_UNREACHABLE();
}
} else {
if (res_info & MAY_BE_LONG) {
| jo >1
} else {
| jno >1
}
}
}
if (Z_MODE(res_addr) == IS_MEM_ZVAL && (res_info & MAY_BE_LONG)) {
| SET_ZVAL_LVAL res_addr, Ra(result_reg)
if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) {
if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) {
| SET_ZVAL_TYPE_INFO res_addr, IS_LONG
}
}
}
if (may_overflow && (!(res_info & MAY_BE_GUARD) || (res_info & MAY_BE_ANY) == MAY_BE_DOUBLE)) {
zend_reg tmp_reg1 = ZREG_XMM0;
zend_reg tmp_reg2 = ZREG_XMM1;
if (res_info & MAY_BE_LONG) {
|.cold_code
|1:
}
do {
if ((sizeof(void*) == 8 || Z_MODE(res_addr) != IS_REG) &&
((Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 1) ||
(Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1))) {
if (opcode == ZEND_ADD) {
|.if X64
| mov64 Ra(tmp_reg), 0x43e0000000000000
if (Z_MODE(res_addr) == IS_REG) {
| movd xmm(Z_REG(res_addr)-ZREG_XMM0), Ra(tmp_reg)
} else {
| SET_ZVAL_LVAL res_addr, Ra(tmp_reg)
}
|.else
| SET_ZVAL_LVAL res_addr, 0
| SET_ZVAL_W2 res_addr, 0x41e00000
|.endif
break;
} else if (opcode == ZEND_SUB) {
|.if X64
| mov64 Ra(tmp_reg), 0xc3e0000000000000
if (Z_MODE(res_addr) == IS_REG) {
| movd xmm(Z_REG(res_addr)-ZREG_XMM0), Ra(tmp_reg)
} else {
| SET_ZVAL_LVAL res_addr, Ra(tmp_reg)
}
|.else
| SET_ZVAL_LVAL res_addr, 0x00200000
| SET_ZVAL_W2 res_addr, 0xc1e00000
|.endif
break;
}
}
| SSE_GET_ZVAL_LVAL tmp_reg1, op1_addr, tmp_reg
| SSE_GET_ZVAL_LVAL tmp_reg2, op2_addr, tmp_reg
if (CAN_USE_AVX()) {
| AVX_MATH_REG opcode, tmp_reg1, tmp_reg1, tmp_reg2
} else {
| SSE_MATH_REG opcode, tmp_reg1, tmp_reg2
}
| SSE_SET_ZVAL_DVAL res_addr, tmp_reg1
} while (0);
if (Z_MODE(res_addr) == IS_MEM_ZVAL
&& (res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE
}
if (res_info & MAY_BE_LONG) {
| jmp >2
|.code
}
|2:
}
return 1;
}
static int zend_jit_math_long_double(dasm_State **Dst,
zend_uchar opcode,
zend_jit_addr op1_addr,
zend_jit_addr op2_addr,
zend_jit_addr res_addr,
uint32_t res_use_info)
{
zend_reg result_reg =
(Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_XMM0;
zend_reg tmp_reg;
if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) {
/* ASSIGN_DIM_OP */
tmp_reg = ZREG_R1;
} else {
tmp_reg = ZREG_R0;
}
| SSE_GET_ZVAL_LVAL result_reg, op1_addr, tmp_reg
if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) {
/* ASSIGN_DIM_OP */
if (CAN_USE_AVX()) {
| AVX_MATH opcode, result_reg, result_reg, op2_addr, r1
} else {
| SSE_MATH opcode, result_reg, op2_addr, r1
}
} else {
if (CAN_USE_AVX()) {
| AVX_MATH opcode, result_reg, result_reg, op2_addr, r0
} else {
| SSE_MATH opcode, result_reg, op2_addr, r0
}
}
| SSE_SET_ZVAL_DVAL res_addr, result_reg
if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE
}
}
return 1;
}
static int zend_jit_math_double_long(dasm_State **Dst,
zend_uchar opcode,
zend_jit_addr op1_addr,
zend_jit_addr op2_addr,
zend_jit_addr res_addr,
uint32_t res_use_info)
{
zend_reg result_reg, tmp_reg_gp;
if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) {
/* ASSIGN_DIM_OP */
tmp_reg_gp = ZREG_R1;
} else {
tmp_reg_gp = ZREG_R0;
}
if (zend_is_commutative(opcode)
&& (Z_MODE(res_addr) != IS_REG || Z_MODE(op1_addr) != IS_REG || Z_REG(res_addr) != Z_REG(op1_addr))) {
if (Z_MODE(res_addr) == IS_REG) {
result_reg = Z_REG(res_addr);
} else {
result_reg = ZREG_XMM0;
}
| SSE_GET_ZVAL_LVAL result_reg, op2_addr, tmp_reg_gp
if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) {
/* ASSIGN_DIM_OP */
if (CAN_USE_AVX()) {
| AVX_MATH opcode, result_reg, result_reg, op1_addr, r1
} else {
| SSE_MATH opcode, result_reg, op1_addr, r1
}
} else {
if (CAN_USE_AVX()) {
| AVX_MATH opcode, result_reg, result_reg, op1_addr, r0
} else {
| SSE_MATH opcode, result_reg, op1_addr, r0
}
}
} else {
zend_reg tmp_reg;
if (Z_MODE(res_addr) == IS_REG) {
result_reg = Z_REG(res_addr);
tmp_reg = (result_reg == ZREG_XMM0) ? ZREG_XMM1 : ZREG_XMM0;
} else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) {
result_reg = Z_REG(op1_addr);
tmp_reg = ZREG_XMM0;
} else {
result_reg = ZREG_XMM0;
tmp_reg = ZREG_XMM1;
}
if (CAN_USE_AVX()) {
zend_reg op1_reg;
if (Z_MODE(op1_addr) == IS_REG) {
op1_reg = Z_REG(op1_addr);
} else {
| SSE_GET_ZVAL_DVAL result_reg, op1_addr
op1_reg = result_reg;
}
if ((opcode == ZEND_ADD || opcode == ZEND_SUB)
&& Z_MODE(op2_addr) == IS_CONST_ZVAL
&& Z_LVAL_P(Z_ZV(op2_addr)) == 0) {
/* +/- 0 */
} else {
| SSE_GET_ZVAL_LVAL tmp_reg, op2_addr, tmp_reg_gp
| AVX_MATH_REG opcode, result_reg, op1_reg, tmp_reg
}
} else {
| SSE_GET_ZVAL_DVAL result_reg, op1_addr
if ((opcode == ZEND_ADD || opcode == ZEND_SUB)
&& Z_MODE(op2_addr) == IS_CONST_ZVAL
&& Z_LVAL_P(Z_ZV(op2_addr)) == 0) {
/* +/- 0 */
} else {
| SSE_GET_ZVAL_LVAL tmp_reg, op2_addr, tmp_reg_gp
| SSE_MATH_REG opcode, result_reg, tmp_reg
}
}
}
| SSE_SET_ZVAL_DVAL res_addr, result_reg
if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) {
if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE
}
}
}
return 1;
}
static int zend_jit_math_double_double(dasm_State **Dst,
zend_uchar opcode,
zend_jit_addr op1_addr,
zend_jit_addr op2_addr,
zend_jit_addr res_addr,
uint32_t res_use_info)
{
zend_bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
zend_reg result_reg;
if (Z_MODE(res_addr) == IS_REG) {
result_reg = Z_REG(res_addr);
} else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) {
result_reg = Z_REG(op1_addr);
} else if (zend_is_commutative(opcode) && Z_MODE(op2_addr) == IS_REG && Z_LAST_USE(op2_addr)) {
result_reg = Z_REG(op2_addr);
} else {
result_reg = ZREG_XMM0;
}
if (CAN_USE_AVX()) {
zend_reg op1_reg;
zend_jit_addr val_addr;
if (Z_MODE(op1_addr) == IS_REG) {
op1_reg = Z_REG(op1_addr);
val_addr = op2_addr;
} else if (Z_MODE(op2_addr) == IS_REG && zend_is_commutative(opcode)) {
op1_reg = Z_REG(op2_addr);
val_addr = op1_addr;
} else {
| SSE_GET_ZVAL_DVAL result_reg, op1_addr
op1_reg = result_reg;
val_addr = op2_addr;
}
if ((opcode == ZEND_MUL) &&
Z_MODE(val_addr) == IS_CONST_ZVAL && Z_DVAL_P(Z_ZV(val_addr)) == 2.0) {
| AVX_MATH_REG ZEND_ADD, result_reg, op1_reg, op1_reg
} else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) {
/* ASSIGN_DIM_OP */
| AVX_MATH opcode, result_reg, op1_reg, val_addr, r1
} else {
| AVX_MATH opcode, result_reg, op1_reg, val_addr, r0
}
} else {
zend_jit_addr val_addr;
if (Z_MODE(op1_addr) != IS_REG && Z_MODE(op2_addr) == IS_REG && zend_is_commutative(opcode)) {
| SSE_GET_ZVAL_DVAL result_reg, op2_addr
val_addr = op1_addr;
} else {
| SSE_GET_ZVAL_DVAL result_reg, op1_addr
val_addr = op2_addr;
}
if (same_ops) {
| SSE_MATH_REG opcode, result_reg, result_reg
} else if ((opcode == ZEND_MUL) &&
Z_MODE(val_addr) == IS_CONST_ZVAL && Z_DVAL_P(Z_ZV(val_addr)) == 2.0) {
| SSE_MATH_REG ZEND_ADD, result_reg, result_reg
} else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) {
/* ASSIGN_DIM_OP */
| SSE_MATH opcode, result_reg, val_addr, r1
} else {
| SSE_MATH opcode, result_reg, val_addr, r0
}
}
| SSE_SET_ZVAL_DVAL res_addr, result_reg
if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) {
if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE
}
}
}
return 1;
}
static int zend_jit_math_helper(dasm_State **Dst,
const zend_op *opline,
zend_uchar opcode,
zend_uchar op1_type,
znode_op op1,
zend_jit_addr op1_addr,
uint32_t op1_info,
zend_uchar op2_type,
znode_op op2,
zend_jit_addr op2_addr,
uint32_t op2_info,
uint32_t res_var,
zend_jit_addr res_addr,
uint32_t res_info,
uint32_t res_use_info,
int may_overflow,
int may_throw)
/* Labels: 1,2,3,4,5,6 */
{
zend_bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) {
if (op1_info & MAY_BE_DOUBLE) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6
}
}
if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) {
if (op2_info & MAY_BE_DOUBLE) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >1
|.cold_code
|1:
if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6
}
if (!zend_jit_math_long_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
return 0;
}
| jmp >5
|.code
} else {
| IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6
}
}
if (!zend_jit_math_long_long(Dst, opline, opcode, op1_addr, op2_addr, res_addr, res_info, res_use_info, may_overflow)) {
return 0;
}
if (op1_info & MAY_BE_DOUBLE) {
|.cold_code
|3:
if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6
}
if (op2_info & MAY_BE_DOUBLE) {
if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) {
if (!same_ops) {
| IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >1
} else {
| IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >6
}
}
if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
return 0;
}
| jmp >5
}
if (!same_ops) {
|1:
if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6
}
if (!zend_jit_math_double_long(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
return 0;
}
| jmp >5
}
|.code
}
} else if ((op1_info & MAY_BE_DOUBLE) &&
!(op1_info & MAY_BE_LONG) &&
(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
(res_info & MAY_BE_DOUBLE)) {
if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6
}
if (op2_info & MAY_BE_DOUBLE) {
if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) {
if (!same_ops && (op2_info & MAY_BE_LONG)) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >1
} else {
| IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6
}
}
if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
return 0;
}
}
if (!same_ops && (op2_info & MAY_BE_LONG)) {
if (op2_info & MAY_BE_DOUBLE) {
|.cold_code
}
|1:
if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6
}
if (!zend_jit_math_double_long(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
return 0;
}
if (op2_info & MAY_BE_DOUBLE) {
| jmp >5
|.code
}
}
} else if ((op2_info & MAY_BE_DOUBLE) &&
!(op2_info & MAY_BE_LONG) &&
(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
(res_info & MAY_BE_DOUBLE)) {
if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6
}
if (op1_info & MAY_BE_DOUBLE) {
if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) {
if (!same_ops && (op1_info & MAY_BE_LONG)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >1
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6
}
}
if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
return 0;
}
}
if (!same_ops && (op1_info & MAY_BE_LONG)) {
if (op1_info & MAY_BE_DOUBLE) {
|.cold_code
}
|1:
if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6
}
if (!zend_jit_math_long_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
return 0;
}
if (op1_info & MAY_BE_DOUBLE) {
| jmp >5
|.code
}
}
}
|5:
if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) ||
(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) {
if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
(res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
|.cold_code
}
|6:
if (Z_MODE(res_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
| LOAD_ZVAL_ADDR FCARG1a, real_addr
} else if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, res_addr
}
if (Z_MODE(op1_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var);
if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) {
return 0;
}
op1_addr = real_addr;
}
| LOAD_ZVAL_ADDR FCARG2a, op1_addr
if (Z_MODE(op2_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var);
if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) {
return 0;
}
op2_addr = real_addr;
}
|.if X64
| LOAD_ZVAL_ADDR CARG3, op2_addr
|.else
| sub r4, 12
| PUSH_ZVAL_ADDR op2_addr, r0
|.endif
| SET_EX_OPLINE opline, r0
if (opcode == ZEND_ADD) {
| EXT_CALL add_function, r0
} else if (opcode == ZEND_SUB) {
| EXT_CALL sub_function, r0
} else if (opcode == ZEND_MUL) {
| EXT_CALL mul_function, r0
} else if (opcode == ZEND_DIV) {
| EXT_CALL div_function, r0
} else {
ZEND_UNREACHABLE();
}
|.if not(X64)
| add r4, 12
|.endif
| FREE_OP op1_type, op1, op1_info, 0, opline
| FREE_OP op2_type, op2, op2_info, 0, opline
if (may_throw) {
if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) {
| MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0
| jne ->exception_handler_free_op2
} else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) {
zend_jit_check_exception_undef_result(Dst, opline);
} else {
zend_jit_check_exception(Dst);
}
}
if (Z_MODE(res_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) {
return 0;
}
}
if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
(res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
| jmp <5
|.code
}
}
return 1;
}
static int zend_jit_math(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw)
{
ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF));
ZEND_ASSERT((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)));
if (!zend_jit_math_helper(Dst, opline, opline->opcode, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->result.var, res_addr, res_info, res_use_info, may_overflow, may_throw)) {
return 0;
}
if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) {
return 0;
}
return 1;
}
static int zend_jit_add_arrays(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, zend_jit_addr res_addr)
{
zend_jit_addr op1_addr = OP1_ADDR();
zend_jit_addr op2_addr = OP2_ADDR();
| GET_ZVAL_LVAL ZREG_FCARG1a, op1_addr
| GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr
| EXT_CALL zend_jit_add_arrays_helper, r0
| SET_ZVAL_PTR res_addr, r0
| SET_ZVAL_TYPE_INFO res_addr, IS_ARRAY_EX
| FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline
| FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline
return 1;
}
static int zend_jit_long_math_helper(dasm_State **Dst,
const zend_op *opline,
zend_uchar opcode,
zend_uchar op1_type,
znode_op op1,
zend_jit_addr op1_addr,
uint32_t op1_info,
zend_ssa_range *op1_range,
zend_uchar op2_type,
znode_op op2,
zend_jit_addr op2_addr,
uint32_t op2_info,
zend_ssa_range *op2_range,
uint32_t res_var,
zend_jit_addr res_addr,
uint32_t res_info,
uint32_t res_use_info,
int may_throw)
/* Labels: 6 */
{
zend_bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
zend_reg result_reg;
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6
}
if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6
}
if (opcode == ZEND_MOD) {
result_reg = ZREG_RAX;
} else if (Z_MODE(res_addr) == IS_REG) {
if ((opline->opcode == ZEND_SL || opline->opcode == ZEND_SR)
&& opline->op2_type != IS_CONST) {
result_reg = ZREG_R0;
} else {
result_reg = Z_REG(res_addr);
}
} else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) {
result_reg = Z_REG(op1_addr);
} else if (Z_REG(res_addr) != ZREG_R0) {
result_reg = ZREG_R0;
} else {
/* ASSIGN_DIM_OP */
if (sizeof(void*) == 4
&& (opcode == ZEND_SL || opcode == ZEND_SR)
&& Z_MODE(op2_addr) != IS_CONST_ZVAL) {
result_reg = ZREG_R2;
} else {
result_reg = ZREG_FCARG1a;
}
}
if (opcode == ZEND_SL) {
if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr));
if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) {
if (EXPECTED(op2_lval > 0)) {
| xor Ra(result_reg), Ra(result_reg)
} else {
zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1);
zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2);
| SET_EX_OPLINE opline, r0
| jmp ->negative_shift
}
} else if (Z_MODE(op1_addr) == IS_REG && op2_lval == 1) {
| lea Ra(result_reg), [Ra(Z_REG(op1_addr))+Ra(Z_REG(op1_addr))]
} else {
| GET_ZVAL_LVAL result_reg, op1_addr
| shl Ra(result_reg), op2_lval
}
} else {
if (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_RCX) {
| GET_ZVAL_LVAL ZREG_RCX, op2_addr
}
if (!op2_range ||
op2_range->min < 0 ||
op2_range->max >= SIZEOF_ZEND_LONG * 8) {
| cmp r1, (SIZEOF_ZEND_LONG*8)
| jae >1
|.cold_code
|1:
| cmp r1, 0
| mov Ra(result_reg), 0
| jg >1
zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1);
zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2);
| SET_EX_OPLINE opline, r0
| jmp ->negative_shift
|.code
}
| GET_ZVAL_LVAL result_reg, op1_addr
| shl Ra(result_reg), cl
|1:
}
} else if (opcode == ZEND_SR) {
| GET_ZVAL_LVAL result_reg, op1_addr
if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr));
if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) {
if (EXPECTED(op2_lval > 0)) {
| sar Ra(result_reg), (SIZEOF_ZEND_LONG * 8) - 1
} else {
zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1);
zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2);
| SET_EX_OPLINE opline, r0
| jmp ->negative_shift
}
} else {
| sar Ra(result_reg), op2_lval
}
} else {
if (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_RCX) {
| GET_ZVAL_LVAL ZREG_RCX, op2_addr
}
if (!op2_range ||
op2_range->min < 0 ||
op2_range->max >= SIZEOF_ZEND_LONG * 8) {
| cmp r1, (SIZEOF_ZEND_LONG*8)
| jae >1
|.cold_code
|1:
| cmp r1, 0
| mov r1, (SIZEOF_ZEND_LONG * 8) - 1
| jg >1
zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1);
zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2);
| SET_EX_OPLINE opline, r0
| jmp ->negative_shift
|.code
}
|1:
| sar Ra(result_reg), cl
}
} else if (opcode == ZEND_MOD) {
if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr));
if (op2_lval == 0) {
zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1);
zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2);
| SET_EX_OPLINE opline, r0
| jmp ->mod_by_zero
} else if (zend_long_is_power_of_two(op2_lval) && op1_range && op1_range->min >= 0) {
zval tmp;
zend_jit_addr tmp_addr;
zend_reg tmp_reg;
/* Optimisation for mod of power of 2 */
ZVAL_LONG(&tmp, op2_lval - 1);
tmp_addr = ZEND_ADDR_CONST_ZVAL(&tmp);
if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) {
tmp_reg = ZREG_R1;
} else if (result_reg != ZREG_R0) {
tmp_reg = ZREG_R0;
} else {
tmp_reg = ZREG_R1;
}
| GET_ZVAL_LVAL result_reg, op1_addr
| LONG_MATH ZEND_BW_AND, result_reg, tmp_addr, tmp_reg
(void)tmp_reg;
} else {
if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RAX) {
| mov aword T1, r0 // save
} else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RCX) {
| mov aword T1, Ra(ZREG_RCX) // save
}
result_reg = ZREG_RDX;
if (op2_lval == -1) {
| xor Ra(result_reg), Ra(result_reg)
} else {
| GET_ZVAL_LVAL ZREG_RAX, op1_addr
| GET_ZVAL_LVAL ZREG_RCX, op2_addr
|.if X64
| cqo
|.else
| cdq
|.endif
| idiv Ra(ZREG_RCX)
}
if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RAX) {
| mov r0, aword T1 // restore
} else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RCX) {
| mov Ra(ZREG_RCX), aword T1 // restore
}
}
} else {
if (!op2_range || (op2_range->min <= 0 && op2_range->max >= 0)) {
if (Z_MODE(op2_addr) == IS_MEM_ZVAL) {
| cmp aword [Ra(Z_REG(op2_addr))+Z_OFFSET(op2_addr)], 0
} else if (Z_MODE(op2_addr) == IS_REG) {
| test Ra(Z_REG(op2_addr)), Ra(Z_REG(op2_addr))
}
| jz >1
|.cold_code
|1:
zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1);
zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2);
| SET_EX_OPLINE opline, r0
| jmp ->mod_by_zero
|.code
}
/* Prevent overflow error/crash if op1 == LONG_MIN and op2 == -1 */
if (!op2_range || (op2_range->min <= -1 && op2_range->max >= -1)) {
if (Z_MODE(op2_addr) == IS_MEM_ZVAL) {
| cmp aword [Ra(Z_REG(op2_addr))+Z_OFFSET(op2_addr)], -1
} else if (Z_MODE(op2_addr) == IS_REG) {
| cmp Ra(Z_REG(op2_addr)), -1
}
| jz >1
|.cold_code
|1:
| SET_ZVAL_LVAL res_addr, 0
if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) {
if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) {
| SET_ZVAL_TYPE_INFO res_addr, IS_LONG
}
}
}
| jmp >5
|.code
}
if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RAX) {
| mov aword T1, r0 // save
}
result_reg = ZREG_RDX;
| GET_ZVAL_LVAL ZREG_RAX, op1_addr
|.if X64
| cqo
|.else
| cdq
|.endif
if (Z_MODE(op2_addr) == IS_MEM_ZVAL) {
| idiv aword [Ra(Z_REG(op2_addr))+Z_OFFSET(op2_addr)]
} else if (Z_MODE(op2_addr) == IS_REG) {
| idiv Ra(Z_REG(op2_addr))
}
if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RAX) {
| mov r0, aword T1 // restore
}
}
} else if (same_ops) {
| GET_ZVAL_LVAL result_reg, op1_addr
| LONG_MATH_REG opcode, Ra(result_reg), Ra(result_reg)
} else {
zend_reg tmp_reg;
if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) {
tmp_reg = ZREG_R1;
} else if (result_reg != ZREG_R0) {
tmp_reg = ZREG_R0;
} else {
tmp_reg = ZREG_R1;
}
| GET_ZVAL_LVAL result_reg, op1_addr
| LONG_MATH opcode, result_reg, op2_addr, tmp_reg
(void)tmp_reg;
}
if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != result_reg) {
| SET_ZVAL_LVAL res_addr, Ra(result_reg)
}
if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) {
if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) {
| SET_ZVAL_TYPE_INFO res_addr, IS_LONG
}
}
}
if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) ||
(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) {
if ((op1_info & MAY_BE_LONG) &&
(op2_info & MAY_BE_LONG)) {
|.cold_code
}
|6:
if (Z_MODE(res_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
| LOAD_ZVAL_ADDR FCARG1a, real_addr
} else if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, res_addr
}
if (Z_MODE(op1_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var);
if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) {
return 0;
}
op1_addr = real_addr;
}
| LOAD_ZVAL_ADDR FCARG2a, op1_addr
if (Z_MODE(op2_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var);
if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) {
return 0;
}
op2_addr = real_addr;
}
|.if X64
| LOAD_ZVAL_ADDR CARG3, op2_addr
|.else
| sub r4, 12
| PUSH_ZVAL_ADDR op2_addr, r0
|.endif
| SET_EX_OPLINE opline, r0
if (opcode == ZEND_BW_OR) {
| EXT_CALL bitwise_or_function, r0
} else if (opcode == ZEND_BW_AND) {
| EXT_CALL bitwise_and_function, r0
} else if (opcode == ZEND_BW_XOR) {
| EXT_CALL bitwise_xor_function, r0
} else if (opcode == ZEND_SL) {
| EXT_CALL shift_left_function, r0
} else if (opcode == ZEND_SR) {
| EXT_CALL shift_right_function, r0
} else if (opcode == ZEND_MOD) {
| EXT_CALL mod_function, r0
} else {
ZEND_UNREACHABLE();
}
|.if not(X64)
| add r4, 12
|.endif
if (op1_addr == res_addr && (op2_info & MAY_BE_RCN)) {
/* compound assignment may decrement "op2" refcount */
op2_info |= MAY_BE_RC1;
}
| FREE_OP op1_type, op1, op1_info, 0, opline
| FREE_OP op2_type, op2, op2_info, 0, opline
if (may_throw) {
if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) {
| MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0
| jne ->exception_handler_free_op2
} else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) {
zend_jit_check_exception_undef_result(Dst, opline);
} else {
zend_jit_check_exception(Dst);
}
}
if (Z_MODE(res_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) {
return 0;
}
}
if ((op1_info & MAY_BE_LONG) &&
(op2_info & MAY_BE_LONG)) {
| jmp >5
|.code
}
}
|5:
return 1;
}
static int zend_jit_long_math(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_ssa_range *op1_range, zend_jit_addr op1_addr, uint32_t op2_info, zend_ssa_range *op2_range, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_throw)
{
ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF));
ZEND_ASSERT((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG));
if (!zend_jit_long_math_helper(Dst, opline, opline->opcode,
opline->op1_type, opline->op1, op1_addr, op1_info, op1_range,
opline->op2_type, opline->op2, op2_addr, op2_info, op2_range,
opline->result.var, res_addr, res_info, res_use_info, may_throw)) {
return 0;
}
if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) {
return 0;
}
return 1;
}
static int zend_jit_concat_helper(dasm_State **Dst,
const zend_op *opline,
zend_uchar op1_type,
znode_op op1,
zend_jit_addr op1_addr,
uint32_t op1_info,
zend_uchar op2_type,
znode_op op2,
zend_jit_addr op2_addr,
uint32_t op2_info,
zend_jit_addr res_addr,
int may_throw)
{
#if 1
if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) {
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6
}
if (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >6
}
if (Z_MODE(op1_addr) == IS_MEM_ZVAL && Z_REG(op1_addr) == Z_REG(res_addr) && Z_OFFSET(op1_addr) == Z_OFFSET(res_addr)) {
if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, res_addr
}
| LOAD_ZVAL_ADDR FCARG2a, op2_addr
| EXT_CALL zend_jit_fast_assign_concat_helper, r0
/* concatination with itself may reduce refcount */
op2_info |= MAY_BE_RC1;
} else {
if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, res_addr
}
| LOAD_ZVAL_ADDR FCARG2a, op1_addr
|.if X64
| LOAD_ZVAL_ADDR CARG3, op2_addr
|.else
| sub r4, 12
| PUSH_ZVAL_ADDR op2_addr, r0
|.endif
| EXT_CALL zend_jit_fast_concat_helper, r0
|.if not(X64)
| add r4, 12
|.endif
}
/* concatination with empty string may increase refcount */
op1_info |= MAY_BE_RCN;
op2_info |= MAY_BE_RCN;
| FREE_OP op1_type, op1, op1_info, 0, opline
| FREE_OP op2_type, op2, op2_info, 0, opline
|5:
}
if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) ||
(op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING))) {
if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) {
|.cold_code
|6:
}
#endif
if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, res_addr
}
| LOAD_ZVAL_ADDR FCARG2a, op1_addr
|.if X64
| LOAD_ZVAL_ADDR CARG3, op2_addr
|.else
| sub r4, 12
| PUSH_ZVAL_ADDR op2_addr, r0
|.endif
| SET_EX_OPLINE opline, r0
| EXT_CALL concat_function, r0
|.if not(X64)
| add r4, 12
|.endif
/* concatination with empty string may increase refcount */
op1_info |= MAY_BE_RCN;
op2_info |= MAY_BE_RCN;
| FREE_OP op1_type, op1, op1_info, 0, opline
| FREE_OP op2_type, op2, op2_info, 0, opline
if (may_throw) {
if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) {
zend_jit_check_exception_undef_result(Dst, opline);
} else {
zend_jit_check_exception(Dst);
}
}
#if 1
if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) {
| jmp <5
|.code
}
}
#endif
return 1;
}
static int zend_jit_concat(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, zend_jit_addr res_addr, int may_throw)
{
zend_jit_addr op1_addr, op2_addr;
ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF));
ZEND_ASSERT((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING));
op1_addr = OP1_ADDR();
op2_addr = OP2_ADDR();
return zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr, may_throw);
}
static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_op *opline, uint32_t type, uint32_t op1_info, uint32_t op2_info, const void *found_exit_addr, const void *not_found_exit_addr, const void *exit_addr)
/* Labels: 1,2,3,4,5 */
{
zend_jit_addr op2_addr = OP2_ADDR();
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& type == BP_VAR_R
&& !exit_addr) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
}
if (op2_info & MAY_BE_LONG) {
zend_bool op2_loaded = 0;
zend_bool packed_loaded = 0;
zend_bool bad_packed_key = 0;
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) {
| // if (EXPECTED(Z_TYPE_P(dim) == IS_LONG))
| IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3
}
if (op1_info & MAY_BE_PACKED_GUARD) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
if (op1_info & MAY_BE_ARRAY_PACKED) {
| test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED
| jz &exit_addr
} else {
| test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED
| jnz &exit_addr
}
}
if (type == BP_VAR_W) {
| // hval = Z_LVAL_P(dim);
| GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr
op2_loaded = 1;
}
if (op1_info & MAY_BE_ARRAY_PACKED) {
zend_long val = -1;
if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
val = Z_LVAL_P(Z_ZV(op2_addr));
if (val >= 0 && val < HT_MAX_SIZE) {
packed_loaded = 1;
} else {
bad_packed_key = 1;
}
} else {
if (!op2_loaded) {
| // hval = Z_LVAL_P(dim);
| GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr
op2_loaded = 1;
}
packed_loaded = 1;
}
if (packed_loaded) {
| // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef);
if (op1_info & MAY_BE_ARRAY_HASH) {
| test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED
| jz >4 // HASH_FIND
}
| // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed))
|.if X64
| mov eax, dword [FCARG1a + offsetof(zend_array, nNumUsed)]
if (val == 0) {
| test r0, r0
} else if (val > 0 && !op2_loaded) {
| cmp r0, val
} else {
| cmp r0, FCARG2a
}
|.else
if (val >= 0 && !op2_loaded) {
| cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], val
} else {
| cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], FCARG2a
}
|.endif
if (type == BP_JIT_IS) {
if (not_found_exit_addr) {
| jbe &not_found_exit_addr
} else {
| jbe >9 // NOT_FOUND
}
} else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
| jbe &exit_addr
} else if (type == BP_VAR_IS && not_found_exit_addr) {
| jbe &not_found_exit_addr
} else if (type == BP_VAR_IS && found_exit_addr) {
| jbe >7 // NOT_FOUND
} else {
| jbe >2 // NOT_FOUND
}
| // _ret = &_ht->arData[_h].val;
if (val >= 0) {
| mov r0, aword [FCARG1a + offsetof(zend_array, arData)]
if (val != 0) {
| add r0, val * sizeof(Bucket)
}
} else {
|.if X64
| mov r0, FCARG2a
| shl r0, 5
|.else
| imul r0, FCARG2a, sizeof(Bucket)
|.endif
| add r0, aword [FCARG1a + offsetof(zend_array, arData)]
}
}
}
switch (type) {
case BP_JIT_IS:
if (op1_info & MAY_BE_ARRAY_HASH) {
if (packed_loaded) {
| jmp >5
}
|4:
if (!op2_loaded) {
| // hval = Z_LVAL_P(dim);
| GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr
}
if (packed_loaded) {
| EXT_CALL _zend_hash_index_find, r0
} else {
| EXT_CALL zend_hash_index_find, r0
}
| test r0, r0
if (not_found_exit_addr) {
| jz &not_found_exit_addr
} else {
| jz >9 // NOT_FOUND
}
if (op2_info & MAY_BE_STRING) {
| jmp >5
}
} else if (packed_loaded) {
if (op2_info & MAY_BE_STRING) {
| jmp >5
}
} else if (not_found_exit_addr) {
| jmp &not_found_exit_addr
} else {
| jmp >9 // NOT_FOUND
}
break;
case BP_VAR_R:
case BP_VAR_IS:
case BP_VAR_UNSET:
if (packed_loaded) {
if (op1_info & MAY_BE_ARRAY_HASH) {
| IF_NOT_Z_TYPE r0, IS_UNDEF, >8
} else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
/* perform IS_UNDEF check only after result type guard (during deoptimization) */
if (!found_exit_addr || (op1_info & MAY_BE_ARRAY_HASH)) {
| IF_Z_TYPE r0, IS_UNDEF, &exit_addr
}
} else if (type == BP_VAR_IS && not_found_exit_addr) {
| IF_Z_TYPE r0, IS_UNDEF, &not_found_exit_addr
} else if (type == BP_VAR_IS && found_exit_addr) {
| IF_Z_TYPE r0, IS_UNDEF, >7 // NOT_FOUND
} else {
| IF_Z_TYPE r0, IS_UNDEF, >2 // NOT_FOUND
}
}
if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (packed_loaded && (op1_info & MAY_BE_ARRAY_HASH))) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
| jmp &exit_addr
} else if (type == BP_VAR_IS && not_found_exit_addr) {
| jmp &not_found_exit_addr
} else if (type == BP_VAR_IS && found_exit_addr) {
| jmp >7 // NOT_FOUND
} else {
| jmp >2 // NOT_FOUND
}
}
if (op1_info & MAY_BE_ARRAY_HASH) {
|4:
if (!op2_loaded) {
| // hval = Z_LVAL_P(dim);
| GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr
}
if (packed_loaded) {
| EXT_CALL _zend_hash_index_find, r0
} else {
| EXT_CALL zend_hash_index_find, r0
}
| test r0, r0
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
| jz &exit_addr
} else if (type == BP_VAR_IS && not_found_exit_addr) {
| jz &not_found_exit_addr
} else if (type == BP_VAR_IS && found_exit_addr) {
| jz >7 // NOT_FOUND
} else {
| jz >2 // NOT_FOUND
}
}
|.cold_code
|2:
switch (type) {
case BP_VAR_R:
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
| // zend_error(E_WARNING,"Undefined array key " ZEND_LONG_FMT, hval);
| // retval = &EG(uninitialized_zval);
| UNDEFINED_OFFSET opline
| jmp >9
}
break;
case BP_VAR_IS:
case BP_VAR_UNSET:
if (!not_found_exit_addr && !found_exit_addr) {
| // retval = &EG(uninitialized_zval);
| SET_ZVAL_TYPE_INFO res_addr, IS_NULL
| jmp >9
}
break;
default:
ZEND_UNREACHABLE();
}
|.code
break;
case BP_VAR_RW:
if (packed_loaded) {
| IF_NOT_Z_TYPE r0, IS_UNDEF, >8
}
|2:
|4:
if (!op2_loaded) {
| // hval = Z_LVAL_P(dim);
| GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr
}
| SET_EX_OPLINE opline, r0
if (packed_loaded) {
| EXT_CALL zend_jit_hash_index_lookup_rw_no_packed, r0
} else {
| EXT_CALL zend_jit_hash_index_lookup_rw, r0
}
| test r0, r0
| jz >9
break;
case BP_VAR_W:
if (packed_loaded) {
| IF_NOT_Z_TYPE r0, IS_UNDEF, >8
}
if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || packed_loaded || bad_packed_key) {
|2:
| //retval = zend_hash_index_add_new(ht, hval, &EG(uninitialized_zval));
if (!op2_loaded) {
| // hval = Z_LVAL_P(dim);
| GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr
}
|.if X64
| LOAD_ADDR_ZTS CARG3, executor_globals, uninitialized_zval
|.else
| sub r4, 12
| PUSH_ADDR_ZTS executor_globals, uninitialized_zval, r0
|.endif
| EXT_CALL zend_hash_index_add_new, r0
|.if not(X64)
| add r4, 12
|.endif
if (op1_info & MAY_BE_ARRAY_HASH) {
| jmp >8
}
}
if (op1_info & MAY_BE_ARRAY_HASH) {
|4:
if (!op2_loaded) {
| // hval = Z_LVAL_P(dim);
| GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr
}
| EXT_CALL zend_jit_hash_index_lookup_w, r0
}
break;
default:
ZEND_UNREACHABLE();
}
if (type != BP_JIT_IS && (op2_info & MAY_BE_STRING)) {
| jmp >8
}
}
if (op2_info & MAY_BE_STRING) {
|3:
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) {
| // if (EXPECTED(Z_TYPE_P(dim) == IS_STRING))
| IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >3
}
| // offset_key = Z_STR_P(dim);
| GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr
| // retval = zend_hash_find(ht, offset_key);
switch (type) {
case BP_JIT_IS:
if (opline->op2_type != IS_CONST) {
| cmp byte [FCARG2a + offsetof(zend_string, val)], '9'
| jle >1
|.cold_code
|1:
| EXT_CALL zend_jit_symtable_find, r0
| jmp >1
|.code
| EXT_CALL zend_hash_find, r0
|1:
} else {
| EXT_CALL _zend_hash_find_known_hash, r0
}
| test r0, r0
if (not_found_exit_addr) {
| jz &not_found_exit_addr
} else {
| jz >9 // NOT_FOUND
}
| // if (UNEXPECTED(Z_TYPE_P(retval) == IS_INDIRECT))
| IF_NOT_Z_TYPE r0, IS_INDIRECT, >1
| GET_Z_PTR r0, r0
|1:
break;
case BP_VAR_R:
case BP_VAR_IS:
case BP_VAR_UNSET:
if (opline->op2_type != IS_CONST) {
| cmp byte [FCARG2a + offsetof(zend_string, val)], '9'
| jle >1
|.cold_code
|1:
| EXT_CALL zend_jit_symtable_find, r0
| jmp >1
|.code
| EXT_CALL zend_hash_find, r0
|1:
} else {
| EXT_CALL _zend_hash_find_known_hash, r0
}
| test r0, r0
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
| jz &exit_addr
} else if (type == BP_VAR_IS && not_found_exit_addr) {
| jz &not_found_exit_addr
} else if (type == BP_VAR_IS && found_exit_addr) {
| jz >7 // NOT_FOUND
} else {
| jz >2 // NOT_FOUND
}
| // if (UNEXPECTED(Z_TYPE_P(retval) == IS_INDIRECT))
| IF_Z_TYPE r0, IS_INDIRECT, >1 // SLOW
|.cold_code
|1:
| // retval = Z_INDIRECT_P(retval);
| GET_Z_PTR r0, r0
| IF_NOT_Z_TYPE r0, IS_UNDEF, >8
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
| jmp &exit_addr
} else if (type == BP_VAR_IS && not_found_exit_addr) {
| jmp &not_found_exit_addr
}
|2:
switch (type) {
case BP_VAR_R:
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
// zend_error(E_WARNING, "Undefined array key \"%s\"", ZSTR_VAL(offset_key));
| UNDEFINED_INDEX opline
| jmp >9
}
break;
case BP_VAR_IS:
case BP_VAR_UNSET:
if (!not_found_exit_addr && !found_exit_addr) {
| // retval = &EG(uninitialized_zval);
| SET_ZVAL_TYPE_INFO res_addr, IS_NULL
| jmp >9
}
break;
default:
ZEND_UNREACHABLE();
}
|.code
break;
case BP_VAR_RW:
| SET_EX_OPLINE opline, r0
if (opline->op2_type != IS_CONST) {
| EXT_CALL zend_jit_symtable_lookup_rw, r0
} else {
| EXT_CALL zend_jit_hash_lookup_rw, r0
}
| test r0, r0
| jz >9
break;
case BP_VAR_W:
if (opline->op2_type != IS_CONST) {
| EXT_CALL zend_jit_symtable_lookup_w, r0
} else {
| EXT_CALL zend_jit_hash_lookup_w, r0
}
break;
default:
ZEND_UNREACHABLE();
}
}
if (type == BP_JIT_IS && (op2_info & (MAY_BE_LONG|MAY_BE_STRING))) {
|5:
if (op1_info & MAY_BE_ARRAY_OF_REF) {
| ZVAL_DEREF r0, MAY_BE_REF
}
| cmp byte [r0 + 8], IS_NULL
if (not_found_exit_addr) {
| jle &not_found_exit_addr
} else if (found_exit_addr) {
| jg &found_exit_addr
} else {
| jle >9 // NOT FOUND
}
}
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) {
if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) {
|.cold_code
|3:
}
| SET_EX_OPLINE opline, r0
| LOAD_ZVAL_ADDR FCARG2a, op2_addr
switch (type) {
case BP_VAR_R:
|.if X64
| LOAD_ZVAL_ADDR CARG3, res_addr
|.else
| sub r4, 12
| PUSH_ZVAL_ADDR res_addr, r0
|.endif
| EXT_CALL zend_jit_fetch_dim_r_helper, r0
|.if not(X64)
| add r4, 12
|.endif
| jmp >9
break;
case BP_JIT_IS:
| EXT_CALL zend_jit_fetch_dim_isset_helper, r0
| test r0, r0
if (not_found_exit_addr) {
| je &not_found_exit_addr
if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) {
| jmp >8
}
} else if (found_exit_addr) {
| jne &found_exit_addr
if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) {
| jmp >9
}
} else {
| jne >8
| jmp >9
}
break;
case BP_VAR_IS:
case BP_VAR_UNSET:
|.if X64
| LOAD_ZVAL_ADDR CARG3, res_addr
|.else
| sub r4, 12
| PUSH_ZVAL_ADDR res_addr, r0
|.endif
| EXT_CALL zend_jit_fetch_dim_is_helper, r0
|.if not(X64)
| add r4, 12
|.endif
| jmp >9
break;
case BP_VAR_RW:
| EXT_CALL zend_jit_fetch_dim_rw_helper, r0
| test r0, r0
| jne >8
| jmp >9
break;
case BP_VAR_W:
| EXT_CALL zend_jit_fetch_dim_w_helper, r0
| test r0, r0
| jne >8
| jmp >9
break;
default:
ZEND_UNREACHABLE();
}
if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) {
|.code
}
}
return 1;
}
static int zend_jit_simple_assign(dasm_State **Dst,
const zend_op *opline,
zend_jit_addr var_addr,
uint32_t var_info,
uint32_t var_def_info,
zend_uchar val_type,
zend_jit_addr val_addr,
uint32_t val_info,
zend_jit_addr res_addr,
int in_cold,
int save_r1)
/* Labels: 1,2,3 */
{
zend_reg tmp_reg;
if (Z_MODE(var_addr) == IS_REG || Z_REG(var_addr) != ZREG_R0) {
tmp_reg = ZREG_R0;
} else {
/* ASSIGN_DIM */
tmp_reg = ZREG_FCARG1a;
}
if (Z_MODE(val_addr) == IS_CONST_ZVAL) {
zval *zv = Z_ZV(val_addr);
if (!res_addr) {
| ZVAL_COPY_CONST var_addr, var_info, var_def_info, zv, tmp_reg
} else {
| ZVAL_COPY_CONST_2 var_addr, res_addr, var_info, var_def_info, zv, tmp_reg
}
if (Z_REFCOUNTED_P(zv)) {
if (!res_addr) {
| ADDREF_CONST zv, Ra(tmp_reg)
} else {
| ADDREF_CONST_2 zv, Ra(tmp_reg)
}
}
} else {
if (val_info & MAY_BE_UNDEF) {
if (in_cold) {
| IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >2
} else {
| IF_ZVAL_TYPE val_addr, IS_UNDEF, >1
|.cold_code
|1:
}
| // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
if (save_r1) {
| mov aword T1, FCARG1a // save
}
| SET_ZVAL_TYPE_INFO var_addr, IS_NULL
if (res_addr) {
| SET_ZVAL_TYPE_INFO res_addr, IS_NULL
}
if (opline) {
| SET_EX_OPLINE opline, Ra(tmp_reg)
}
ZEND_ASSERT(Z_MODE(val_addr) == IS_MEM_ZVAL && Z_REG(val_addr) == ZREG_FP);
| mov FCARG1d, Z_OFFSET(val_addr)
| EXT_CALL zend_jit_undefined_op_helper, r0
| test r0, r0
| jz ->exception_handler_undef
if (save_r1) {
| mov FCARG1a, aword T1 // restore
}
| jmp >3
if (in_cold) {
|2:
} else {
|.code
}
}
if (val_info & MAY_BE_REF) {
if (val_type == IS_CV) {
ZEND_ASSERT(Z_REG(var_addr) != ZREG_R2);
if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_R2 || Z_OFFSET(val_addr) != 0) {
| LOAD_ZVAL_ADDR r2, val_addr
}
| ZVAL_DEREF r2, val_info
val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 0);
} else {
zend_jit_addr ref_addr;
zend_reg type_reg = tmp_reg;
if (in_cold) {
| IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, >1
} else {
| IF_ZVAL_TYPE val_addr, IS_REFERENCE, >1
|.cold_code
|1:
}
| // zend_refcounted *ref = Z_COUNTED_P(retval_ptr);
| GET_ZVAL_PTR r2, val_addr
| GC_DELREF r2
| // ZVAL_COPY_VALUE(return_value, &ref->value);
ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 8);
if (!res_addr) {
| ZVAL_COPY_VALUE var_addr, var_info, ref_addr, val_info, type_reg, tmp_reg
} else {
| ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, ref_addr, val_info, type_reg, tmp_reg
}
| je >2
if (tmp_reg == ZREG_R0) {
| IF_NOT_REFCOUNTED ah, >3
} else {
| IF_NOT_FLAGS Rd(tmp_reg), (IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT), >3
}
| GET_ZVAL_PTR Ra(tmp_reg), var_addr
if (!res_addr) {
| GC_ADDREF Ra(tmp_reg)
} else {
| add dword [Ra(tmp_reg)], 2
}
| jmp >3
|2:
if (res_addr) {
if (tmp_reg == ZREG_R0) {
| IF_NOT_REFCOUNTED ah, >2
} else {
| IF_NOT_FLAGS Rd(tmp_reg), (IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT), >2
}
| GET_ZVAL_PTR Ra(tmp_reg), var_addr
| GC_ADDREF Ra(tmp_reg)
|2:
}
if (save_r1) {
| mov aword T1, FCARG1a // save
}
| EFREE_REFERENCE r2
if (save_r1) {
| mov FCARG1a, aword T1 // restore
}
| jmp >3
if (in_cold) {
|1:
} else {
|.code
}
}
}
if (!res_addr) {
| ZVAL_COPY_VALUE var_addr, var_info, val_addr, val_info, ZREG_R2, tmp_reg
} else {
| ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, val_addr, val_info, ZREG_R2, tmp_reg
}
if (val_type == IS_CV) {
if (!res_addr) {
| TRY_ADDREF val_info, dh, Ra(tmp_reg)
} else {
| TRY_ADDREF_2 val_info, dh, Ra(tmp_reg)
}
} else {
if (res_addr) {
| TRY_ADDREF val_info, dh, Ra(tmp_reg)
}
}
|3:
}
return 1;
}
static int zend_jit_assign_to_typed_ref(dasm_State **Dst,
const zend_op *opline,
zend_uchar val_type,
zend_jit_addr val_addr,
zend_jit_addr res_addr,
zend_bool check_exception)
{
| // if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) {
| cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0
| jnz >2
|.cold_code
|2:
if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2a || Z_OFFSET(val_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG2a, val_addr
}
if (opline) {
| SET_EX_OPLINE opline, r0
}
if (val_type == IS_CONST) {
| EXT_CALL zend_jit_assign_const_to_typed_ref, r0
} else if (val_type == IS_TMP_VAR) {
| EXT_CALL zend_jit_assign_tmp_to_typed_ref, r0
} else if (val_type == IS_VAR) {
| EXT_CALL zend_jit_assign_var_to_typed_ref, r0
} else if (val_type == IS_CV) {
| EXT_CALL zend_jit_assign_cv_to_typed_ref, r0
} else {
ZEND_UNREACHABLE();
}
if (res_addr) {
zend_jit_addr ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0);
| ZVAL_COPY_VALUE res_addr, -1, ret_addr, -1, ZREG_R1, ZREG_R2
| TRY_ADDREF -1, ch, r2
}
if (check_exception) {
| // if (UNEXPECTED(EG(exception) != NULL)) {
| MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0
| je >8 // END OF zend_jit_assign_to_variable()
| jmp ->exception_handler
} else {
| jmp >8
}
|.code
return 1;
}
static int zend_jit_assign_to_variable_call(dasm_State **Dst,
const zend_op *opline,
zend_jit_addr __var_use_addr,
zend_jit_addr var_addr,
uint32_t __var_info,
uint32_t __var_def_info,
zend_uchar val_type,
zend_jit_addr val_addr,
uint32_t val_info,
zend_jit_addr __res_addr,
zend_bool __check_exception)
{
if (val_info & MAY_BE_UNDEF) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_ZVAL_TYPE val_addr, IS_UNDEF, &exit_addr
} else {
| IF_ZVAL_TYPE val_addr, IS_UNDEF, >1
|.cold_code
|1:
ZEND_ASSERT(Z_REG(val_addr) == ZREG_FP);
if (Z_REG(var_addr) != ZREG_FP) {
| mov aword T1, Ra(Z_REG(var_addr)) // save
}
| SET_EX_OPLINE opline, r0
| mov FCARG1d, Z_OFFSET(val_addr)
| EXT_CALL zend_jit_undefined_op_helper, r0
if (Z_REG(var_addr) != ZREG_FP) {
| mov Ra(Z_REG(var_addr)), aword T1 // restore
}
if (Z_MODE(var_addr) != IS_MEM_ZVAL || Z_REG(var_addr) != ZREG_FCARG1a || Z_OFFSET(var_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, var_addr
}
| LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval
| call ->assign_const
| jmp >9
|.code
}
}
if (Z_MODE(var_addr) != IS_MEM_ZVAL || Z_REG(var_addr) != ZREG_FCARG1a || Z_OFFSET(var_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, var_addr
}
if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2a || Z_OFFSET(val_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG2a, val_addr
}
if (opline) {
| SET_EX_OPLINE opline, r0
}
if (!(val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) {
| call ->assign_tmp
} else if (val_type == IS_CONST) {
| call ->assign_const
} else if (val_type == IS_TMP_VAR) {
| call ->assign_tmp
} else if (val_type == IS_VAR) {
if (!(val_info & MAY_BE_REF)) {
| call ->assign_tmp
} else {
| call ->assign_var
}
} else if (val_type == IS_CV) {
if (!(val_info & MAY_BE_REF)) {
| call ->assign_cv_noref
} else {
| call ->assign_cv
}
if ((val_info & MAY_BE_UNDEF) && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
|9:
}
} else {
ZEND_UNREACHABLE();
}
return 1;
}
static int zend_jit_assign_to_variable(dasm_State **Dst,
const zend_op *opline,
zend_jit_addr var_use_addr,
zend_jit_addr var_addr,
uint32_t var_info,
uint32_t var_def_info,
zend_uchar val_type,
zend_jit_addr val_addr,
uint32_t val_info,
zend_jit_addr res_addr,
zend_bool check_exception)
/* Labels: 1,2,3,4,5,8 */
{
int done = 0;
zend_reg ref_reg, tmp_reg;
if (Z_MODE(var_addr) == IS_REG || Z_REG(var_use_addr) != ZREG_R0) {
ref_reg = ZREG_FCARG1a;
tmp_reg = ZREG_R0;
} else {
/* ASSIGN_DIM */
ref_reg = ZREG_R0;
tmp_reg = ZREG_FCARG1a;
}
if (var_info & MAY_BE_REF) {
if (Z_MODE(var_use_addr) != IS_MEM_ZVAL || Z_REG(var_use_addr) != ref_reg || Z_OFFSET(var_use_addr) != 0) {
| LOAD_ZVAL_ADDR Ra(ref_reg), var_use_addr
var_addr = var_use_addr = ZEND_ADDR_MEM_ZVAL(ref_reg, 0);
}
| // if (Z_ISREF_P(variable_ptr)) {
| IF_NOT_Z_TYPE, Ra(ref_reg), IS_REFERENCE, >3
| // if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) {
| GET_Z_PTR FCARG1a, Ra(ref_reg)
if (!zend_jit_assign_to_typed_ref(Dst, opline, val_type, val_addr, res_addr, check_exception)) {
return 0;
}
| lea Ra(ref_reg), [FCARG1a + offsetof(zend_reference, val)]
|3:
}
if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
if (RC_MAY_BE_1(var_info)) {
int in_cold = 0;
if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| IF_ZVAL_REFCOUNTED var_use_addr, >1
|.cold_code
|1:
in_cold = 1;
}
if (Z_REG(var_use_addr) == ZREG_FCARG1a || Z_REG(var_use_addr) == ZREG_R0) {
zend_bool keep_gc = 0;
| GET_ZVAL_PTR Ra(tmp_reg), var_use_addr
if (tmp_reg == ZREG_FCARG1a) {
if (Z_MODE(val_addr) == IS_REG) {
keep_gc = 1;
} else if ((val_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_GUARD)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) == 0) {
keep_gc = 1;
} else if (Z_MODE(val_addr) == IS_CONST_ZVAL) {
if (sizeof(void*) == 4) {
keep_gc = 1;
} else {
zval *zv = Z_ZV(val_addr);
if (Z_TYPE_P(zv) == IS_DOUBLE) {
if (Z_DVAL_P(zv) == 0 || IS_SIGNED_32BIT(zv)) {
keep_gc = 1;
}
} else if (IS_SIGNED_32BIT(Z_LVAL_P(zv))) {
keep_gc = 1;
}
}
} else if (Z_MODE(val_addr) == IS_MEM_ZVAL) {
if ((val_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) {
keep_gc = 1;
}
}
}
if (!keep_gc) {
| mov aword T1, Ra(tmp_reg) // save
}
if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, in_cold, 0)) {
return 0;
}
if (!keep_gc) {
| mov FCARG1a, aword T1 // restore
}
} else {
| GET_ZVAL_PTR FCARG1a, var_use_addr
if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, in_cold, 1)) {
return 0;
}
}
| GC_DELREF FCARG1a
if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) {
| jnz >4
} else {
| jnz >8
}
| ZVAL_DTOR_FUNC var_info, opline
if (in_cold || (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0)) {
if (check_exception) {
| MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0
| je >8
| jmp ->exception_handler
} else {
| jmp >8
}
}
if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) {
|4:
| IF_GC_MAY_NOT_LEAK FCARG1a, >8
| EXT_CALL gc_possible_root, r0
if (in_cold) {
| jmp >8
}
}
if (in_cold) {
|.code
} else {
done = 1;
}
} else /* if (RC_MAY_BE_N(var_info)) */ {
if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| IF_NOT_ZVAL_REFCOUNTED var_use_addr, >5
}
if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) {
if (Z_REG(var_use_addr) == ZREG_FP) {
| mov T1, Ra(Z_REG(var_use_addr)) // save
}
| GET_ZVAL_PTR FCARG1a, var_use_addr
| GC_DELREF FCARG1a
| IF_GC_MAY_NOT_LEAK FCARG1a, >5
| EXT_CALL gc_possible_root, r0
if (Z_REG(var_use_addr) != ZREG_FP) {
| mov Ra(Z_REG(var_use_addr)), T1 // restore
}
} else {
| GET_ZVAL_PTR Ra(tmp_reg), var_use_addr
| GC_DELREF Ra(tmp_reg)
}
|5:
}
}
if (!done && !zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, 0, 0)) {
return 0;
}
|8:
return 1;
}
static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t val_info, int may_throw)
{
zend_jit_addr op2_addr, op3_addr, res_addr;
op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0;
op3_addr = OP1_DATA_ADDR();
if (opline->result_type == IS_UNUSED) {
res_addr = 0;
} else {
res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
}
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && (val_info & MAY_BE_UNDEF)) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_ZVAL_TYPE op3_addr, IS_UNDEF, &exit_addr
val_info &= ~MAY_BE_UNDEF;
}
if (op1_info & MAY_BE_REF) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
| IF_NOT_Z_TYPE FCARG1a, IS_REFERENCE, >1
| GET_Z_PTR FCARG2a, FCARG1a
| IF_NOT_TYPE byte [FCARG2a + offsetof(zend_reference, val) + offsetof(zval, u1.v.type)], IS_ARRAY, >2
| lea FCARG1a, [FCARG2a + offsetof(zend_reference, val)]
| jmp >3
|.cold_code
|2:
| SET_EX_OPLINE opline, r0
| EXT_CALL zend_jit_prepare_assign_dim_ref, r0
| test r0, r0
| mov FCARG1a, r0
| jne >1
| jmp ->exception_handler_undef
|.code
|1:
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
}
if (op1_info & MAY_BE_ARRAY) {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7
}
|3:
| SEPARATE_ARRAY op1_addr, op1_info, 1
} else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) {
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) {
| CMP_ZVAL_TYPE op1_addr, IS_FALSE
| jg >7
}
| // ZVAL_ARR(container, zend_new_array(8));
if (Z_REG(op1_addr) != ZREG_FP) {
| mov T1, Ra(Z_REG(op1_addr)) // save
}
| EXT_CALL _zend_new_array_0, r0
if (Z_REG(op1_addr) != ZREG_FP) {
| mov Ra(Z_REG(op1_addr)), T1 // restore
}
| SET_ZVAL_LVAL op1_addr, r0
| SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX
| mov FCARG1a, r0
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) {
|6:
if (opline->op2_type == IS_UNUSED) {
uint32_t var_info = MAY_BE_NULL;
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0);
| // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval));
| LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval
| EXT_CALL zend_hash_next_index_insert, r0
| // if (UNEXPECTED(!var_ptr)) {
| test r0, r0
| jz >1
|.cold_code
|1:
| // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied");
| CANNOT_ADD_ELEMENT opline
| //ZEND_VM_C_GOTO(assign_dim_op_ret_null);
| jmp >9
|.code
if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0, 0)) {
return 0;
}
} else {
uint32_t var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0);
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0);
if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_W, op1_info, op2_info, NULL, NULL, NULL)) {
return 0;
}
if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) {
var_info |= MAY_BE_REF;
}
if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
var_info |= MAY_BE_RC1;
}
|8:
| // value = zend_assign_to_variable(variable_ptr, value, OP_DATA_TYPE);
if (opline->op1_type == IS_VAR) {
ZEND_ASSERT(opline->result_type == IS_UNUSED);
if (!zend_jit_assign_to_variable_call(Dst, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) {
return 0;
}
} else {
if (!zend_jit_assign_to_variable(Dst, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) {
return 0;
}
}
}
}
if (((op1_info & MAY_BE_ARRAY) &&
(op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE))) ||
(op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)))) {
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) {
|.cold_code
|7:
}
if ((op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) &&
(op1_info & MAY_BE_ARRAY)) {
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) {
| CMP_ZVAL_TYPE op1_addr, IS_FALSE
| jg >2
}
| // ZVAL_ARR(container, zend_new_array(8));
if (Z_REG(op1_addr) != ZREG_FP) {
| mov T1, Ra(Z_REG(op1_addr)) // save
}
| EXT_CALL _zend_new_array_0, r0
if (Z_REG(op1_addr) != ZREG_FP) {
| mov Ra(Z_REG(op1_addr)), T1 // restore
}
| SET_ZVAL_LVAL op1_addr, r0
| SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX
| mov FCARG1a, r0
| // ZEND_VM_C_GOTO(assign_dim_op_new_array);
| jmp <6
|2:
}
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) {
| SET_EX_OPLINE opline, r0
if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
if (opline->op2_type == IS_UNUSED) {
| xor FCARG2a, FCARG2a
} else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
| LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1)
} else {
| LOAD_ZVAL_ADDR FCARG2a, op2_addr
}
|.if not(X64)
| sub r4, 8
|.endif
if (opline->result_type == IS_UNUSED) {
|.if X64
| xor CARG4, CARG4
|.else
| push 0
|.endif
} else {
|.if X64
| LOAD_ZVAL_ADDR CARG4, res_addr
|.else
| PUSH_ZVAL_ADDR res_addr, r0
|.endif
}
|.if X64
| LOAD_ZVAL_ADDR CARG3, op3_addr
|.else
| PUSH_ZVAL_ADDR op3_addr, r0
|.endif
| EXT_CALL zend_jit_assign_dim_helper, r0
|.if not(X64)
| add r4, 8
|.endif
#ifdef ZEND_JIT_USE_RC_INFERENCE
if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) && (val_info & MAY_BE_RC1)) {
/* ASSIGN_DIM may increase refcount of the value */
val_info |= MAY_BE_RCN;
}
#endif
| FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) {
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) {
| jmp >9 // END
}
|.code
}
}
#ifdef ZEND_JIT_USE_RC_INFERENCE
if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) {
/* ASSIGN_DIM may increase refcount of the key */
op2_info |= MAY_BE_RCN;
}
#endif
|9:
| FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline
if (may_throw) {
zend_jit_check_exception(Dst);
}
return 1;
}
static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t op1_data_info, zend_ssa_range *op1_data_range, int may_throw)
{
zend_jit_addr op2_addr, op3_addr, var_addr;
ZEND_ASSERT(opline->result_type == IS_UNUSED);
op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0;
op3_addr = OP1_DATA_ADDR();
if (op1_info & MAY_BE_REF) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
| IF_NOT_Z_TYPE FCARG1a, IS_REFERENCE, >1
| GET_Z_PTR FCARG2a, FCARG1a
| IF_NOT_TYPE byte [FCARG2a + offsetof(zend_reference, val) + offsetof(zval, u1.v.type)], IS_ARRAY, >2
| lea FCARG1a, [FCARG2a + offsetof(zend_reference, val)]
| jmp >3
|.cold_code
|2:
| SET_EX_OPLINE opline, r0
| EXT_CALL zend_jit_prepare_assign_dim_ref, r0
| test r0, r0
| mov FCARG1a, r0
| jne >1
| jmp ->exception_handler_undef
|.code
|1:
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
}
if (op1_info & MAY_BE_ARRAY) {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7
}
|3:
| SEPARATE_ARRAY op1_addr, op1_info, 1
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) {
if (op1_info & MAY_BE_ARRAY) {
|.cold_code
|7:
}
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) {
| CMP_ZVAL_TYPE op1_addr, IS_FALSE
| jg >7
}
if (Z_REG(op1_addr) != ZREG_FP) {
| mov T1, Ra(Z_REG(op1_addr)) // save
}
if (op1_info & MAY_BE_UNDEF) {
if (op1_info & (MAY_BE_NULL|MAY_BE_FALSE)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1
}
| SET_EX_OPLINE opline, r0
| mov FCARG1a, opline->op1.var
| EXT_CALL zend_jit_undefined_op_helper, r0
|1:
}
| // ZVAL_ARR(container, zend_new_array(8));
| EXT_CALL _zend_new_array_0, r0
if (Z_REG(op1_addr) != ZREG_FP) {
| mov Ra(Z_REG(op1_addr)), T1 // restore
}
| SET_ZVAL_LVAL op1_addr, r0
| SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX
| mov FCARG1a, r0
if (op1_info & MAY_BE_ARRAY) {
| jmp >1
|.code
|1:
}
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) {
uint32_t var_info;
uint32_t var_def_info = zend_array_element_type(op1_def_info, opline->op1_type, 1, 0);
|6:
if (opline->op2_type == IS_UNUSED) {
var_info = MAY_BE_NULL;
| // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval));
| LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval
| EXT_CALL zend_hash_next_index_insert, r0
| // if (UNEXPECTED(!var_ptr)) {
| test r0, r0
| jz >1
|.cold_code
|1:
| // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied");
| CANNOT_ADD_ELEMENT opline
| //ZEND_VM_C_GOTO(assign_dim_op_ret_null);
| jmp >9
|.code
} else {
var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0);
if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) {
var_info |= MAY_BE_REF;
}
if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
var_info |= MAY_BE_RC1;
}
if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_RW, op1_info, op2_info, NULL, NULL, NULL)) {
return 0;
}
|8:
if (op1_info & (MAY_BE_ARRAY_OF_REF)) {
binary_op_type binary_op = get_binary_op(opline->extended_value);
| IF_NOT_Z_TYPE, r0, IS_REFERENCE, >1
| GET_Z_PTR FCARG1a, r0
| cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0
| jnz >2
| lea r0, aword [FCARG1a + offsetof(zend_reference, val)]
|.cold_code
|2:
| LOAD_ZVAL_ADDR FCARG2a, op3_addr
|.if X64
| LOAD_ADDR CARG3, binary_op
|.else
| sub r4, 12
| PUSH_ADDR binary_op, r0
|.endif
| SET_EX_OPLINE opline, r0
if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR))
&& (op1_data_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, r0
} else {
| EXT_CALL zend_jit_assign_op_to_typed_ref, r0
}
|.if not(X64)
| add r4, 12
|.endif
zend_jit_check_exception(Dst);
| jmp >9
|.code
|1:
}
}
var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0);
switch (opline->extended_value) {
case ZEND_ADD:
case ZEND_SUB:
case ZEND_MUL:
case ZEND_DIV:
if (!zend_jit_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, 0, var_addr, var_def_info, var_info,
1 /* may overflow */, may_throw)) {
return 0;
}
break;
case ZEND_BW_OR:
case ZEND_BW_AND:
case ZEND_BW_XOR:
case ZEND_SL:
case ZEND_SR:
case ZEND_MOD:
if (!zend_jit_long_math_helper(Dst, opline, opline->extended_value,
IS_CV, opline->op1, var_addr, var_info, NULL,
(opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info,
op1_data_range,
0, var_addr, var_def_info, var_info, may_throw)) {
return 0;
}
break;
case ZEND_CONCAT:
if (!zend_jit_concat_helper(Dst, opline, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, var_addr,
may_throw)) {
return 0;
}
break;
default:
ZEND_UNREACHABLE();
}
}
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) {
binary_op_type binary_op;
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) {
|.cold_code
|7:
}
| SET_EX_OPLINE opline, r0
if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
if (opline->op2_type == IS_UNUSED) {
| xor FCARG2a, FCARG2a
} else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
| LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1)
} else {
| LOAD_ZVAL_ADDR FCARG2a, op2_addr
}
binary_op = get_binary_op(opline->extended_value);
|.if X64
| LOAD_ZVAL_ADDR CARG3, op3_addr
| LOAD_ADDR CARG4, binary_op
|.else
| sub r4, 8
| PUSH_ADDR binary_op, r0
| PUSH_ZVAL_ADDR op3_addr, r0
|.endif
| EXT_CALL zend_jit_assign_dim_op_helper, r0
|.if not(X64)
| add r4, 8
|.endif
if (!zend_jit_check_exception(Dst)) {
return 0;
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) {
| jmp >9 // END
|.code
}
}
|9:
| FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline
return 1;
}
static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_ssa_range *op1_range, uint32_t op2_info, zend_ssa_range *op2_range, int may_overflow, int may_throw)
{
zend_jit_addr op1_addr, op2_addr;
ZEND_ASSERT(opline->op1_type == IS_CV && opline->result_type == IS_UNUSED);
ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF));
op1_addr = OP1_ADDR();
op2_addr = OP2_ADDR();
if (op1_info & MAY_BE_REF) {
binary_op_type binary_op = get_binary_op(opline->extended_value);
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
| IF_NOT_Z_TYPE, FCARG1a, IS_REFERENCE, >1
| GET_Z_PTR FCARG1a, FCARG1a
| cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0
| jnz >2
| add FCARG1a, offsetof(zend_reference, val)
|.cold_code
|2:
| LOAD_ZVAL_ADDR FCARG2a, op2_addr
|.if X64
| LOAD_ADDR CARG3, binary_op
|.else
| sub r4, 12
| PUSH_ADDR binary_op, r0
|.endif
| SET_EX_OPLINE opline, r0
if ((opline->op2_type & (IS_TMP_VAR|IS_VAR))
&& (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, r0
} else {
| EXT_CALL zend_jit_assign_op_to_typed_ref, r0
}
|.if not(X64)
| add r4, 12
|.endif
zend_jit_check_exception(Dst);
| jmp >9
|.code
|1:
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
}
int result;
switch (opline->extended_value) {
case ZEND_ADD:
case ZEND_SUB:
case ZEND_MUL:
case ZEND_DIV:
result = zend_jit_math_helper(Dst, opline, opline->extended_value, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->op1.var, op1_addr, op1_def_info, op1_info, may_overflow, may_throw);
break;
case ZEND_BW_OR:
case ZEND_BW_AND:
case ZEND_BW_XOR:
case ZEND_SL:
case ZEND_SR:
case ZEND_MOD:
result = zend_jit_long_math_helper(Dst, opline, opline->extended_value,
opline->op1_type, opline->op1, op1_addr, op1_info, op1_range,
opline->op2_type, opline->op2, op2_addr, op2_info, op2_range,
opline->op1.var, op1_addr, op1_def_info, op1_info, may_throw);
break;
case ZEND_CONCAT:
result = zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op1_addr, may_throw);
break;
default:
ZEND_UNREACHABLE();
}
|9:
return result;
}
static int zend_jit_is_constant_cmp_long_long(const zend_op *opline,
zend_ssa_range *op1_range,
zend_jit_addr op1_addr,
zend_ssa_range *op2_range,
zend_jit_addr op2_addr,
zend_bool *result)
{
zend_long op1_min;
zend_long op1_max;
zend_long op2_min;
zend_long op2_max;
if (op1_range) {
op1_min = op1_range->min;
op1_max = op1_range->max;
} else if (Z_MODE(op1_addr) == IS_CONST_ZVAL) {
ZEND_ASSERT(Z_TYPE_P(Z_ZV(op1_addr)) == IS_LONG);
op1_min = op1_max = Z_LVAL_P(Z_ZV(op1_addr));
} else {
return 0;
}
if (op2_range) {
op2_min = op2_range->min;
op2_max = op2_range->max;
} else if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
ZEND_ASSERT(Z_TYPE_P(Z_ZV(op2_addr)) == IS_LONG);
op2_min = op2_max = Z_LVAL_P(Z_ZV(op2_addr));
} else {
return 0;
}
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_CASE:
case ZEND_CASE_STRICT:
if (op1_min == op1_max && op2_min == op2_max && op1_min == op2_min) {
*result = 1;
return 1;
} else if (op1_max < op2_min || op1_min > op2_max) {
*result = 0;
return 1;
}
return 0;
case ZEND_IS_NOT_EQUAL:
case ZEND_IS_NOT_IDENTICAL:
if (op1_min == op1_max && op2_min == op2_max && op1_min == op2_min) {
*result = 0;
return 1;
} else if (op1_max < op2_min || op1_min > op2_max) {
*result = 1;
return 1;
}
return 0;
case ZEND_IS_SMALLER:
if (op1_max < op2_min) {
*result = 1;
return 1;
} else if (op1_min >= op2_max) {
*result = 0;
return 1;
}
return 0;
case ZEND_IS_SMALLER_OR_EQUAL:
if (op1_max <= op2_min) {
*result = 1;
return 1;
} else if (op1_min > op2_max) {
*result = 0;
return 1;
}
return 0;
default:
ZEND_UNREACHABLE();
}
return 0;
}
static int zend_jit_cmp_long_long(dasm_State **Dst,
const zend_op *opline,
zend_ssa_range *op1_range,
zend_jit_addr op1_addr,
zend_ssa_range *op2_range,
zend_jit_addr op2_addr,
zend_jit_addr res_addr,
zend_uchar smart_branch_opcode,
uint32_t target_label,
uint32_t target_label2,
const void *exit_addr,
zend_bool skip_comparison)
{
zend_bool swap = 0;
zend_bool result;
if (zend_jit_is_constant_cmp_long_long(opline, op1_range, op1_addr, op2_range, op2_addr, &result)) {
if (!smart_branch_opcode ||
smart_branch_opcode == ZEND_JMPZ_EX ||
smart_branch_opcode == ZEND_JMPNZ_EX) {
| SET_ZVAL_TYPE_INFO res_addr, (result ? IS_TRUE : IS_FALSE)
}
if (smart_branch_opcode && !exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ ||
smart_branch_opcode == ZEND_JMPZ_EX) {
if (!result) {
| jmp => target_label
}
} else if (smart_branch_opcode == ZEND_JMPNZ ||
smart_branch_opcode == ZEND_JMPNZ_EX) {
if (result) {
| jmp => target_label
}
} else if (smart_branch_opcode == ZEND_JMPZNZ) {
if (!result) {
| jmp => target_label
} else {
| jmp => target_label2
}
} else {
ZEND_UNREACHABLE();
}
}
return 1;
}
if (skip_comparison) {
if (Z_MODE(op1_addr) != IS_REG &&
(Z_MODE(op2_addr) == IS_REG ||
(Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL))) {
swap = 1;
}
} else if (Z_MODE(op1_addr) == IS_REG) {
if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) {
| test Ra(Z_REG(op1_addr)), Ra(Z_REG(op1_addr))
} else {
| LONG_OP cmp, Z_REG(op1_addr), op2_addr, r0
}
} else if (Z_MODE(op2_addr) == IS_REG) {
if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 0) {
| test Ra(Z_REG(op2_addr)), Ra(Z_REG(op2_addr))
} else {
| LONG_OP cmp, Z_REG(op2_addr), op1_addr, r0
}
swap = 1;
} else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL) {
| LONG_OP_WITH_CONST cmp, op2_addr, Z_LVAL_P(Z_ZV(op1_addr))
swap = 1;
} else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_MODE(op1_addr) != IS_CONST_ZVAL) {
| LONG_OP_WITH_CONST cmp, op1_addr, Z_LVAL_P(Z_ZV(op2_addr))
} else {
| GET_ZVAL_LVAL ZREG_R0, op1_addr
if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) {
| test r0, r0
} else {
| LONG_OP cmp, ZREG_R0, op2_addr, r0
}
}
if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ_EX ||
smart_branch_opcode == ZEND_JMPNZ_EX) {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_CASE:
case ZEND_CASE_STRICT:
| sete al
break;
case ZEND_IS_NOT_EQUAL:
case ZEND_IS_NOT_IDENTICAL:
| setne al
break;
case ZEND_IS_SMALLER:
if (swap) {
| setg al
} else {
| setl al
}
break;
case ZEND_IS_SMALLER_OR_EQUAL:
if (swap) {
| setge al
} else {
| setle al
}
break;
default:
ZEND_UNREACHABLE();
}
| movzx eax, al
| lea eax, [eax + 2]
| SET_ZVAL_TYPE_INFO res_addr, eax
}
if (smart_branch_opcode == ZEND_JMPZ ||
smart_branch_opcode == ZEND_JMPZ_EX) {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_CASE:
case ZEND_CASE_STRICT:
if (exit_addr) {
| jne &exit_addr
} else {
| jne => target_label
}
break;
case ZEND_IS_NOT_EQUAL:
if (exit_addr) {
| je &exit_addr
} else {
| je => target_label
}
break;
case ZEND_IS_NOT_IDENTICAL:
if (exit_addr) {
| jne &exit_addr
} else {
| je => target_label
}
break;
case ZEND_IS_SMALLER:
if (swap) {
if (exit_addr) {
| jle &exit_addr
} else {
| jle => target_label
}
} else {
if (exit_addr) {
| jge &exit_addr
} else {
| jge => target_label
}
}
break;
case ZEND_IS_SMALLER_OR_EQUAL:
if (swap) {
if (exit_addr) {
| jl &exit_addr
} else {
| jl => target_label
}
} else {
if (exit_addr) {
| jg &exit_addr
} else {
| jg => target_label
}
}
break;
default:
ZEND_UNREACHABLE();
}
} else if (smart_branch_opcode == ZEND_JMPNZ ||
smart_branch_opcode == ZEND_JMPNZ_EX) {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_CASE:
case ZEND_CASE_STRICT:
if (exit_addr) {
| je &exit_addr
} else {
| je => target_label
}
break;
case ZEND_IS_NOT_EQUAL:
if (exit_addr) {
| jne &exit_addr
} else {
| jne => target_label
}
break;
case ZEND_IS_NOT_IDENTICAL:
if (exit_addr) {
| je &exit_addr
} else {
| jne => target_label
}
break;
case ZEND_IS_SMALLER:
if (swap) {
if (exit_addr) {
| jg &exit_addr
} else {
| jg => target_label
}
} else {
if (exit_addr) {
| jl &exit_addr
} else {
| jl => target_label
}
}
break;
case ZEND_IS_SMALLER_OR_EQUAL:
if (swap) {
if (exit_addr) {
| jge &exit_addr
} else {
| jge => target_label
}
} else {
if (exit_addr) {
| jle &exit_addr
} else {
| jle => target_label
}
}
break;
default:
ZEND_UNREACHABLE();
}
} else if (smart_branch_opcode == ZEND_JMPZNZ) {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_CASE:
case ZEND_CASE_STRICT:
| jne => target_label
break;
case ZEND_IS_NOT_EQUAL:
case ZEND_IS_NOT_IDENTICAL:
| je => target_label
break;
case ZEND_IS_SMALLER:
if (swap) {
| jle => target_label
} else {
| jge => target_label
}
break;
case ZEND_IS_SMALLER_OR_EQUAL:
if (swap) {
| jl => target_label
} else {
| jg => target_label
}
break;
default:
ZEND_UNREACHABLE();
}
| jmp => target_label2
} else {
ZEND_UNREACHABLE();
}
} else {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_CASE:
case ZEND_CASE_STRICT:
| sete al
break;
case ZEND_IS_NOT_EQUAL:
case ZEND_IS_NOT_IDENTICAL:
| setne al
break;
case ZEND_IS_SMALLER:
if (swap) {
| setg al
} else {
| setl al
}
break;
case ZEND_IS_SMALLER_OR_EQUAL:
if (swap) {
| setge al
} else {
| setle al
}
break;
default:
ZEND_UNREACHABLE();
}
| movzx eax, al
| add eax, 2
| SET_ZVAL_TYPE_INFO res_addr, eax
}
return 1;
}
static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, zend_bool swap, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ) {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_CASE:
case ZEND_CASE_STRICT:
if (exit_addr) {
| jne &exit_addr
| jp &exit_addr
} else {
| jne => target_label
| jp => target_label
}
break;
case ZEND_IS_NOT_EQUAL:
| jp >1
if (exit_addr) {
| je &exit_addr
} else {
| je => target_label
}
|1:
break;
case ZEND_IS_NOT_IDENTICAL:
if (exit_addr) {
| jne &exit_addr
| jp &exit_addr
} else {
| jp >1
| je => target_label
|1:
}
break;
case ZEND_IS_SMALLER:
if (swap) {
if (exit_addr) {
| jbe &exit_addr
} else {
| jbe => target_label
}
} else {
if (exit_addr) {
| jae &exit_addr
| jp &exit_addr
} else {
| jae => target_label
| jp => target_label
}
}
break;
case ZEND_IS_SMALLER_OR_EQUAL:
if (swap) {
if (exit_addr) {
| jb &exit_addr
} else {
| jb => target_label
}
} else {
if (exit_addr) {
| ja &exit_addr
| jp &exit_addr
} else {
| ja => target_label
| jp => target_label
}
}
break;
default:
ZEND_UNREACHABLE();
}
} else if (smart_branch_opcode == ZEND_JMPNZ) {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_CASE:
case ZEND_CASE_STRICT:
| jp >1
if (exit_addr) {
| je &exit_addr
} else {
| je => target_label
}
|1:
break;
case ZEND_IS_NOT_EQUAL:
if (exit_addr) {
| jne &exit_addr
| jp &exit_addr
} else {
| jne => target_label
| jp => target_label
}
break;
case ZEND_IS_NOT_IDENTICAL:
if (exit_addr) {
| jp >1
| je &exit_addr
|1:
} else {
| jne => target_label
| jp => target_label
}
break;
case ZEND_IS_SMALLER:
if (swap) {
if (exit_addr) {
| ja &exit_addr
} else {
| ja => target_label
}
} else {
| jp >1
if (exit_addr) {
| jb &exit_addr
} else {
| jb => target_label
}
|1:
}
break;
case ZEND_IS_SMALLER_OR_EQUAL:
if (swap) {
if (exit_addr) {
| jae &exit_addr
} else {
| jae => target_label
}
} else {
| jp >1
if (exit_addr) {
| jbe &exit_addr
} else {
| jbe => target_label
}
|1:
}
break;
default:
ZEND_UNREACHABLE();
}
} else if (smart_branch_opcode == ZEND_JMPZNZ) {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_CASE:
case ZEND_CASE_STRICT:
| jne => target_label
| jp => target_label
break;
case ZEND_IS_NOT_EQUAL:
case ZEND_IS_NOT_IDENTICAL:
| jp => target_label2
| je => target_label
break;
case ZEND_IS_SMALLER:
if (swap) {
| jbe => target_label
} else {
| jae => target_label
| jp => target_label
}
break;
case ZEND_IS_SMALLER_OR_EQUAL:
if (swap) {
| jb => target_label
} else {
| ja => target_label
| jp => target_label
}
break;
default:
ZEND_UNREACHABLE();
}
| jmp => target_label2
} else if (smart_branch_opcode == ZEND_JMPZ_EX) {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_CASE:
case ZEND_CASE_STRICT:
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
| jne => target_label
| jp => target_label
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
break;
case ZEND_IS_NOT_EQUAL:
case ZEND_IS_NOT_IDENTICAL:
| jp >1
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
| je => target_label
|1:
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
break;
case ZEND_IS_SMALLER:
if (swap) {
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
| jbe => target_label
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
} else {
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
| jae => target_label
| jp => target_label
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
}
break;
case ZEND_IS_SMALLER_OR_EQUAL:
if (swap) {
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
| jb => target_label
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
} else {
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
| ja => target_label
| jp => target_label
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
}
break;
default:
ZEND_UNREACHABLE();
}
} else if (smart_branch_opcode == ZEND_JMPNZ_EX) {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_CASE:
case ZEND_CASE_STRICT:
| jp >1
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
| je => target_label
|1:
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
break;
case ZEND_IS_NOT_EQUAL:
case ZEND_IS_NOT_IDENTICAL:
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
| jne => target_label
| jp => target_label
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
break;
case ZEND_IS_SMALLER:
if (swap) {
| seta al
| movzx eax, al
| lea eax, [eax + 2]
| SET_ZVAL_TYPE_INFO res_addr, eax
| ja => target_label
} else {
| jp >1
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
| jb => target_label
|1:
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
}
break;
case ZEND_IS_SMALLER_OR_EQUAL:
if (swap) {
| setae al
| movzx eax, al
| lea eax, [eax + 2]
| SET_ZVAL_TYPE_INFO res_addr, eax
| jae => target_label
} else {
| jp >1
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
| jbe => target_label
|1:
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
}
break;
default:
ZEND_UNREACHABLE();
}
} else {
ZEND_UNREACHABLE();
}
} else {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_CASE:
case ZEND_CASE_STRICT:
| jp >1
| mov eax, IS_TRUE
| je >2
|1:
| mov eax, IS_FALSE
|2:
break;
case ZEND_IS_NOT_EQUAL:
case ZEND_IS_NOT_IDENTICAL:
| jp >1
| mov eax, IS_FALSE
| je >2
|1:
| mov eax, IS_TRUE
|2:
break;
case ZEND_IS_SMALLER:
if (swap) {
| seta al
| movzx eax, al
| add eax, 2
} else {
| jp >1
| mov eax, IS_TRUE
| jb >2
|1:
| mov eax, IS_FALSE
|2:
}
break;
case ZEND_IS_SMALLER_OR_EQUAL:
if (swap) {
| setae al
| movzx eax, al
| add eax, 2
} else {
| jp >1
| mov eax, IS_TRUE
| jbe >2
|1:
| mov eax, IS_FALSE
|2:
}
break;
default:
ZEND_UNREACHABLE();
}
| SET_ZVAL_TYPE_INFO res_addr, eax
}
return 1;
}
static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
zend_reg tmp_reg = ZREG_XMM0;
| SSE_GET_ZVAL_LVAL tmp_reg, op1_addr, ZREG_R0
| SSE_AVX_OP ucomisd, vucomisd, tmp_reg, op2_addr
return zend_jit_cmp_double_common(Dst, opline, res_addr, 0, smart_branch_opcode, target_label, target_label2, exit_addr);
}
static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
zend_reg tmp_reg = ZREG_XMM0;
| SSE_GET_ZVAL_LVAL tmp_reg, op2_addr, ZREG_R0
| SSE_AVX_OP ucomisd, vucomisd, tmp_reg, op1_addr
return zend_jit_cmp_double_common(Dst, opline, res_addr, /* swap */ 1, smart_branch_opcode, target_label, target_label2, exit_addr);
}
static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
zend_bool swap = 0;
if (Z_MODE(op1_addr) == IS_REG) {
| SSE_AVX_OP ucomisd, vucomisd, Z_REG(op1_addr), op2_addr
} else if (Z_MODE(op2_addr) == IS_REG) {
| SSE_AVX_OP ucomisd, vucomisd, Z_REG(op2_addr), op1_addr
swap = 1;
} else {
zend_reg tmp_reg = ZREG_XMM0;
| SSE_GET_ZVAL_DVAL tmp_reg, op1_addr
| SSE_AVX_OP ucomisd, vucomisd, tmp_reg, op2_addr
}
return zend_jit_cmp_double_common(Dst, opline, res_addr, swap, smart_branch_opcode, target_label, target_label2, exit_addr);
}
static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
| LONG_OP_WITH_CONST cmp, res_addr, Z_L(0)
if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ_EX ||
smart_branch_opcode == ZEND_JMPNZ_EX) {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_CASE:
| sete al
break;
case ZEND_IS_NOT_EQUAL:
| setne al
break;
case ZEND_IS_SMALLER:
| setl al
break;
case ZEND_IS_SMALLER_OR_EQUAL:
| setle al
break;
default:
ZEND_UNREACHABLE();
}
| movzx eax, al
| lea eax, [eax + 2]
| SET_ZVAL_TYPE_INFO res_addr, eax
}
if (smart_branch_opcode == ZEND_JMPZ ||
smart_branch_opcode == ZEND_JMPZ_EX) {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_CASE:
if (exit_addr) {
| jne &exit_addr
} else {
| jne => target_label
}
break;
case ZEND_IS_NOT_EQUAL:
if (exit_addr) {
| je &exit_addr
} else {
| je => target_label
}
break;
case ZEND_IS_SMALLER:
if (exit_addr) {
| jge &exit_addr
} else {
| jge => target_label
}
break;
case ZEND_IS_SMALLER_OR_EQUAL:
if (exit_addr) {
| jg &exit_addr
} else {
| jg => target_label
}
break;
default:
ZEND_UNREACHABLE();
}
} else if (smart_branch_opcode == ZEND_JMPNZ ||
smart_branch_opcode == ZEND_JMPNZ_EX) {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_CASE:
if (exit_addr) {
| je &exit_addr
} else {
| je => target_label
}
break;
case ZEND_IS_NOT_EQUAL:
if (exit_addr) {
| jne &exit_addr
} else {
| jne => target_label
}
break;
case ZEND_IS_SMALLER:
if (exit_addr) {
| jl &exit_addr
} else {
| jl => target_label
}
break;
case ZEND_IS_SMALLER_OR_EQUAL:
if (exit_addr) {
| jle &exit_addr
} else {
| jle => target_label
}
break;
default:
ZEND_UNREACHABLE();
}
} else if (smart_branch_opcode == ZEND_JMPZNZ) {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_CASE:
| jne => target_label
break;
case ZEND_IS_NOT_EQUAL:
| je => target_label
break;
case ZEND_IS_SMALLER:
| jge => target_label
break;
case ZEND_IS_SMALLER_OR_EQUAL:
| jg => target_label
break;
default:
ZEND_UNREACHABLE();
}
| jmp => target_label2
} else {
ZEND_UNREACHABLE();
}
} else {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_CASE:
| sete al
break;
case ZEND_IS_NOT_EQUAL:
| setne al
break;
case ZEND_IS_SMALLER:
| setl al
break;
case ZEND_IS_SMALLER_OR_EQUAL:
| setle al
break;
default:
ZEND_UNREACHABLE();
}
| movzx eax, al
| add eax, 2
| SET_ZVAL_TYPE_INFO res_addr, eax
}
return 1;
}
static int zend_jit_cmp(dasm_State **Dst,
const zend_op *opline,
uint32_t op1_info,
zend_ssa_range *op1_range,
zend_jit_addr op1_addr,
uint32_t op2_info,
zend_ssa_range *op2_range,
zend_jit_addr op2_addr,
zend_jit_addr res_addr,
int may_throw,
zend_uchar smart_branch_opcode,
uint32_t target_label,
uint32_t target_label2,
const void *exit_addr,
zend_bool skip_comparison)
{
zend_bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var);
zend_bool has_slow;
has_slow =
(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) ||
(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))));
if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
if (op1_info & MAY_BE_DOUBLE) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >4
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9
}
}
if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) {
if (op2_info & MAY_BE_DOUBLE) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3
|.cold_code
|3:
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9
}
if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
return 0;
}
| jmp >6
|.code
} else {
| IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9
}
}
if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) {
return 0;
}
if (op1_info & MAY_BE_DOUBLE) {
|.cold_code
|4:
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9
}
if (op2_info & MAY_BE_DOUBLE) {
if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) {
if (!same_ops) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >5
} else {
| IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9
}
}
if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
return 0;
}
| jmp >6
}
if (!same_ops) {
|5:
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9
}
if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
return 0;
}
| jmp >6
}
|.code
}
} else if ((op1_info & MAY_BE_DOUBLE) &&
!(op1_info & MAY_BE_LONG) &&
(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9
}
if (op2_info & MAY_BE_DOUBLE) {
if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) {
if (!same_ops && (op2_info & MAY_BE_LONG)) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >3
} else {
| IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9
}
}
if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
return 0;
}
}
if (!same_ops && (op2_info & MAY_BE_LONG)) {
if (op2_info & MAY_BE_DOUBLE) {
|.cold_code
}
|3:
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9
}
if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
return 0;
}
if (op2_info & MAY_BE_DOUBLE) {
| jmp >6
|.code
}
}
} else if ((op2_info & MAY_BE_DOUBLE) &&
!(op2_info & MAY_BE_LONG) &&
(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9
}
if (op1_info & MAY_BE_DOUBLE) {
if (!same_ops && (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) {
if (!same_ops && (op1_info & MAY_BE_LONG)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >3
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9
}
}
if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
return 0;
}
}
if (!same_ops && (op1_info & MAY_BE_LONG)) {
if (op1_info & MAY_BE_DOUBLE) {
|.cold_code
}
|3:
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9
}
if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
return 0;
}
if (op1_info & MAY_BE_DOUBLE) {
| jmp >6
|.code
}
}
}
if (has_slow ||
(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) ||
(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) {
if (has_slow) {
|.cold_code
|9:
}
| SET_EX_OPLINE opline, r0
if (Z_MODE(op1_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) {
return 0;
}
op1_addr = real_addr;
}
if (Z_MODE(op2_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) {
return 0;
}
op2_addr = real_addr;
}
| LOAD_ZVAL_ADDR FCARG2a, op1_addr
if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) {
| IF_NOT_Z_TYPE FCARG2a, IS_UNDEF, >1
| mov FCARG1a, opline->op1.var
| EXT_CALL zend_jit_undefined_op_helper, r0
| LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval
|1:
}
if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1
| mov T1, FCARG2a // save
| mov FCARG1a, opline->op2.var
| EXT_CALL zend_jit_undefined_op_helper, r0
| mov FCARG2a, T1 // restore
|.if X64
| LOAD_ADDR_ZTS CARG3, executor_globals, uninitialized_zval
|.else
| sub r4, 12
| PUSH_ADDR_ZTS executor_globals, uninitialized_zval, r0
|.endif
| jmp >2
|1:
|.if X64
| LOAD_ZVAL_ADDR CARG3, op2_addr
|.else
| sub r4, 12
| PUSH_ZVAL_ADDR op2_addr, r0
|.endif
|2:
} else {
|.if X64
| LOAD_ZVAL_ADDR CARG3, op2_addr
|.else
| sub r4, 12
| PUSH_ZVAL_ADDR op2_addr, r0
|.endif
}
| LOAD_ZVAL_ADDR FCARG1a, res_addr
| EXT_CALL compare_function, r0
|.if not(X64)
| add r4, 12
|.endif
if (opline->opcode != ZEND_CASE) {
| FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline
}
| FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline
if (may_throw) {
zend_jit_check_exception_undef_result(Dst, opline);
}
if (!zend_jit_cmp_slow(Dst, opline, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
return 0;
}
if (has_slow) {
| jmp >6
|.code
}
}
|6:
return 1;
}
static int zend_jit_identical(dasm_State **Dst,
const zend_op *opline,
uint32_t op1_info,
zend_ssa_range *op1_range,
zend_jit_addr op1_addr,
uint32_t op2_info,
zend_ssa_range *op2_range,
zend_jit_addr op2_addr,
zend_jit_addr res_addr,
int may_throw,
zend_uchar smart_branch_opcode,
uint32_t target_label,
uint32_t target_label2,
const void *exit_addr,
zend_bool skip_comparison)
{
uint32_t identical_label = (uint32_t)-1;
uint32_t not_identical_label = (uint32_t)-1;
if (smart_branch_opcode && !exit_addr) {
if (opline->opcode != ZEND_IS_NOT_IDENTICAL) {
if (smart_branch_opcode == ZEND_JMPZ) {
not_identical_label = target_label;
} else if (smart_branch_opcode == ZEND_JMPNZ) {
identical_label = target_label;
} else if (smart_branch_opcode == ZEND_JMPZNZ) {
not_identical_label = target_label;
identical_label = target_label2;
} else {
ZEND_UNREACHABLE();
}
} else {
if (smart_branch_opcode == ZEND_JMPZ) {
identical_label = target_label;
} else if (smart_branch_opcode == ZEND_JMPNZ) {
not_identical_label = target_label;
} else if (smart_branch_opcode == ZEND_JMPZNZ) {
identical_label = target_label;
not_identical_label = target_label2;
} else {
ZEND_UNREACHABLE();
}
}
}
if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG &&
(op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) {
if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) {
return 0;
}
return 1;
} else if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE &&
(op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE) {
if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
return 0;
}
return 1;
}
if ((op1_info & MAY_BE_UNDEF) && (op2_info & MAY_BE_UNDEF)) {
op1_info |= MAY_BE_NULL;
op2_info |= MAY_BE_NULL;
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
| IF_Z_TYPE FCARG1a, IS_UNDEF, >1
|.cold_code
|1:
| // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
| SET_EX_OPLINE opline, r0
| mov FCARG1d, opline->op1.var
| EXT_CALL zend_jit_undefined_op_helper, r0
if (may_throw) {
zend_jit_check_exception_undef_result(Dst, opline);
}
| LOAD_ADDR_ZTS FCARG1a, executor_globals, uninitialized_zval
| jmp >1
|.code
|1:
| LOAD_ZVAL_ADDR FCARG2a, op2_addr
| IF_Z_TYPE FCARG2a, IS_UNDEF, >1
|.cold_code
|1:
| // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
| SET_EX_OPLINE opline, r0
| mov aword T1, FCARG1a // save
| mov FCARG1d, opline->op2.var
| EXT_CALL zend_jit_undefined_op_helper, r0
if (may_throw) {
zend_jit_check_exception_undef_result(Dst, opline);
}
| mov FCARG1a, aword T1 // restore
| LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval
| jmp >1
|.code
|1:
} else if (op1_info & MAY_BE_UNDEF) {
op1_info |= MAY_BE_NULL;
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
| IF_Z_TYPE FCARG1a, IS_UNDEF, >1
|.cold_code
|1:
| // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
| SET_EX_OPLINE opline, r0
| mov FCARG1d, opline->op1.var
| EXT_CALL zend_jit_undefined_op_helper, r0
if (may_throw) {
zend_jit_check_exception_undef_result(Dst, opline);
}
| LOAD_ADDR_ZTS FCARG1a, executor_globals, uninitialized_zval
| jmp >1
|.code
|1:
if (opline->op2_type != IS_CONST) {
| LOAD_ZVAL_ADDR FCARG2a, op2_addr
}
} else if (op2_info & MAY_BE_UNDEF) {
op2_info |= MAY_BE_NULL;
| LOAD_ZVAL_ADDR FCARG2a, op2_addr
| IF_Z_TYPE FCARG2a, IS_UNDEF, >1
|.cold_code
|1:
| // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
| SET_EX_OPLINE opline, r0
| mov FCARG1d, opline->op2.var
| EXT_CALL zend_jit_undefined_op_helper, r0
if (may_throw) {
zend_jit_check_exception_undef_result(Dst, opline);
}
| LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval
| jmp >1
|.code
|1:
if (opline->op1_type != IS_CONST) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
} else {
if (opline->op1_type != IS_CONST) {
if (Z_MODE(op1_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) {
return 0;
}
op1_addr = real_addr;
}
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
if (opline->op2_type != IS_CONST) {
if (Z_MODE(op2_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) {
return 0;
}
op2_addr = real_addr;
}
| LOAD_ZVAL_ADDR FCARG2a, op2_addr
}
}
if (opline->op1_type & (IS_CV|IS_VAR)) {
| ZVAL_DEREF FCARG1a, op1_info
}
if (opline->op2_type & (IS_CV|IS_VAR)) {
| ZVAL_DEREF FCARG2a, op2_info
}
if ((op1_info & op2_info & MAY_BE_ANY) == 0) {
if ((opline->opcode != ZEND_CASE_STRICT &&
(opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) ||
((opline->op2_type & (IS_VAR|IS_TMP_VAR)) &&
(op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) {
| SET_EX_OPLINE opline, r0
if (opline->opcode != ZEND_CASE_STRICT) {
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline
}
| FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline
}
if (smart_branch_opcode) {
zend_jit_check_exception_undef_result(Dst, opline);
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
| jmp &exit_addr
}
} else if (not_identical_label != (uint32_t)-1) {
| jmp =>not_identical_label
}
} else {
| SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE)
zend_jit_check_exception(Dst);
}
} else if (has_concrete_type(op1_info) &&
has_concrete_type(op2_info) &&
concrete_type(op1_info) == concrete_type(op2_info) &&
concrete_type(op1_info) <= IS_TRUE) {
if (smart_branch_opcode) {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
| jmp &exit_addr
}
} else if (identical_label != (uint32_t)-1) {
| jmp =>identical_label
}
} else {
| SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE)
}
} else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) == IS_CONST_ZVAL) {
if (zend_is_identical(Z_ZV(op1_addr), Z_ZV(op2_addr))) {
if (smart_branch_opcode) {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
| jmp &exit_addr
}
} else if (identical_label != (uint32_t)-1) {
| jmp =>identical_label
}
} else {
| SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE)
}
} else {
if (smart_branch_opcode) {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
| jmp &exit_addr
}
} else if (not_identical_label != (uint32_t)-1) {
| jmp =>not_identical_label
}
} else {
| SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE)
}
}
} else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op1_addr)) <= IS_TRUE) {
zval *val = Z_ZV(op1_addr);
| cmp byte [FCARG2a + offsetof(zval, u1.v.type)], Z_TYPE_P(val)
if (smart_branch_opcode) {
if (opline->op2_type == IS_VAR && (op2_info & MAY_BE_REF)) {
| jne >8
| SET_EX_OPLINE opline, r0
| FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline
zend_jit_check_exception_undef_result(Dst, opline);
if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) {
| jmp &exit_addr
} else if (identical_label != (uint32_t)-1) {
| jmp =>identical_label
} else {
| jmp >9
}
|8:
} else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) {
| je &exit_addr
} else if (identical_label != (uint32_t)-1) {
| je =>identical_label
} else {
| je >9
}
} else {
if (opline->opcode != ZEND_IS_NOT_IDENTICAL) {
| sete al
} else {
| setne al
}
| movzx eax, al
| lea eax, [eax + 2]
| SET_ZVAL_TYPE_INFO res_addr, eax
}
if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) &&
(op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) {
| SET_EX_OPLINE opline, r0
| FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline
zend_jit_check_exception_undef_result(Dst, opline);
}
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
| jmp &exit_addr
}
} else if (smart_branch_opcode && not_identical_label != (uint32_t)-1) {
| jmp =>not_identical_label
}
} else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op2_addr)) <= IS_TRUE) {
zval *val = Z_ZV(op2_addr);
| cmp byte [FCARG1a + offsetof(zval, u1.v.type)], Z_TYPE_P(val)
if (smart_branch_opcode) {
if (opline->opcode != ZEND_CASE_STRICT
&& opline->op1_type == IS_VAR && (op1_info & MAY_BE_REF)) {
| jne >8
| SET_EX_OPLINE opline, r0
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline
zend_jit_check_exception_undef_result(Dst, opline);
if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) {
| jmp &exit_addr
} else if (identical_label != (uint32_t)-1) {
| jmp =>identical_label
} else {
| jmp >9
}
|8:
} else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) {
| je &exit_addr
} else if (identical_label != (uint32_t)-1) {
| je =>identical_label
} else {
| je >9
}
} else {
if (opline->opcode != ZEND_IS_NOT_IDENTICAL) {
| sete al
} else {
| setne al
}
| movzx eax, al
| lea eax, [eax + 2]
| SET_ZVAL_TYPE_INFO res_addr, eax
}
if (opline->opcode != ZEND_CASE_STRICT
&& (opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) {
| SET_EX_OPLINE opline, r0
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline
zend_jit_check_exception_undef_result(Dst, opline);
}
if (smart_branch_opcode) {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
| jmp &exit_addr
}
} else if (not_identical_label != (uint32_t)-1) {
| jmp =>not_identical_label
}
}
} else {
if (opline->op1_type == IS_CONST) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
if (opline->op2_type == IS_CONST) {
| LOAD_ZVAL_ADDR FCARG2a, op2_addr
}
| EXT_CALL zend_is_identical, r0
if ((opline->opcode != ZEND_CASE_STRICT &&
(opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) ||
((opline->op2_type & (IS_VAR|IS_TMP_VAR)) &&
(op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) {
| mov aword T1, r0 // save
| SET_EX_OPLINE opline, r0
if (opline->opcode != ZEND_CASE_STRICT) {
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline
}
| FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline
zend_jit_check_exception_undef_result(Dst, opline);
| mov r0, aword T1 // restore
}
if (smart_branch_opcode) {
| test al, al
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
| jnz &exit_addr
} else {
| jz &exit_addr
}
} else if (not_identical_label != (uint32_t)-1) {
| jz =>not_identical_label
if (identical_label != (uint32_t)-1) {
| jmp =>identical_label
}
} else if (identical_label != (uint32_t)-1) {
| jnz =>identical_label
}
} else {
| movzx eax, al
if (opline->opcode != ZEND_IS_NOT_IDENTICAL) {
| lea eax, [eax + 2]
} else {
| neg eax
| lea eax, [eax + 3]
}
| SET_ZVAL_TYPE_INFO res_addr, eax
}
}
|9:
if (may_throw) {
zend_jit_check_exception(Dst);
}
return 1;
}
static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, uint32_t target_label, uint32_t target_label2, int may_throw, zend_uchar branch_opcode, const void *exit_addr)
{
uint32_t true_label = -1;
uint32_t false_label = -1;
zend_bool set_bool = 0;
zend_bool set_bool_not = 0;
zend_bool set_delayed = 0;
zend_bool jmp_done = 0;
if (branch_opcode == ZEND_BOOL) {
set_bool = 1;
} else if (branch_opcode == ZEND_BOOL_NOT) {
set_bool = 1;
set_bool_not = 1;
} else if (branch_opcode == ZEND_JMPZ) {
false_label = target_label;
} else if (branch_opcode == ZEND_JMPNZ) {
true_label = target_label;
} else if (branch_opcode == ZEND_JMPZNZ) {
true_label = target_label2;
false_label = target_label;
} else if (branch_opcode == ZEND_JMPZ_EX) {
set_bool = 1;
false_label = target_label;
} else if (branch_opcode == ZEND_JMPNZ_EX) {
set_bool = 1;
true_label = target_label;
} else {
ZEND_UNREACHABLE();
}
if (Z_MODE(op1_addr) == IS_CONST_ZVAL) {
if (zend_is_true(Z_ZV(op1_addr))) {
/* Always TRUE */
if (set_bool) {
if (set_bool_not) {
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
} else {
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
}
}
if (true_label != (uint32_t)-1) {
| jmp =>true_label;
}
} else {
/* Always FALSE */
if (set_bool) {
if (set_bool_not) {
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
} else {
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
}
}
if (false_label != (uint32_t)-1) {
| jmp =>false_label;
}
}
return 1;
}
if (opline->op1_type == IS_CV && (op1_info & MAY_BE_REF)) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
| ZVAL_DEREF FCARG1a, op1_info
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) {
if (!(op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_TRUE))) {
/* Always TRUE */
if (set_bool) {
if (set_bool_not) {
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
} else {
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
}
}
if (true_label != (uint32_t)-1) {
| jmp =>true_label;
}
} else {
if (!(op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE)))) {
/* Always FALSE */
if (set_bool) {
if (set_bool_not) {
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
} else {
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
}
}
} else {
| CMP_ZVAL_TYPE op1_addr, IS_TRUE
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) {
if ((op1_info & MAY_BE_LONG) &&
!(op1_info & MAY_BE_UNDEF) &&
!set_bool) {
if (exit_addr) {
if (branch_opcode == ZEND_JMPNZ) {
| jl >9
} else {
| jl &exit_addr
}
} else if (false_label != (uint32_t)-1) {
| jl =>false_label
} else {
| jl >9
}
jmp_done = 1;
} else {
| jg >2
}
}
if (!(op1_info & MAY_BE_TRUE)) {
/* It's FALSE */
if (set_bool) {
if (set_bool_not) {
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
} else {
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
}
}
} else {
if (exit_addr) {
if (set_bool) {
| jne >1
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
| jmp &exit_addr
} else {
| jmp >9
}
|1:
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) {
if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) {
| jne &exit_addr
}
}
} else {
if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
| je &exit_addr
} else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) {
| jne &exit_addr
} else {
| je >9
}
}
} else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) {
if (set_bool) {
| jne >1
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
if (true_label != (uint32_t)-1) {
| jmp =>true_label
} else {
| jmp >9
}
|1:
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
} else {
if (true_label != (uint32_t)-1) {
| je =>true_label
} else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) {
| jne =>false_label
jmp_done = 1;
} else {
| je >9
}
}
} else if (set_bool) {
| sete al
| movzx eax, al
if (set_bool_not) {
| neg eax
| add eax, 3
} else {
| add eax, 2
}
if ((op1_info & MAY_BE_UNDEF) && (op1_info & MAY_BE_ANY)) {
set_delayed = 1;
} else {
| SET_ZVAL_TYPE_INFO res_addr, eax
}
}
}
}
/* It's FALSE, but may be UNDEF */
if (op1_info & MAY_BE_UNDEF) {
if (op1_info & MAY_BE_ANY) {
if (set_delayed) {
| CMP_ZVAL_TYPE op1_addr, IS_UNDEF
| SET_ZVAL_TYPE_INFO res_addr, eax
| jz >1
} else {
| IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1
}
|.cold_code
|1:
}
| mov FCARG1d, opline->op1.var
| SET_EX_OPLINE opline, r0
| EXT_CALL zend_jit_undefined_op_helper, r0
if (may_throw) {
if (!zend_jit_check_exception_undef_result(Dst, opline)) {
return 0;
}
}
if (exit_addr) {
if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) {
| jmp &exit_addr
}
} else if (false_label != (uint32_t)-1) {
| jmp =>false_label
}
if (op1_info & MAY_BE_ANY) {
if (exit_addr) {
if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
| jmp >9
}
} else if (false_label == (uint32_t)-1) {
| jmp >9
}
|.code
}
}
if (!jmp_done) {
if (exit_addr) {
if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
if (op1_info & MAY_BE_LONG) {
| jmp >9
}
} else if (op1_info & MAY_BE_LONG) {
| jmp &exit_addr
}
} else if (false_label != (uint32_t)-1) {
| jmp =>false_label
} else if ((op1_info & MAY_BE_LONG) || (op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
| jmp >9
}
}
}
}
if (op1_info & MAY_BE_LONG) {
|2:
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2
}
if (Z_MODE(op1_addr) == IS_REG) {
| test Ra(Z_REG(op1_addr)), Ra(Z_REG(op1_addr))
} else {
| LONG_OP_WITH_CONST, cmp, op1_addr, Z_L(0)
}
if (set_bool) {
| setne al
| movzx eax, al
if (set_bool_not) {
| neg eax
| add eax, 3
} else {
| lea eax, [eax + 2]
}
| SET_ZVAL_TYPE_INFO res_addr, eax
}
if (exit_addr) {
if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
| jne &exit_addr
} else {
| je &exit_addr
}
} else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) {
if (true_label != (uint32_t)-1) {
| jne =>true_label
if (false_label != (uint32_t)-1) {
| jmp =>false_label
}
} else {
| je =>false_label
}
}
}
if ((op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) == MAY_BE_DOUBLE) {
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
|.cold_code
}
|2:
if (CAN_USE_AVX()) {
| vxorps xmm0, xmm0, xmm0
} else {
| xorps xmm0, xmm0
}
| SSE_AVX_OP ucomisd, vucomisd, ZREG_XMM0, op1_addr
if (set_bool) {
if (exit_addr) {
if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
| jp &exit_addr
| jne &exit_addr
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
} else {
| jp >1
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
| je &exit_addr
|1:
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
}
} else if (false_label != (uint32_t)-1) { // JMPZ_EX (p=>true, z=>false, false=>jmp)
| jp >1
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
| je => false_label
|1:
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
} else if (true_label != (uint32_t)-1) { // JMPNZ_EX (p=>true, z=>false, true=>jmp)
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
| jp => true_label
| jne => true_label
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
} else if (set_bool_not) { // BOOL_NOT (p=>false, z=>true)
| mov eax, IS_FALSE
| jp >1
| jne >1
| mov eax, IS_TRUE
|1:
| SET_ZVAL_TYPE_INFO res_addr, eax
} else { // BOOL (p=>true, z=>false)
| mov eax, IS_TRUE
| jp >1
| jne >1
| mov eax, IS_FALSE
|1:
| SET_ZVAL_TYPE_INFO res_addr, eax
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
| jmp >9
|.code
}
} else {
if (exit_addr) {
if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
| jp &exit_addr
| jne &exit_addr
|1:
} else {
| jp >1
| je &exit_addr
|1:
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
| jmp >9
}
} else {
ZEND_ASSERT(true_label != (uint32_t)-1 || false_label != (uint32_t)-1);
if (false_label != (uint32_t)-1 ) {
| jp >1
| je => false_label
|1:
if (true_label != (uint32_t)-1) {
| jmp =>true_label
} else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
| jmp >9
}
} else {
| jp => true_label
| jne => true_label
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
| jmp >9
}
}
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
|.code
}
}
} else if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) {
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
|.cold_code
|2:
}
if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
| SET_EX_OPLINE opline, r0
| EXT_CALL zend_is_true, r0
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| IF_NOT_ZVAL_REFCOUNTED op1_addr, >3
}
| GET_ZVAL_PTR FCARG1a, op1_addr
| GC_DELREF FCARG1a
| jnz >3
| mov aword T1, r0 // save
| ZVAL_DTOR_FUNC op1_info, opline
| mov r0, aword T1 // restore
|3:
}
if (may_throw) {
| MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r1
| jne ->exception_handler_undef
}
if (set_bool) {
if (set_bool_not) {
| neg eax
| add eax, 3
} else {
| add eax, 2
}
| SET_ZVAL_TYPE_INFO res_addr, eax
if (exit_addr) {
| CMP_ZVAL_TYPE res_addr, IS_FALSE
if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
| jne &exit_addr
} else {
| je &exit_addr
}
} else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) {
| CMP_ZVAL_TYPE res_addr, IS_FALSE
if (true_label != (uint32_t)-1) {
| jne =>true_label
if (false_label != (uint32_t)-1) {
| jmp =>false_label
} else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
| jmp >9
}
} else {
| je =>false_label
}
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
| jmp >9
|.code
}
} else {
| test r0, r0
if (exit_addr) {
if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
| jne &exit_addr
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
| jmp >9
}
} else {
| je &exit_addr
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
| jmp >9
}
}
} else if (true_label != (uint32_t)-1) {
| jne =>true_label
if (false_label != (uint32_t)-1) {
| jmp =>false_label
} else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
| jmp >9
}
} else {
| je =>false_label
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
| jmp >9
}
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
|.code
}
}
}
|9:
return 1;
}
static int zend_jit_qm_assign(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr)
{
if (op1_addr != op1_def_addr) {
if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, op1_info)) {
return 0;
}
if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) {
op1_addr = op1_def_addr;
}
}
if (!zend_jit_simple_assign(Dst, opline, res_addr, res_use_info, res_info, opline->op1_type, op1_addr, op1_info, 0, 0, 0)) {
return 0;
}
if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) {
return 0;
}
if (op1_info & MAY_BE_UNDEF) {
zend_jit_check_exception(Dst);
}
return 1;
}
static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_use_addr, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr op2_def_addr, uint32_t res_info, zend_jit_addr res_addr, int may_throw)
{
ZEND_ASSERT(opline->op1_type == IS_CV);
if (op2_addr != op2_def_addr) {
if (!zend_jit_update_regs(Dst, opline->op2.var, op2_addr, op2_def_addr, op2_info)) {
return 0;
}
if (Z_MODE(op2_def_addr) == IS_REG && Z_MODE(op2_addr) != IS_REG) {
op2_addr = op2_def_addr;
}
}
if (Z_MODE(op1_addr) != IS_REG
&& Z_MODE(op1_use_addr) == IS_REG
&& !Z_LOAD(op1_use_addr)
&& !Z_STORE(op1_use_addr)) {
/* Force type update */
op1_info |= MAY_BE_UNDEF;
}
if (!zend_jit_assign_to_variable(Dst, opline, op1_use_addr, op1_addr, op1_info, op1_def_info, opline->op2_type, op2_addr, op2_info, res_addr,
may_throw)) {
return 0;
}
if (!zend_jit_store_var_if_necessary_ex(Dst, opline->op1.var, op1_addr, op1_def_info, op1_use_addr, op1_info)) {
return 0;
}
if (opline->result_type != IS_UNUSED) {
if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) {
return 0;
}
}
return 1;
}
/* copy of hidden zend_closure */
typedef struct _zend_closure {
zend_object std;
zend_function func;
zval this_ptr;
zend_class_entry *called_scope;
zif_handler orig_internal_handler;
} zend_closure;
static int zend_jit_stack_check(dasm_State **Dst, const zend_op *opline, uint32_t used_stack)
{
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| // Check Stack Overflow
| MEM_OP2_2_ZTS mov, r1, aword, executor_globals, vm_stack_end, r0
| MEM_OP2_2_ZTS sub, r1, aword, executor_globals, vm_stack_top, r0
| cmp r1, used_stack
| jb &exit_addr
return 1;
}
static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_function *func, zend_bool is_closure, zend_bool use_this, zend_bool stack_check)
{
uint32_t used_stack;
if (func) {
used_stack = zend_vm_calc_used_stack(opline->extended_value, func);
} else {
used_stack = (ZEND_CALL_FRAME_SLOT + opline->extended_value) * sizeof(zval);
| // if (EXPECTED(ZEND_USER_CODE(func->type))) {
if (!is_closure) {
| test byte [r0 + offsetof(zend_function, type)], 1
| mov FCARG1a, used_stack
| jnz >1
} else {
| mov FCARG1a, used_stack
}
| // used_stack += (func->op_array.last_var + func->op_array.T - MIN(func->op_array.num_args, num_args)) * sizeof(zval);
| mov edx, opline->extended_value
if (!is_closure) {
| cmp edx, dword [r0 + offsetof(zend_function, op_array.num_args)]
| cmova edx, dword [r0 + offsetof(zend_function, op_array.num_args)]
| sub edx, dword [r0 + offsetof(zend_function, op_array.last_var)]
| sub edx, dword [r0 + offsetof(zend_function, op_array.T)]
} else {
| cmp edx, dword [r0 + offsetof(zend_closure, func.op_array.num_args)]
| cmova edx, dword [r0 + offsetof(zend_closure, func.op_array.num_args)]
| sub edx, dword [r0 + offsetof(zend_closure, func.op_array.last_var)]
| sub edx, dword [r0 + offsetof(zend_closure, func.op_array.T)]
}
| shl edx, 4
|.if X64
| movsxd r2, edx
|.endif
| sub FCARG1a, r2
|1:
}
zend_jit_start_reuse_ip();
| // if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) {
| MEM_OP2_2_ZTS mov, RX, aword, executor_globals, vm_stack_top, RX
if (stack_check) {
| // Check Stack Overflow
| MEM_OP2_2_ZTS mov, r2, aword, executor_globals, vm_stack_end, r2
| sub r2, RX
if (func) {
| cmp r2, used_stack
} else {
| cmp r2, FCARG1a
}
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| jb &exit_addr
} else {
| jb >1
| // EG(vm_stack_top) = (zval*)((char*)call + used_stack);
|.cold_code
|1:
if (func) {
| mov FCARG1d, used_stack
}
#ifdef _WIN32
if (0) {
#else
if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) {
#endif
| SET_EX_OPLINE opline, r0
| EXT_CALL zend_jit_int_extend_stack_helper, r0
} else {
if (!is_closure) {
if (func
&& op_array == &func->op_array
&& (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE)
&& (sizeof(void*) != 8 || IS_SIGNED_32BIT(func))) {
| LOAD_ADDR FCARG2a, func
} else {
| mov FCARG2a, r0
}
} else {
| lea FCARG2a, aword [r0 + offsetof(zend_closure, func)]
}
| SET_EX_OPLINE opline, r0
| EXT_CALL zend_jit_extend_stack_helper, r0
}
| mov RX, r0
| jmp >1
|.code
}
}
if (func) {
| MEM_OP2_1_ZTS add, aword, executor_globals, vm_stack_top, used_stack, r2
} else {
| MEM_OP2_1_ZTS add, aword, executor_globals, vm_stack_top, FCARG1a, r2
}
| // zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object);
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || opline->opcode != ZEND_INIT_METHOD_CALL) {
| // ZEND_SET_CALL_INFO(call, 0, call_info);
| mov dword EX:RX->This.u1.type_info, (IS_UNDEF | ZEND_CALL_NESTED_FUNCTION)
}
#ifdef _WIN32
if (0) {
#else
if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) {
#endif
| // call->func = func;
|1:
| ADDR_OP2_2 mov, aword EX:RX->func, func, r1
} else {
if (!is_closure) {
| // call->func = func;
if (func
&& op_array == &func->op_array
&& (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE)
&& (sizeof(void*) != 8 || IS_SIGNED_32BIT(func))) {
| ADDR_OP2_2 mov, aword EX:RX->func, func, r1
} else {
| mov aword EX:RX->func, r0
}
} else {
| // call->func = &closure->func;
| lea r1, aword [r0 + offsetof(zend_closure, func)]
| mov aword EX:RX->func, r1
}
|1:
}
if (opline->opcode == ZEND_INIT_METHOD_CALL) {
| // Z_PTR(call->This) = obj;
| mov r1, aword T1
| mov aword EX:RX->This.value.ptr, r1
if (opline->op1_type == IS_UNUSED || use_this) {
| // call->call_info |= ZEND_CALL_HAS_THIS;
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
| mov dword EX:RX->This.u1.type_info, ZEND_CALL_HAS_THIS
} else {
| or dword EX:RX->This.u1.type_info, ZEND_CALL_HAS_THIS
}
} else {
if (opline->op1_type == IS_CV) {
| // GC_ADDREF(obj);
| add dword [r1], 1
}
| // call->call_info |= ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS;
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
| mov dword EX:RX->This.u1.type_info, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS)
} else {
| or dword EX:RX->This.u1.type_info, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS)
}
}
} else if (!is_closure) {
| // Z_CE(call->This) = called_scope;
| mov aword EX:RX->This.value.ptr, 0
} else {
if (opline->op2_type == IS_CV) {
| // GC_ADDREF(closure);
| add dword [r0], 1
}
| // object_or_called_scope = closure->called_scope;
| mov r1, aword [r0 + offsetof(zend_closure, called_scope)]
| mov aword EX:RX->This.value.ptr, r1
| // call_info = ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE |
| // (closure->func->common.fn_flags & ZEND_ACC_FAKE_CLOSURE);
| mov edx, dword [r0 + offsetof(zend_closure, func.common.fn_flags)]
| and edx, ZEND_ACC_FAKE_CLOSURE
| or edx, (ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE)
| // if (Z_TYPE(closure->this_ptr) != IS_UNDEF) {
| cmp byte [r0 + offsetof(zend_closure, this_ptr.u1.v.type)], IS_UNDEF
| jz >1
| // call_info |= ZEND_CALL_HAS_THIS;
| or edx, ZEND_CALL_HAS_THIS
| // object_or_called_scope = Z_OBJ(closure->this_ptr);
| mov r1, aword [r0 + offsetof(zend_closure, this_ptr.value.ptr)]
|1:
| // ZEND_SET_CALL_INFO(call, 0, call_info);
| or dword EX:RX->This.u1.type_info, edx
| // Z_PTR(call->This) = object_or_called_scope;
| mov aword EX:RX->This.value.ptr, r1
| cmp aword [r0 + offsetof(zend_closure, func.op_array.run_time_cache__ptr)], 0
| jnz >1
| lea FCARG1a, aword [r0 + offsetof(zend_closure, func)]
| EXT_CALL zend_jit_init_func_run_time_cache_helper, r0
|1:
}
| // ZEND_CALL_NUM_ARGS(call) = num_args;
| mov dword EX:RX->This.u2.num_args, opline->extended_value
return 1;
}
static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, const zend_op *opline, int call_level, zend_jit_trace_rec *trace)
{
int skip;
if (trace) {
zend_jit_trace_rec *p = trace;
ssa_op++;
while (1) {
if (p->op == ZEND_JIT_TRACE_VM) {
switch (p->opline->opcode) {
case ZEND_SEND_ARRAY:
case ZEND_SEND_USER:
case ZEND_SEND_UNPACK:
case ZEND_INIT_FCALL:
case ZEND_INIT_METHOD_CALL:
case ZEND_INIT_STATIC_METHOD_CALL:
case ZEND_INIT_FCALL_BY_NAME:
case ZEND_INIT_NS_FCALL_BY_NAME:
case ZEND_INIT_DYNAMIC_CALL:
case ZEND_NEW:
case ZEND_INIT_USER_CALL:
case ZEND_FAST_CALL:
case ZEND_JMP:
case ZEND_JMPZNZ:
case ZEND_JMPZ:
case ZEND_JMPNZ:
case ZEND_JMPZ_EX:
case ZEND_JMPNZ_EX:
case ZEND_FE_RESET_R:
case ZEND_FE_RESET_RW:
case ZEND_JMP_SET:
case ZEND_COALESCE:
case ZEND_JMP_NULL:
case ZEND_ASSERT_CHECK:
case ZEND_CATCH:
case ZEND_DECLARE_ANON_CLASS:
case ZEND_FE_FETCH_R:
case ZEND_FE_FETCH_RW:
return 1;
case ZEND_DO_ICALL:
case ZEND_DO_UCALL:
case ZEND_DO_FCALL_BY_NAME:
case ZEND_DO_FCALL:
return 0;
case ZEND_SEND_VAL:
case ZEND_SEND_VAR:
case ZEND_SEND_VAL_EX:
case ZEND_SEND_VAR_EX:
case ZEND_SEND_FUNC_ARG:
case ZEND_SEND_REF:
case ZEND_SEND_VAR_NO_REF:
case ZEND_SEND_VAR_NO_REF_EX:
/* skip */
break;
default:
if (zend_may_throw(opline, ssa_op, op_array, ssa)) {
return 1;
}
}
ssa_op += zend_jit_trace_op_len(opline);
} else if (p->op == ZEND_JIT_TRACE_ENTER ||
p->op == ZEND_JIT_TRACE_BACK ||
p->op == ZEND_JIT_TRACE_END) {
return 1;
}
p++;
}
}
if (!call_info) {
const zend_op *end = op_array->opcodes + op_array->last;
opline++;
ssa_op++;
skip = (call_level == 1);
while (opline != end) {
if (!skip) {
if (zend_may_throw(opline, ssa_op, op_array, ssa)) {
return 1;
}
}
switch (opline->opcode) {
case ZEND_SEND_VAL:
case ZEND_SEND_VAR:
case ZEND_SEND_VAL_EX:
case ZEND_SEND_VAR_EX:
case ZEND_SEND_FUNC_ARG:
case ZEND_SEND_REF:
case ZEND_SEND_VAR_NO_REF:
case ZEND_SEND_VAR_NO_REF_EX:
skip = 0;
break;
case ZEND_SEND_ARRAY:
case ZEND_SEND_USER:
case ZEND_SEND_UNPACK:
case ZEND_INIT_FCALL:
case ZEND_INIT_METHOD_CALL:
case ZEND_INIT_STATIC_METHOD_CALL:
case ZEND_INIT_FCALL_BY_NAME:
case ZEND_INIT_NS_FCALL_BY_NAME:
case ZEND_INIT_DYNAMIC_CALL:
case ZEND_NEW:
case ZEND_INIT_USER_CALL:
case ZEND_FAST_CALL:
case ZEND_JMP:
case ZEND_JMPZNZ:
case ZEND_JMPZ:
case ZEND_JMPNZ:
case ZEND_JMPZ_EX:
case ZEND_JMPNZ_EX:
case ZEND_FE_RESET_R:
case ZEND_FE_RESET_RW:
case ZEND_JMP_SET:
case ZEND_COALESCE:
case ZEND_JMP_NULL:
case ZEND_ASSERT_CHECK:
case ZEND_CATCH:
case ZEND_DECLARE_ANON_CLASS:
case ZEND_FE_FETCH_R:
case ZEND_FE_FETCH_RW:
return 1;
case ZEND_DO_ICALL:
case ZEND_DO_UCALL:
case ZEND_DO_FCALL_BY_NAME:
case ZEND_DO_FCALL:
end = opline;
if (end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) {
/* INIT_FCALL and DO_FCALL in different BasicBlocks */
return 1;
}
return 0;
}
opline++;
ssa_op++;
}
return 1;
} else {
const zend_op *end = call_info->caller_call_opline;
if (end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) {
/* INIT_FCALL and DO_FCALL in different BasicBlocks */
return 1;
}
opline++;
ssa_op++;
skip = (call_level == 1);
while (opline != end) {
if (skip) {
switch (opline->opcode) {
case ZEND_SEND_VAL:
case ZEND_SEND_VAR:
case ZEND_SEND_VAL_EX:
case ZEND_SEND_VAR_EX:
case ZEND_SEND_FUNC_ARG:
case ZEND_SEND_REF:
case ZEND_SEND_VAR_NO_REF:
case ZEND_SEND_VAR_NO_REF_EX:
skip = 0;
break;
case ZEND_SEND_ARRAY:
case ZEND_SEND_USER:
case ZEND_SEND_UNPACK:
return 1;
}
} else {
if (zend_may_throw(opline, ssa_op, op_array, ssa)) {
return 1;
}
}
opline++;
ssa_op++;
}
return 0;
}
}
static int zend_jit_init_fcall_guard(dasm_State **Dst, uint32_t level, const zend_function *func, const zend_op *to_opline)
{
int32_t exit_point;
const void *exit_addr;
if (func->type == ZEND_INTERNAL_FUNCTION) {
#ifdef ZEND_WIN32
// TODO: ASLR may cause different addresses in different workers ???
return 0;
#endif
} else if (func->type == ZEND_USER_FUNCTION) {
if (!zend_accel_in_shm(func->op_array.opcodes)) {
/* op_array and op_array->opcodes are not persistent. We can't link. */
return 0;
}
} else {
ZEND_UNREACHABLE();
return 0;
}
exit_point = zend_jit_trace_get_exit_point(to_opline, ZEND_JIT_EXIT_POLYMORPHISM);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| // call = EX(call);
| mov r1, EX->call
while (level > 0) {
| mov r1, EX:r1->prev_execute_data
level--;
}
if (func->type == ZEND_USER_FUNCTION &&
(!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) ||
(func->common.fn_flags & ZEND_ACC_CLOSURE) ||
!func->common.function_name)) {
const zend_op *opcodes = func->op_array.opcodes;
| mov r1, aword EX:r1->func
| .if X64
|| if (!IS_SIGNED_32BIT(opcodes)) {
| mov64 r2, ((ptrdiff_t)opcodes)
| cmp aword [r1 + offsetof(zend_op_array, opcodes)], r2
|| } else {
| cmp aword [r1 + offsetof(zend_op_array, opcodes)], opcodes
|| }
| .else
| cmp aword [r1 + offsetof(zend_op_array, opcodes)], opcodes
| .endif
| jne &exit_addr
} else {
| .if X64
|| if (!IS_SIGNED_32BIT(func)) {
| mov64 r2, ((ptrdiff_t)func)
| cmp aword EX:r1->func, r2
|| } else {
| cmp aword EX:r1->func, func
|| }
| .else
| cmp aword EX:r1->func, func
| .endif
| jne &exit_addr
}
return 1;
}
static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, int call_level, zend_jit_trace_rec *trace, zend_bool stack_check)
{
zend_func_info *info = ZEND_FUNC_INFO(op_array);
zend_call_info *call_info = NULL;
zend_function *func = NULL;
if (delayed_call_chain) {
if (!zend_jit_save_call_chain(Dst, delayed_call_level)) {
return 0;
}
}
if (info) {
call_info = info->callee_info;
while (call_info && call_info->caller_init_opline != opline) {
call_info = call_info->next_callee;
}
if (call_info && call_info->callee_func) {
func = call_info->callee_func;
}
}
if (!func
&& trace
&& trace->op == ZEND_JIT_TRACE_INIT_CALL) {
#ifdef _WIN32
/* ASLR */
if (trace->func->type != ZEND_INTERNAL_FUNCTION) {
func = (zend_function*)trace->func;
}
#else
func = (zend_function*)trace->func;
#endif
}
#ifdef _WIN32
if (0) {
#else
if (opline->opcode == ZEND_INIT_FCALL
&& func
&& func->type == ZEND_INTERNAL_FUNCTION) {
#endif
/* load constant address later */
} else if (func && op_array == &func->op_array) {
/* recursive call */
if (!(func->op_array.fn_flags & ZEND_ACC_IMMUTABLE) ||
(sizeof(void*) == 8 && !IS_SIGNED_32BIT(func))) {
| mov r0, EX->func
}
} else {
| // if (CACHED_PTR(opline->result.num))
| mov r0, EX->run_time_cache
| mov r0, aword [r0 + opline->result.num]
| test r0, r0
| jz >1
|.cold_code
|1:
if (opline->opcode == ZEND_INIT_FCALL
&& func
&& func->type == ZEND_USER_FUNCTION
&& (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE)) {
| LOAD_ADDR FCARG1a, func
| EXT_CALL zend_jit_init_func_run_time_cache_helper, r0
| mov r1, EX->run_time_cache
| mov aword [r1 + opline->result.num], r0
| jmp >3
} else {
zval *zv = RT_CONSTANT(opline, opline->op2);
if (opline->opcode == ZEND_INIT_FCALL) {
| LOAD_ADDR FCARG1a, Z_STR_P(zv);
| EXT_CALL zend_jit_find_func_helper, r0
} else if (opline->opcode == ZEND_INIT_FCALL_BY_NAME) {
| LOAD_ADDR FCARG1a, Z_STR_P(zv + 1);
| EXT_CALL zend_jit_find_func_helper, r0
} else if (opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME) {
| LOAD_ADDR FCARG1a, zv;
| EXT_CALL zend_jit_find_ns_func_helper, r0
} else {
ZEND_UNREACHABLE();
}
| // CACHE_PTR(opline->result.num, fbc);
| mov r1, EX->run_time_cache
| mov aword [r1 + opline->result.num], r0
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
if (!func || opline->opcode == ZEND_INIT_FCALL) {
| test r0, r0
| jnz >3
} else if (func->type == ZEND_USER_FUNCTION
&& !(func->common.fn_flags & ZEND_ACC_IMMUTABLE)) {
const zend_op *opcodes = func->op_array.opcodes;
| .if X64
|| if (!IS_SIGNED_32BIT(opcodes)) {
| mov64 r1, ((ptrdiff_t)opcodes)
| cmp aword [r0 + offsetof(zend_op_array, opcodes)], r1
|| } else {
| cmp aword [r0 + offsetof(zend_op_array, opcodes)], opcodes
|| }
| .else
| cmp aword [r0 + offsetof(zend_op_array, opcodes)], opcodes
| .endif
| jz >3
} else {
| .if X64
|| if (!IS_SIGNED_32BIT(func)) {
| mov64 r1, ((ptrdiff_t)func)
| cmp r0, r1
|| } else {
| cmp r0, func
|| }
| .else
| cmp r0, func
| .endif
| jz >3
}
| jmp &exit_addr
} else {
| test r0, r0
| jnz >3
| // SAVE_OPLINE();
| SET_EX_OPLINE opline, r0
| jmp ->undefined_function
}
}
|.code
|3:
}
if (!zend_jit_push_call_frame(Dst, opline, op_array, func, 0, 0, stack_check)) {
return 0;
}
if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) {
if (!zend_jit_save_call_chain(Dst, call_level)) {
return 0;
}
} else {
delayed_call_chain = 1;
delayed_call_level = call_level;
}
return 1;
}
static int zend_jit_init_method_call(dasm_State **Dst,
const zend_op *opline,
uint32_t b,
const zend_op_array *op_array,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
int call_level,
uint32_t op1_info,
zend_jit_addr op1_addr,
zend_class_entry *ce,
zend_bool ce_is_instanceof,
zend_bool use_this,
zend_class_entry *trace_ce,
zend_jit_trace_rec *trace,
zend_bool stack_check,
zend_bool polymorphic_side_trace)
{
zend_func_info *info = ZEND_FUNC_INFO(op_array);
zend_call_info *call_info = NULL;
zend_function *func = NULL;
zval *function_name;
ZEND_ASSERT(opline->op2_type == IS_CONST);
ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
function_name = RT_CONSTANT(opline, opline->op2);
if (info) {
call_info = info->callee_info;
while (call_info && call_info->caller_init_opline != opline) {
call_info = call_info->next_callee;
}
if (call_info && call_info->callee_func) {
func = call_info->callee_func;
}
}
if (polymorphic_side_trace) {
/* function is passed in r0 from parent_trace */
} else {
if (opline->op1_type == IS_UNUSED || use_this) {
zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
| GET_ZVAL_PTR FCARG1a, this_addr
} else {
if (op1_info & MAY_BE_REF) {
if (opline->op1_type == IS_CV) {
if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
| ZVAL_DEREF FCARG1a, op1_info
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
} else {
/* Hack: Convert reference to regular value to simplify JIT code */
ZEND_ASSERT(Z_REG(op1_addr) == ZREG_FP);
| IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >1
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
| EXT_CALL zend_jit_unref_helper, r0
|1:
}
}
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1
|.cold_code
|1:
if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
| SET_EX_OPLINE opline, r0
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !use_this) {
| EXT_CALL zend_jit_invalid_method_call_tmp, r0
} else {
| EXT_CALL zend_jit_invalid_method_call, r0
}
| jmp ->exception_handler
|.code
}
}
| GET_ZVAL_PTR FCARG1a, op1_addr
}
if (delayed_call_chain) {
if (!zend_jit_save_call_chain(Dst, delayed_call_level)) {
return 0;
}
}
| mov aword T1, FCARG1a // save
if (func) {
| // fbc = CACHED_PTR(opline->result.num + sizeof(void*));
| mov r0, EX->run_time_cache
| mov r0, aword [r0 + opline->result.num + sizeof(void*)]
| test r0, r0
| jz >1
} else {
| // if (CACHED_PTR(opline->result.num) == obj->ce)) {
| mov r0, EX->run_time_cache
| mov r2, aword [r0 + opline->result.num]
| cmp r2, [FCARG1a + offsetof(zend_object, ce)]
| jnz >1
| // fbc = CACHED_PTR(opline->result.num + sizeof(void*));
| mov r0, aword [r0 + opline->result.num + sizeof(void*)]
}
|.cold_code
|1:
| LOAD_ADDR FCARG2a, function_name
|.if X64
| lea CARG3, aword T1
|.else
| lea r0, aword T1
| sub r4, 12
| push r0
|.endif
| SET_EX_OPLINE opline, r0
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !use_this) {
| EXT_CALL zend_jit_find_method_tmp_helper, r0
} else {
| EXT_CALL zend_jit_find_method_helper, r0
}
|.if not(X64)
| add r4, 12
|.endif
| test r0, r0
| jnz >2
| jmp ->exception_handler
|.code
|2:
}
if (!func
&& trace
&& trace->op == ZEND_JIT_TRACE_INIT_CALL
&& trace->func
#ifdef _WIN32
&& trace->func->type != ZEND_INTERNAL_FUNCTION
#endif
) {
int32_t exit_point;
const void *exit_addr;
exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_METHOD_CALL);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
func = (zend_function*)trace->func;
if (func->type == ZEND_USER_FUNCTION &&
(!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) ||
(func->common.fn_flags & ZEND_ACC_CLOSURE) ||
!func->common.function_name)) {
const zend_op *opcodes = func->op_array.opcodes;
| .if X64
|| if (!IS_SIGNED_32BIT(opcodes)) {
| mov64 r1, ((ptrdiff_t)opcodes)
| cmp aword [r0 + offsetof(zend_op_array, opcodes)], r1
|| } else {
| cmp aword [r0 + offsetof(zend_op_array, opcodes)], opcodes
|| }
| .else
| cmp aword [r0 + offsetof(zend_op_array, opcodes)], opcodes
| .endif
| jne &exit_addr
} else {
| .if X64
|| if (!IS_SIGNED_32BIT(func)) {
| mov64 r1, ((ptrdiff_t)func)
| cmp r0, r1
|| } else {
| cmp r0, func
|| }
| .else
| cmp r0, func
| .endif
| jne &exit_addr
}
}
if (!func) {
| // if (fbc->common.fn_flags & ZEND_ACC_STATIC) {
| test dword [r0 + offsetof(zend_function, common.fn_flags)], ZEND_ACC_STATIC
| jnz >1
|.cold_code
|1:
}
if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) != 0) {
| mov FCARG1a, aword T1 // restore
| mov FCARG2a, r0
|.if X64
| mov CARG3d, opline->extended_value
|.else
| sub r4, 12
| push opline->extended_value
|.endif
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !use_this) {
| EXT_CALL zend_jit_push_static_metod_call_frame_tmp, r0
} else {
| EXT_CALL zend_jit_push_static_metod_call_frame, r0
}
|.if not(X64)
| add r4, 12
|.endif
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR) && !use_this)) {
| test r0, r0
| jz ->exception_handler
}
| mov RX, r0
}
if (!func) {
| jmp >9
|.code
}
if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) == 0) {
if (!zend_jit_push_call_frame(Dst, opline, NULL, func, 0, use_this, stack_check)) {
return 0;
}
}
if (!func) {
|9:
}
zend_jit_start_reuse_ip();
if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) {
if (!zend_jit_save_call_chain(Dst, call_level)) {
return 0;
}
} else {
delayed_call_chain = 1;
delayed_call_level = call_level;
}
return 1;
}
static int zend_jit_init_closure_call(dasm_State **Dst,
const zend_op *opline,
uint32_t b,
const zend_op_array *op_array,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
int call_level,
zend_jit_trace_rec *trace,
zend_bool stack_check)
{
zend_function *func = NULL;
zend_jit_addr op2_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
| GET_ZVAL_PTR r0, op2_addr
if (ssa->var_info[ssa_op->op2_use].ce != zend_ce_closure
&& !(ssa->var_info[ssa_op->op2_use].type & MAY_BE_CLASS_GUARD)) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
|.if X64
|| if (!IS_SIGNED_32BIT(zend_ce_closure)) {
| mov64 FCARG1a, ((ptrdiff_t)zend_ce_closure)
| cmp aword [r0 + offsetof(zend_object, ce)], FCARG1a
|| } else {
| cmp aword [r0 + offsetof(zend_object, ce)], zend_ce_closure
|| }
|.else
| cmp aword [r0 + offsetof(zend_object, ce)], zend_ce_closure
|.endif
| jne &exit_addr
if (ssa->var_info && ssa_op->op2_use >= 0) {
ssa->var_info[ssa_op->op2_use].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op2_use].ce = zend_ce_closure;
ssa->var_info[ssa_op->op2_use].is_instanceof = 0;
}
}
if (trace
&& trace->op == ZEND_JIT_TRACE_INIT_CALL
&& trace->func
&& trace->func->type == ZEND_USER_FUNCTION) {
const zend_op *opcodes;
int32_t exit_point;
const void *exit_addr;
func = (zend_function*)trace->func;
opcodes = func->op_array.opcodes;
exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_CLOSURE_CALL);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| .if X64
|| if (!IS_SIGNED_32BIT(opcodes)) {
| mov64 FCARG1a, ((ptrdiff_t)opcodes)
| cmp aword [r0 + offsetof(zend_closure, func.op_array.opcodes)], FCARG1a
|| } else {
| cmp aword [r0 + offsetof(zend_closure, func.op_array.opcodes)], opcodes
|| }
| .else
| cmp aword [r0 + offsetof(zend_closure, func.op_array.opcodes)], opcodes
| .endif
| jne &exit_addr
}
if (delayed_call_chain) {
if (!zend_jit_save_call_chain(Dst, delayed_call_level)) {
return 0;
}
}
if (!zend_jit_push_call_frame(Dst, opline, NULL, func, 1, 0, stack_check)) {
return 0;
}
if (zend_jit_needs_call_chain(NULL, b, op_array, ssa, ssa_op, opline, call_level, trace)) {
if (!zend_jit_save_call_chain(Dst, call_level)) {
return 0;
}
} else {
delayed_call_chain = 1;
delayed_call_level = call_level;
}
if (trace
&& trace->op == ZEND_JIT_TRACE_END
&& trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
if (!zend_jit_set_valid_ip(Dst, opline + 1)) {
return 0;
}
}
return 1;
}
static uint32_t skip_valid_arguments(const zend_op_array *op_array, zend_ssa *ssa, const zend_call_info *call_info)
{
uint32_t num_args = 0;
zend_function *func = call_info->callee_func;
while (num_args < call_info->num_args) {
zend_arg_info *arg_info = func->op_array.arg_info + num_args;
if (ZEND_TYPE_IS_SET(arg_info->type)) {
if (ZEND_TYPE_IS_ONLY_MASK(arg_info->type)) {
zend_op *opline = call_info->arg_info[num_args].opline;
zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type);
if ((OP1_INFO() & (MAY_BE_ANY|MAY_BE_UNDEF)) & ~type_mask) {
break;
}
} else {
break;
}
}
num_args++;
}
return num_args;
}
static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, int call_level, unsigned int next_block, zend_jit_trace_rec *trace)
{
zend_func_info *info = ZEND_FUNC_INFO(op_array);
zend_call_info *call_info = NULL;
const zend_function *func = NULL;
uint32_t i;
zend_jit_addr res_addr;
uint32_t call_num_args = 0;
zend_bool unknown_num_args = 0;
const void *exit_addr = NULL;
const zend_op *prev_opline;
if (RETURN_VALUE_USED(opline)) {
res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
} else {
/* CPU stack allocated temporary zval */
res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R4, TMP_ZVAL_OFFSET);
}
prev_opline = opline - 1;
while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) {
prev_opline--;
}
if (prev_opline->opcode == ZEND_SEND_UNPACK || prev_opline->opcode == ZEND_SEND_ARRAY ||
prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) {
unknown_num_args = 1;
}
if (info) {
call_info = info->callee_info;
while (call_info && call_info->caller_call_opline != opline) {
call_info = call_info->next_callee;
}
if (call_info && call_info->callee_func) {
func = call_info->callee_func;
}
if ((op_array->fn_flags & ZEND_ACC_TRAIT_CLONE)
&& JIT_G(current_frame)
&& JIT_G(current_frame)->call
&& !JIT_G(current_frame)->call->func) {
call_info = NULL; func = NULL; /* megamorphic call from trait */
}
}
if (!func) {
/* resolve function at run time */
} else if (func->type == ZEND_USER_FUNCTION) {
ZEND_ASSERT(opline->opcode != ZEND_DO_ICALL);
call_num_args = call_info->num_args;
} else if (func->type == ZEND_INTERNAL_FUNCTION) {
ZEND_ASSERT(opline->opcode != ZEND_DO_UCALL);
call_num_args = call_info->num_args;
} else {
ZEND_UNREACHABLE();
}
if (trace && !func) {
if (trace->op == ZEND_JIT_TRACE_DO_ICALL) {
ZEND_ASSERT(trace->func->type == ZEND_INTERNAL_FUNCTION);
#ifndef ZEND_WIN32
// TODO: ASLR may cause different addresses in different workers ???
func = trace->func;
if (JIT_G(current_frame) &&
JIT_G(current_frame)->call &&
TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) {
call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call);
} else {
unknown_num_args = 1;
}
#endif
} else if (trace->op == ZEND_JIT_TRACE_ENTER) {
ZEND_ASSERT(trace->func->type == ZEND_USER_FUNCTION);
if (zend_accel_in_shm(trace->func->op_array.opcodes)) {
func = trace->func;
if (JIT_G(current_frame) &&
JIT_G(current_frame)->call &&
TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) {
call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call);
} else {
unknown_num_args = 1;
}
}
}
}
bool may_have_extra_named_params =
opline->extended_value == ZEND_FCALL_MAY_HAVE_EXTRA_NAMED_PARAMS &&
(!func || func->common.fn_flags & ZEND_ACC_VARIADIC);
if (!reuse_ip) {
zend_jit_start_reuse_ip();
| // call = EX(call);
| mov RX, EX->call
}
zend_jit_stop_reuse_ip();
| // fbc = call->func;
| // mov r2, EX:RX->func ???
| // SAVE_OPLINE();
| SET_EX_OPLINE opline, r0
if (opline->opcode == ZEND_DO_FCALL) {
if (!func) {
if (trace) {
uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| mov r0, EX:RX->func
| test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED
| jnz &exit_addr
}
}
}
if (!delayed_call_chain) {
if (call_level == 1) {
| mov aword EX->call, 0
} else {
| //EX(call) = call->prev_execute_data;
| mov r0, EX:RX->prev_execute_data
| mov EX->call, r0
}
}
delayed_call_chain = 0;
| //call->prev_execute_data = execute_data;
| mov EX:RX->prev_execute_data, EX
if (!func) {
| mov r0, EX:RX->func
}
if (opline->opcode == ZEND_DO_FCALL) {
if (!func) {
if (!trace) {
| test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED
| jnz >1
|.cold_code
|1:
if (!GCC_GLOBAL_REGS) {
| mov FCARG1a, RX
}
| EXT_CALL zend_jit_deprecated_helper, r0
| test al, al
| mov r0, EX:RX->func // reload
| jne >1
| jmp ->exception_handler
|.code
|1:
}
} else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) {
if (!GCC_GLOBAL_REGS) {
| mov FCARG1a, RX
}
| EXT_CALL zend_jit_deprecated_helper, r0
| test al, al
| je ->exception_handler
}
}
if (!func
&& opline->opcode != ZEND_DO_UCALL
&& opline->opcode != ZEND_DO_ICALL) {
| cmp byte [r0 + offsetof(zend_function, type)], ZEND_USER_FUNCTION
| jne >8
}
if ((!func || func->type == ZEND_USER_FUNCTION)
&& opline->opcode != ZEND_DO_ICALL) {
| // EX(call) = NULL;
| mov aword EX:RX->call, 0
if (RETURN_VALUE_USED(opline)) {
| // EX(return_value) = EX_VAR(opline->result.var);
| LOAD_ZVAL_ADDR r2, res_addr
| mov aword EX:RX->return_value, r2
} else {
| // EX(return_value) = 0;
| mov aword EX:RX->return_value, 0
}
//EX_LOAD_RUN_TIME_CACHE(op_array);
if (!func || func->op_array.cache_size) {
if (func && op_array == &func->op_array) {
/* recursive call */
if (trace || func->op_array.cache_size > sizeof(void*)) {
| mov r2, EX->run_time_cache
| mov EX:RX->run_time_cache, r2
}
} else {
if (func) {
| mov r0, EX:RX->func
}
| mov r2, aword [r0 + offsetof(zend_op_array, run_time_cache__ptr)]
#if ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR
| mov r2, aword [r2]
#elif ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR_OR_OFFSET
if (func && !(func->op_array.fn_flags & ZEND_ACC_CLOSURE)) {
if (ZEND_MAP_PTR_IS_OFFSET(func->op_array.run_time_cache)) {
| MEM_OP2_2_ZTS add, r2, aword, compiler_globals, map_ptr_base, r1
} else if (!zend_accel_in_shm(func->op_array.opcodes)) {
/* the called op_array may be not persisted yet */
| test r2, 1
| jz >1
| MEM_OP2_2_ZTS add, r2, aword, compiler_globals, map_ptr_base, r1
|1:
}
| mov r2, aword [r2]
} else {
| test r2, 1
| jz >1
| MEM_OP2_2_ZTS add, r2, aword, compiler_globals, map_ptr_base, r1
|1:
| mov r2, aword [r2]
}
#else
# error "Unknown ZEND_MAP_PTR_KIND"
#endif
| mov EX:RX->run_time_cache, r2
}
}
| // EG(current_execute_data) = execute_data;
| MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, RX, r1
| mov FP, RX
| // opline = op_array->opcodes;
if (func && !unknown_num_args) {
for (i = call_num_args; i < func->op_array.last_var; i++) {
uint32_t n = EX_NUM_TO_VAR(i);
| SET_Z_TYPE_INFO RX + n, IS_UNDEF
}
if (call_num_args <= func->op_array.num_args) {
if (!trace || (trace->op == ZEND_JIT_TRACE_END
&& trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) {
uint32_t num_args;
if ((func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0) {
if (trace) {
num_args = 0;
} else if (call_info) {
num_args = skip_valid_arguments(op_array, ssa, call_info);
} else {
num_args = call_num_args;
}
} else {
num_args = call_num_args;
}
if (zend_accel_in_shm(func->op_array.opcodes)) {
| LOAD_IP_ADDR (func->op_array.opcodes + num_args)
} else {
| mov r0, EX->func
if (GCC_GLOBAL_REGS) {
| mov IP, aword [r0 + offsetof(zend_op_array, opcodes)]
if (num_args) {
| add IP, (num_args * sizeof(zend_op))
}
} else {
| mov FCARG1a, aword [r0 + offsetof(zend_op_array, opcodes)]
if (num_args) {
| add FCARG1a, (num_args * sizeof(zend_op))
}
| mov aword EX->opline, FCARG1a
}
}
if (GCC_GLOBAL_REGS && !trace && op_array == &func->op_array
&& num_args >= op_array->required_num_args) {
/* recursive call */
if (ZEND_OBSERVER_ENABLED) {
| SAVE_IP
| mov FCARG1a, FP
| EXT_CALL zend_observer_fcall_begin, r0
}
#ifdef CONTEXT_THREADED_JIT
| call >1
|.cold_code
|1:
| pop r0
| jmp =>num_args
|.code
#else
| jmp =>num_args
#endif
return 1;
}
}
} else {
if (!trace || (trace->op == ZEND_JIT_TRACE_END
&& trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) {
if (func && zend_accel_in_shm(func->op_array.opcodes)) {
| LOAD_IP_ADDR (func->op_array.opcodes)
} else if (GCC_GLOBAL_REGS) {
| mov IP, aword [r0 + offsetof(zend_op_array, opcodes)]
} else {
| mov FCARG1a, aword [r0 + offsetof(zend_op_array, opcodes)]
| mov aword EX->opline, FCARG1a
}
}
if (!GCC_GLOBAL_REGS) {
| mov FCARG1a, FP
}
| EXT_CALL zend_jit_copy_extra_args_helper, r0
}
} else {
| // opline = op_array->opcodes
if (func && zend_accel_in_shm(func->op_array.opcodes)) {
| LOAD_IP_ADDR (func->op_array.opcodes)
} else if (GCC_GLOBAL_REGS) {
| mov IP, aword [r0 + offsetof(zend_op_array, opcodes)]
} else {
| mov FCARG1a, aword [r0 + offsetof(zend_op_array, opcodes)]
| mov aword EX->opline, FCARG1a
}
if (func) {
| // num_args = EX_NUM_ARGS();
| mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)]
| // if (UNEXPECTED(num_args > first_extra_arg))
| cmp ecx, (func->op_array.num_args)
} else {
| // first_extra_arg = op_array->num_args;
| mov edx, dword [r0 + offsetof(zend_op_array, num_args)]
| // num_args = EX_NUM_ARGS();
| mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)]
| // if (UNEXPECTED(num_args > first_extra_arg))
| cmp ecx, edx
}
| jg >1
|.cold_code
|1:
if (!GCC_GLOBAL_REGS) {
| mov FCARG1a, FP
}
| EXT_CALL zend_jit_copy_extra_args_helper, r0
if (!func) {
| mov r0, EX->func // reload
}
| mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)] // reload
| jmp >1
|.code
if (!func || (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0) {
if (!func) {
| // if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0))
| test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_HAS_TYPE_HINTS
| jnz >1
}
| // opline += num_args;
|.if X64
|| ZEND_ASSERT(sizeof(zend_op) == 32);
| mov edx, ecx
| shl r2, 5
|.else
| imul r2, ecx, sizeof(zend_op)
|.endif
| ADD_IP r2
}
|1:
| // if (EXPECTED((int)num_args < op_array->last_var)) {
if (func) {
| mov edx, (func->op_array.last_var)
} else {
| mov edx, dword [r0 + offsetof(zend_op_array, last_var)]
}
| sub edx, ecx
| jle >3 //???
| // zval *var = EX_VAR_NUM(num_args);
// |.if X64
// | movsxd r1, ecx
// |.endif
| shl r1, 4
| lea r1, [FP + r1 + (ZEND_CALL_FRAME_SLOT * sizeof(zval))]
|2:
| SET_Z_TYPE_INFO r1, IS_UNDEF
| sub edx, 1
| lea r1, [r1 + 16]
| jne <2
|3:
}
if (ZEND_OBSERVER_ENABLED) {
| SAVE_IP
| mov FCARG1a, FP
| EXT_CALL zend_observer_fcall_begin, r0
}
if (trace) {
if (!func && (opline->opcode != ZEND_DO_UCALL)) {
| jmp >9
}
} else {
#ifdef CONTEXT_THREADED_JIT
| call ->context_threaded_call
if (!func && (opline->opcode != ZEND_DO_UCALL)) {
| jmp >9
}
| call ->context_threaded_call
if (!func) {
| jmp >9
}
#else
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| ADD_HYBRID_SPAD
| JMP_IP
} else if (GCC_GLOBAL_REGS) {
| add r4, SPAD // stack alignment
| JMP_IP
} else {
| mov FP, aword T2 // restore FP
| mov RX, aword T3 // restore IP
| add r4, NR_SPAD // stack alignment
| mov r0, 1 // ZEND_VM_ENTER
| ret
}
}
#endif
}
if ((!func || func->type == ZEND_INTERNAL_FUNCTION)
&& (opline->opcode != ZEND_DO_UCALL)) {
if (!func && (opline->opcode != ZEND_DO_ICALL)) {
|8:
}
if (opline->opcode == ZEND_DO_FCALL_BY_NAME) {
if (!func) {
if (trace) {
uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED
| jnz &exit_addr
} else {
| test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED
| jnz >1
|.cold_code
|1:
if (!GCC_GLOBAL_REGS) {
| mov FCARG1a, RX
}
| EXT_CALL zend_jit_deprecated_helper, r0
| test al, al
| mov r0, EX:RX->func // reload
| jne >1
| jmp ->exception_handler
|.code
|1:
}
} else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) {
if (!GCC_GLOBAL_REGS) {
| mov FCARG1a, RX
}
| EXT_CALL zend_jit_deprecated_helper, r0
| test al, al
| je ->exception_handler
| mov r0, EX:RX->func // reload
}
}
| // ZVAL_NULL(EX_VAR(opline->result.var));
| LOAD_ZVAL_ADDR FCARG2a, res_addr
| SET_Z_TYPE_INFO FCARG2a, IS_NULL
| // EG(current_execute_data) = execute_data;
| MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, RX, r1
zend_jit_reset_last_valid_opline();
| // fbc->internal_function.handler(call, ret);
| mov FCARG1a, RX
if (func) {
| EXT_CALL func->internal_function.handler, r0
} else {
| call aword [r0 + offsetof(zend_internal_function, handler)]
}
| // EG(current_execute_data) = execute_data;
| MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, FP, r0
| // zend_vm_stack_free_args(call);
if (func && !unknown_num_args) {
for (i = 0; i < call_num_args; i++ ) {
uint32_t offset = EX_NUM_TO_VAR(i);
| ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_RX, offset), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 0, 1, opline
}
} else {
| mov FCARG1a, RX
| EXT_CALL zend_jit_vm_stack_free_args_helper, r0
}
if (may_have_extra_named_params) {
| test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 3], (ZEND_CALL_HAS_EXTRA_NAMED_PARAMS >> 24)
| jnz >1
|.cold_code
|1:
| mov FCARG1a, aword [RX + offsetof(zend_execute_data, extra_named_params)]
| EXT_CALL zend_free_extra_named_params, r0
| jmp >2
|.code
|2:
}
|8:
if (opline->opcode == ZEND_DO_FCALL) {
// TODO: optimize ???
| // if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_RELEASE_THIS))
| test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 2], (ZEND_CALL_RELEASE_THIS >> 16)
| jnz >1
|.cold_code
|1:
| GET_Z_PTR FCARG1a, RX + offsetof(zend_execute_data, This)
| // OBJ_RELEASE(object);
| OBJ_RELEASE ZREG_FCARG1a, >2
| jmp >2
|.code
|2:
}
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
!JIT_G(current_frame) ||
!JIT_G(current_frame)->call ||
!TRACE_FRAME_IS_NESTED(JIT_G(current_frame)->call) ||
prev_opline->opcode == ZEND_SEND_UNPACK ||
prev_opline->opcode == ZEND_SEND_ARRAY ||
prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) {
| // zend_vm_stack_free_call_frame(call);
| test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 2], (ZEND_CALL_ALLOCATED >> 16)
| jnz >1
|.cold_code
|1:
| mov FCARG1a, RX
| EXT_CALL zend_jit_free_call_frame, r0
| jmp >1
|.code
}
| MEM_OP2_1_ZTS mov, aword, executor_globals, vm_stack_top, RX, r0
|1:
if (!RETURN_VALUE_USED(opline)) {
zend_class_entry *ce;
zend_bool ce_is_instanceof;
uint32_t func_info = call_info ?
zend_get_func_info(call_info, ssa, &ce, &ce_is_instanceof) :
(MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN);
/* If an exception is thrown, the return_value may stay at the
* original value of null. */
func_info |= MAY_BE_NULL;
if (func_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
| ZVAL_PTR_DTOR res_addr, func_info, 1, 1, opline
}
}
| // if (UNEXPECTED(EG(exception) != NULL)) {
| MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0
| jne ->icall_throw_handler
// TODO: Can we avoid checking for interrupts after each call ???
if (trace && last_valid_opline != opline) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline + 1, ZEND_JIT_EXIT_TO_VM);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
} else {
exit_addr = NULL;
}
if (!zend_jit_check_timeout(Dst, opline + 1, exit_addr)) {
return 0;
}
if ((!trace || !func) && opline->opcode != ZEND_DO_ICALL) {
| LOAD_IP_ADDR (opline + 1)
} else if (trace
&& trace->op == ZEND_JIT_TRACE_END
&& trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
| LOAD_IP_ADDR (opline + 1)
}
}
if (!func) {
|9:
}
return 1;
}
static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr)
{
uint32_t arg_num = opline->op2.num;
zend_jit_addr arg_addr;
ZEND_ASSERT(opline->opcode == ZEND_SEND_VAL || arg_num <= MAX_ARG_FLAG_NUM);
if (!zend_jit_reuse_ip(Dst)) {
return 0;
}
if (opline->opcode == ZEND_SEND_VAL_EX) {
uint32_t mask = ZEND_SEND_BY_REF << ((arg_num + 3) * 2);
ZEND_ASSERT(arg_num <= MAX_ARG_FLAG_NUM);
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& JIT_G(current_frame)
&& JIT_G(current_frame)->call
&& JIT_G(current_frame)->call->func) {
if (ARG_MUST_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
/* Don't generate code that always throws exception */
return 0;
}
} else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| mov r0, EX:RX->func
| test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask
| jnz &exit_addr
} else {
| mov r0, EX:RX->func
| test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask
| jnz >1
|.cold_code
|1:
| SET_EX_OPLINE opline, r0
| jmp ->throw_cannot_pass_by_ref
|.code
}
}
arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var);
if (opline->op1_type == IS_CONST) {
zval *zv = RT_CONSTANT(opline, opline->op1);
| ZVAL_COPY_CONST arg_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_R0
if (Z_REFCOUNTED_P(zv)) {
| ADDREF_CONST zv, r0
}
} else {
| ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2
}
return 1;
}
static int zend_jit_check_undef_args(dasm_State **Dst, const zend_op *opline)
{
| mov FCARG1a, EX->call
| test byte [FCARG1a + offsetof(zend_execute_data, This.u1.type_info) + 3], (ZEND_CALL_MAY_HAVE_UNDEF >> 24)
| jnz >1
|.cold_code
|1:
| SET_EX_OPLINE opline, r0
| EXT_CALL zend_handle_undef_args, r0
| test r0, r0
| jnz ->exception_handler
| jmp >2
|.code
|2:
return 1;
}
static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, int cold)
{
zend_jit_addr op1_addr, arg_addr, ref_addr;
op1_addr = OP1_ADDR();
arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var);
if (!zend_jit_reuse_ip(Dst)) {
return 0;
}
if (opline->op1_type == IS_VAR) {
if (op1_info & MAY_BE_INDIRECT) {
| LOAD_ZVAL_ADDR r0, op1_addr
| // if (EXPECTED(Z_TYPE_P(ret) == IS_INDIRECT)) {
| IF_NOT_Z_TYPE r0, IS_INDIRECT, >1
| // ret = Z_INDIRECT_P(ret);
| GET_Z_PTR r0, r0
|1:
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0);
}
} else if (opline->op1_type == IS_CV) {
if (op1_info & MAY_BE_UNDEF) {
if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1
| SET_ZVAL_TYPE_INFO op1_addr, IS_NULL
| jmp >2
|1:
}
op1_info &= ~MAY_BE_UNDEF;
op1_info |= MAY_BE_NULL;
}
} else {
ZEND_UNREACHABLE();
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) {
if (op1_info & MAY_BE_REF) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >2
| GET_ZVAL_PTR r1, op1_addr
| GC_ADDREF r1
| SET_ZVAL_PTR arg_addr, r1
| SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX
| jmp >6
}
|2:
| // ZVAL_NEW_REF(arg, varptr);
if (opline->op1_type == IS_VAR) {
if (Z_REG(op1_addr) != ZREG_R0 || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR r0, op1_addr
}
| mov aword T1, r0 // save
}
| EMALLOC sizeof(zend_reference), op_array, opline
| mov dword [r0], 2
| mov dword [r0 + offsetof(zend_reference, gc.u.type_info)], GC_REFERENCE
| mov aword [r0 + offsetof(zend_reference, sources.ptr)], 0
ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, offsetof(zend_reference, val));
if (opline->op1_type == IS_VAR) {
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R1, 0);
| mov r1, aword T1 // restore
| ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_R2, ZREG_R2
| SET_ZVAL_PTR val_addr, r0
| SET_ZVAL_TYPE_INFO val_addr, IS_REFERENCE_EX
} else {
| ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2
| SET_ZVAL_PTR op1_addr, r0
| SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX
}
| SET_ZVAL_PTR arg_addr, r0
| SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX
}
|6:
| FREE_OP opline->op1_type, opline->op1, op1_info, !cold, opline
|7:
return 1;
}
static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr)
{
uint32_t arg_num = opline->op2.num;
zend_jit_addr arg_addr;
ZEND_ASSERT((opline->opcode != ZEND_SEND_VAR_EX &&
opline->opcode != ZEND_SEND_VAR_NO_REF_EX) ||
arg_num <= MAX_ARG_FLAG_NUM);
arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var);
if (!zend_jit_reuse_ip(Dst)) {
return 0;
}
if (opline->opcode == ZEND_SEND_VAR_EX) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& JIT_G(current_frame)
&& JIT_G(current_frame)->call
&& JIT_G(current_frame)->call->func) {
if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 0)) {
return 0;
}
return 1;
}
} else {
uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2);
| mov r0, EX:RX->func
| test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask
| jnz >1
|.cold_code
|1:
if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) {
return 0;
}
| jmp >7
|.code
}
} else if (opline->opcode == ZEND_SEND_VAR_NO_REF_EX) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& JIT_G(current_frame)
&& JIT_G(current_frame)->call
&& JIT_G(current_frame)->call->func) {
if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
| ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2
if (!ARG_MAY_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
if (!(op1_info & MAY_BE_REF)) {
/* Don't generate code that always throws exception */
return 0;
} else {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| cmp cl, IS_REFERENCE
| jne &exit_addr
}
}
return 1;
}
} else {
uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2);
| mov r0, EX:RX->func
| test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask
| jnz >1
|.cold_code
|1:
mask = ZEND_SEND_PREFER_REF << ((arg_num + 3) * 2);
| ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2
if (op1_info & MAY_BE_REF) {
| cmp cl, IS_REFERENCE
| je >7
}
| test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask
| jnz >7
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| jmp &exit_addr
} else {
| SET_EX_OPLINE opline, r0
| LOAD_ZVAL_ADDR FCARG1a, arg_addr
| EXT_CALL zend_jit_only_vars_by_reference, r0
if (!zend_jit_check_exception(Dst)) {
return 0;
}
| jmp >7
}
|.code
}
} else if (opline->opcode == ZEND_SEND_FUNC_ARG) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& JIT_G(current_frame)
&& JIT_G(current_frame)->call
&& JIT_G(current_frame)->call->func) {
if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 0)) {
return 0;
}
return 1;
}
} else {
| test dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ZEND_CALL_SEND_ARG_BY_REF
| jnz >1
|.cold_code
|1:
if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) {
return 0;
}
| jmp >7
|.code
}
}
if (op1_info & MAY_BE_UNDEF) {
if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
| IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1
|.cold_code
|1:
}
| SET_EX_OPLINE opline, r0
| mov FCARG1d, opline->op1.var
| EXT_CALL zend_jit_undefined_op_helper, r0
| SET_ZVAL_TYPE_INFO arg_addr, IS_NULL
| test r0, r0
| jz ->exception_handler
if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
| jmp >7
|.code
} else {
|7:
return 1;
}
}
if (opline->opcode == ZEND_SEND_VAR_NO_REF) {
| ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2
if (op1_info & MAY_BE_REF) {
| cmp cl, IS_REFERENCE
| je >7
}
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| jmp &exit_addr
} else {
| SET_EX_OPLINE opline, r0
| LOAD_ZVAL_ADDR FCARG1a, arg_addr
| EXT_CALL zend_jit_only_vars_by_reference, r0
if (!zend_jit_check_exception(Dst)) {
return 0;
}
}
} else {
if (op1_info & MAY_BE_REF) {
if (opline->op1_type == IS_CV) {
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
| ZVAL_DEREF FCARG1a, op1_info
| ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_R0, ZREG_R2
| TRY_ADDREF op1_info, ah, r2
} else {
zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 8);
| IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1
|.cold_code
|1:
| // zend_refcounted *ref = Z_COUNTED_P(retval_ptr);
| GET_ZVAL_PTR FCARG1a, op1_addr
| // ZVAL_COPY_VALUE(return_value, &ref->value);
| ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_R0, ZREG_R2
| GC_DELREF FCARG1a
| je >1
| IF_NOT_REFCOUNTED ah, >2
| GC_ADDREF r2
| jmp >2
|1:
| EFREE_REG_REFERENCE
| jmp >2
|.code
| ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2
|2:
}
} else {
if (op1_addr != op1_def_addr) {
if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, op1_info)) {
return 0;
}
if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) {
op1_addr= op1_def_addr;
}
}
| ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2
if (opline->op1_type == IS_CV) {
| TRY_ADDREF op1_info, ah, r2
}
}
}
|7:
return 1;
}
static int zend_jit_check_func_arg(dasm_State **Dst, const zend_op *opline)
{
uint32_t arg_num = opline->op2.num;
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& JIT_G(current_frame)
&& JIT_G(current_frame)->call
&& JIT_G(current_frame)->call->func) {
if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
if (!TRACE_FRAME_IS_LAST_SEND_BY_REF(JIT_G(current_frame)->call)) {
TRACE_FRAME_SET_LAST_SEND_BY_REF(JIT_G(current_frame)->call);
| // ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF);
|| if (reuse_ip) {
| or dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ZEND_CALL_SEND_ARG_BY_REF
|| } else {
| mov r0, EX->call
| or dword [r0 + offsetof(zend_execute_data, This.u1.type_info)], ZEND_CALL_SEND_ARG_BY_REF
|| }
}
} else {
if (!TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
TRACE_FRAME_SET_LAST_SEND_BY_VAL(JIT_G(current_frame)->call);
| // ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF);
|| if (reuse_ip) {
| and dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ~ZEND_CALL_SEND_ARG_BY_REF
|| } else {
| mov r0, EX->call
| and dword [r0 + offsetof(zend_execute_data, This.u1.type_info)], ~ZEND_CALL_SEND_ARG_BY_REF
|| }
}
}
} else {
// if (QUICK_ARG_SHOULD_BE_SENT_BY_REF(EX(call)->func, arg_num)) {
uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2);
if (!zend_jit_reuse_ip(Dst)) {
return 0;
}
| mov r0, EX:RX->func
| test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask
| jnz >1
|.cold_code
|1:
| // ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF);
| or dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ZEND_CALL_SEND_ARG_BY_REF
| jmp >1
|.code
| // ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF);
| and dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ~ZEND_CALL_SEND_ARG_BY_REF
|1:
}
return 1;
}
static int zend_jit_smart_true(dasm_State **Dst, const zend_op *opline, int jmp, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2)
{
if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ) {
if (jmp) {
| jmp >7
}
} else if (smart_branch_opcode == ZEND_JMPNZ) {
| jmp =>target_label
} else if (smart_branch_opcode == ZEND_JMPZNZ) {
| jmp =>target_label2
} else {
ZEND_UNREACHABLE();
}
} else {
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
if (jmp) {
| jmp >7
}
}
return 1;
}
static int zend_jit_smart_false(dasm_State **Dst, const zend_op *opline, int jmp, zend_uchar smart_branch_opcode, uint32_t target_label)
{
if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ) {
| jmp =>target_label
} else if (smart_branch_opcode == ZEND_JMPNZ) {
if (jmp) {
| jmp >7
}
} else if (smart_branch_opcode == ZEND_JMPZNZ) {
| jmp =>target_label
} else {
ZEND_UNREACHABLE();
}
} else {
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
if (jmp) {
| jmp >7
}
}
return 1;
}
static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
uint32_t defined_label = (uint32_t)-1;
uint32_t undefined_label = (uint32_t)-1;
zval *zv = RT_CONSTANT(opline, opline->op1);
zend_jit_addr res_addr = 0;
if (smart_branch_opcode && !exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
undefined_label = target_label;
} else if (smart_branch_opcode == ZEND_JMPNZ) {
defined_label = target_label;
} else if (smart_branch_opcode == ZEND_JMPZNZ) {
undefined_label = target_label;
defined_label = target_label2;
} else {
ZEND_UNREACHABLE();
}
}
| // if (CACHED_PTR(opline->extended_value)) {
| mov r0, EX->run_time_cache
| mov r0, aword [r0 + opline->extended_value]
| test r0, r0
| jz >1
| test r0, 0x1
| jnz >4
|.cold_code
|4:
| MEM_OP2_2_ZTS mov, FCARG1a, aword, executor_globals, zend_constants, FCARG1a
| shr r0, 1
| cmp dword [FCARG1a + offsetof(HashTable, nNumOfElements)], eax
if (smart_branch_opcode) {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
| jz &exit_addr
} else {
| jz >3
}
} else if (undefined_label != (uint32_t)-1) {
| jz =>undefined_label
} else {
| jz >3
}
} else {
| jz >2
}
|1:
| SET_EX_OPLINE opline, r0
| LOAD_ADDR FCARG1a, zv
| EXT_CALL zend_jit_check_constant, r0
| test r0, r0
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
| jz >3
} else {
| jnz >3
}
| jmp &exit_addr
} else if (smart_branch_opcode) {
if (undefined_label != (uint32_t)-1) {
| jz =>undefined_label
} else {
| jz >3
}
if (defined_label != (uint32_t)-1) {
| jmp =>defined_label
} else {
| jmp >3
}
} else {
res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
| jnz >1
|2:
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
| jmp >3
}
|.code
if (smart_branch_opcode) {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
| jmp &exit_addr
}
} else if (defined_label != (uint32_t)-1) {
| jmp =>defined_label
}
} else {
|1:
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
}
|3:
return 1;
}
static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
uint32_t mask;
zend_uchar type;
zend_jit_addr op1_addr = OP1_ADDR();
// TODO: support for is_resource() ???
ZEND_ASSERT(opline->extended_value != MAY_BE_RESOURCE);
if (op1_info & MAY_BE_UNDEF) {
if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
| IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1
|.cold_code
|1:
}
| SET_EX_OPLINE opline, r0
| mov FCARG1d, opline->op1.var
| EXT_CALL zend_jit_undefined_op_helper, r0
zend_jit_check_exception_undef_result(Dst, opline);
if (opline->extended_value & MAY_BE_NULL) {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
| jmp &exit_addr
} else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) {
| jmp >7
}
} else if (!zend_jit_smart_true(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label, target_label2)) {
return 0;
}
} else {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
| jmp &exit_addr
} else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) {
| jmp >7
}
} else if (!zend_jit_smart_false(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label)) {
return 0;
}
}
if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
|.code
}
}
if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
mask = opline->extended_value;
switch (mask) {
case MAY_BE_NULL: type = IS_NULL; break;
case MAY_BE_FALSE: type = IS_FALSE; break;
case MAY_BE_TRUE: type = IS_TRUE; break;
case MAY_BE_LONG: type = IS_LONG; break;
case MAY_BE_DOUBLE: type = IS_DOUBLE; break;
case MAY_BE_STRING: type = IS_STRING; break;
case MAY_BE_ARRAY: type = IS_ARRAY; break;
case MAY_BE_OBJECT: type = IS_OBJECT; break;
default:
type = 0;
}
if (!(op1_info & MAY_BE_GUARD) && !(op1_info & (MAY_BE_ANY - mask))) {
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
| jmp &exit_addr
}
} else if (!zend_jit_smart_true(Dst, opline, 0, smart_branch_opcode, target_label, target_label2)) {
return 0;
}
} else if (!(op1_info & MAY_BE_GUARD) && !(op1_info & mask)) {
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
| jmp &exit_addr
}
} else if (!zend_jit_smart_false(Dst, opline, 0, smart_branch_opcode, target_label)) {
return 0;
}
} else {
if (op1_info & MAY_BE_REF) {
| LOAD_ZVAL_ADDR r0, op1_addr
| ZVAL_DEREF r0, op1_info
}
if (type == 0) {
if (smart_branch_opcode &&
(opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| // if (Z_REFCOUNTED_P(cv)) {
| IF_ZVAL_REFCOUNTED op1_addr, >1
|.cold_code
|1:
}
| // if (!Z_DELREF_P(cv)) {
| GET_ZVAL_PTR FCARG1a, op1_addr
| GC_DELREF FCARG1a
if (RC_MAY_BE_1(op1_info)) {
if (RC_MAY_BE_N(op1_info)) {
| jnz >3
}
if (op1_info & MAY_BE_REF) {
| mov al, byte [r0 + 8]
} else {
| mov al, byte [FP + opline->op1.var + 8]
}
| mov byte T1, al // save
| // zval_dtor_func(r);
| ZVAL_DTOR_FUNC op1_info, opline
| mov cl, byte T1 // restore
|jmp >2
}
if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if (!RC_MAY_BE_1(op1_info)) {
| jmp >3
}
|.code
}
|3:
if (op1_info & MAY_BE_REF) {
| mov cl, byte [r0 + 8]
} else {
| mov cl, byte [FP + opline->op1.var + 8]
}
|2:
} else {
if (op1_info & MAY_BE_REF) {
| mov cl, byte [r0 + 8]
} else {
| mov cl, byte [FP + opline->op1.var + 8]
}
}
| mov eax, 1
| shl eax, cl
| test eax, mask
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
| jne &exit_addr
} else {
| je &exit_addr
}
} else if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ) {
| je =>target_label
} else if (smart_branch_opcode == ZEND_JMPNZ) {
| jne =>target_label
} else if (smart_branch_opcode == ZEND_JMPZNZ) {
| je =>target_label
| jmp =>target_label2
} else {
ZEND_UNREACHABLE();
}
} else {
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
| setne al
| movzx eax, al
| add eax, 2
| SET_ZVAL_TYPE_INFO res_addr, eax
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline
}
} else {
if (smart_branch_opcode &&
(opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| // if (Z_REFCOUNTED_P(cv)) {
| IF_ZVAL_REFCOUNTED op1_addr, >1
|.cold_code
|1:
}
| // if (!Z_DELREF_P(cv)) {
| GET_ZVAL_PTR FCARG1a, op1_addr
| GC_DELREF FCARG1a
if (RC_MAY_BE_1(op1_info)) {
if (RC_MAY_BE_N(op1_info)) {
| jnz >3
}
if (op1_info & MAY_BE_REF) {
| mov al, byte [r0 + 8]
} else {
| mov al, byte [FP + opline->op1.var + 8]
}
| mov byte T1, al // save
| // zval_dtor_func(r);
| ZVAL_DTOR_FUNC op1_info, opline
| mov cl, byte T1 // restore
|jmp >2
}
if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if (!RC_MAY_BE_1(op1_info)) {
| jmp >3
}
|.code
}
|3:
if (op1_info & MAY_BE_REF) {
| mov cl, byte [r0 + 8]
} else {
| mov cl, byte [FP + opline->op1.var + 8]
}
|2:
| cmp cl, type
} else {
if (op1_info & MAY_BE_REF) {
| cmp byte [r0 + 8], type
} else {
| cmp byte [FP + opline->op1.var + 8], type
}
}
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
| je &exit_addr
} else {
| jne &exit_addr
}
} else if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ) {
| jne =>target_label
} else if (smart_branch_opcode == ZEND_JMPNZ) {
| je =>target_label
} else if (smart_branch_opcode == ZEND_JMPZNZ) {
| jne =>target_label
| jmp =>target_label2
} else {
ZEND_UNREACHABLE();
}
} else {
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
| sete al
| movzx eax, al
| add eax, 2
| SET_ZVAL_TYPE_INFO res_addr, eax
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline
}
}
}
}
|7:
return 1;
}
static uint32_t zend_ssa_cv_info(const zend_op_array *op_array, zend_ssa *ssa, uint32_t var)
{
uint32_t j, info;
if (ssa->vars && ssa->var_info) {
info = ssa->var_info[var].type;
for (j = op_array->last_var; j < ssa->vars_count; j++) {
if (ssa->vars[j].var == var) {
info |= ssa->var_info[j].type;
}
}
} else {
info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_UNDEF |
MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
}
#ifdef ZEND_JIT_USE_RC_INFERENCE
/* Refcount may be increased by RETURN opcode */
if ((info & MAY_BE_RC1) && !(info & MAY_BE_RCN)) {
for (j = 0; j < ssa->cfg.blocks_count; j++) {
if ((ssa->cfg.blocks[j].flags & ZEND_BB_REACHABLE) &&
ssa->cfg.blocks[j].len > 0) {
const zend_op *opline = op_array->opcodes + ssa->cfg.blocks[j].start + ssa->cfg.blocks[j].len - 1;
if (opline->opcode == ZEND_RETURN) {
if (opline->op1_type == IS_CV && opline->op1.var == EX_NUM_TO_VAR(var)) {
info |= MAY_BE_RCN;
break;
}
}
}
}
}
#endif
return info;
}
static int zend_jit_leave_frame(dasm_State **Dst)
{
| // EG(current_execute_data) = EX(prev_execute_data);
| mov r0, EX->prev_execute_data
| MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, r0, r2
return 1;
}
static int zend_jit_free_cv(dasm_State **Dst, uint32_t info, uint32_t var)
{
if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
uint32_t offset = EX_NUM_TO_VAR(var);
| ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, offset), info, 1, 1, NULL
}
return 1;
}
static int zend_jit_free_op(dasm_State **Dst, const zend_op *opline, uint32_t info, uint32_t var_offset)
{
if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
| ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, var_offset), info, 0, 1, opline
}
return 1;
}
static int zend_jit_leave_func(dasm_State **Dst,
const zend_op_array *op_array,
const zend_op *opline,
uint32_t op1_info,
zend_bool left_frame,
zend_jit_trace_rec *trace,
zend_jit_trace_info *trace_info,
int indirect_var_access,
int may_throw)
{
zend_bool may_be_top_frame =
JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
!JIT_G(current_frame) ||
!TRACE_FRAME_IS_NESTED(JIT_G(current_frame));
zend_bool may_need_call_helper =
indirect_var_access || /* may have symbol table */
!op_array->function_name || /* may have symbol table */
may_be_top_frame ||
(op_array->fn_flags & ZEND_ACC_VARIADIC) || /* may have extra named args */
JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
!JIT_G(current_frame) ||
TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) == -1 || /* unknown number of args */
(uint32_t)TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) > op_array->num_args; /* extra args */
zend_bool may_need_release_this =
!(op_array->fn_flags & ZEND_ACC_CLOSURE) &&
op_array->scope &&
!(op_array->fn_flags & ZEND_ACC_STATIC) &&
(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
!JIT_G(current_frame) ||
!TRACE_FRAME_NO_NEED_RELEASE_THIS(JIT_G(current_frame)));
if (may_need_call_helper || may_need_release_this) {
| mov FCARG1d, dword [FP + offsetof(zend_execute_data, This.u1.type_info)]
}
if (may_need_call_helper) {
if (!left_frame) {
left_frame = 1;
if (!zend_jit_leave_frame(Dst)) {
return 0;
}
}
/* ZEND_CALL_FAKE_CLOSURE handled on slow path to eliminate check for ZEND_CALL_CLOSURE on fast path */
| test FCARG1d, (ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_HAS_EXTRA_NAMED_PARAMS|ZEND_CALL_FAKE_CLOSURE)
if (trace && trace->op != ZEND_JIT_TRACE_END) {
| jnz >1
|.cold_code
|1:
if (!GCC_GLOBAL_REGS) {
| mov FCARG2a, FP
}
| EXT_CALL zend_jit_leave_func_helper, r0
if (may_be_top_frame) {
// TODO: try to avoid this check ???
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
#if 0
/* this check should be handled by the following OPLINE guard */
| cmp IP, zend_jit_halt_op
| je ->trace_halt
#endif
} else if (GCC_GLOBAL_REGS) {
| test IP, IP
| je ->trace_halt
} else {
| test eax, eax
| jl ->trace_halt
}
}
if (!GCC_GLOBAL_REGS) {
| // execute_data = EG(current_execute_data)
| MEM_OP2_2_ZTS mov, FP, aword, executor_globals, current_execute_data, r0
}
| jmp >8
|.code
} else {
| jnz ->leave_function_handler
}
}
if (op_array->fn_flags & ZEND_ACC_CLOSURE) {
if (!left_frame) {
left_frame = 1;
if (!zend_jit_leave_frame(Dst)) {
return 0;
}
}
| // OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func)));
| mov FCARG1a, EX->func
| sub FCARG1a, sizeof(zend_object)
| OBJ_RELEASE ZREG_FCARG1a, >4
|4:
} else if (may_need_release_this) {
if (!left_frame) {
left_frame = 1;
if (!zend_jit_leave_frame(Dst)) {
return 0;
}
}
| // if (call_info & ZEND_CALL_RELEASE_THIS)
| test FCARG1d, ZEND_CALL_RELEASE_THIS
| je >4
| // zend_object *object = Z_OBJ(execute_data->This);
| mov FCARG1a, EX->This.value.obj
| // OBJ_RELEASE(object);
| OBJ_RELEASE ZREG_FCARG1a, >4
|4:
// TODO: avoid EG(excption) check for $this->foo() calls
may_throw = 1;
}
| // EG(vm_stack_top) = (zval*)execute_data;
| MEM_OP2_1_ZTS mov, aword, executor_globals, vm_stack_top, FP, r0
| // execute_data = EX(prev_execute_data);
| mov FP, EX->prev_execute_data
if (!left_frame) {
| // EG(current_execute_data) = execute_data;
| MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, FP, r0
}
|9:
if (trace) {
if (trace->op != ZEND_JIT_TRACE_END
&& (JIT_G(current_frame) && !TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) {
zend_jit_reset_last_valid_opline();
} else {
| LOAD_IP
| ADD_IP sizeof(zend_op)
}
|8:
if (trace->op == ZEND_JIT_TRACE_BACK
&& (!JIT_G(current_frame) || TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) {
const zend_op *next_opline = trace->opline;
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
&& (op1_info & MAY_BE_RC1)
&& (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) {
/* exception might be thrown during destruction of unused return value */
| // if (EG(exception))
| MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0
| jne ->leave_throw_handler
}
do {
trace++;
} while (trace->op == ZEND_JIT_TRACE_INIT_CALL);
ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END);
next_opline = trace->opline;
ZEND_ASSERT(next_opline != NULL);
if (trace->op == ZEND_JIT_TRACE_END
&& trace->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
trace_info->flags |= ZEND_JIT_TRACE_LOOP;
| CMP_IP next_opline
| je =>0 // LOOP
#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
| JMP_IP
#else
| jmp ->trace_escape
#endif
} else {
| CMP_IP next_opline
| jne ->trace_escape
}
zend_jit_set_last_valid_opline(trace->opline);
return 1;
} else if (may_throw ||
(((opline->op1_type & (IS_VAR|IS_TMP_VAR))
&& (op1_info & MAY_BE_RC1)
&& (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY)))
&& (!JIT_G(current_frame) || TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))))) {
| // if (EG(exception))
| MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0
| jne ->leave_throw_handler
}
return 1;
} else {
| // if (EG(exception))
| MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0
| LOAD_IP
| jne ->leave_throw_handler
| // opline = EX(opline) + 1
| ADD_IP sizeof(zend_op)
}
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| ADD_HYBRID_SPAD
#ifdef CONTEXT_THREADED_JIT
| push aword [IP]
| ret
#else
| JMP_IP
#endif
} else if (GCC_GLOBAL_REGS) {
| add r4, SPAD // stack alignment
#ifdef CONTEXT_THREADED_JIT
| push aword [IP]
| ret
#else
| JMP_IP
#endif
} else {
#ifdef CONTEXT_THREADED_JIT
ZEND_UNREACHABLE();
// TODO: context threading can't work without GLOBAL REGS because we have to change
// the value of execute_data in execute_ex()
| mov FCARG1a, FP
| mov r0, aword [FP]
| mov FP, aword T2 // restore FP
| mov RX, aword T3 // restore IP
| add r4, NR_SPAD // stack alignment
| push aword [r0]
| ret
#else
| mov FP, aword T2 // restore FP
| mov RX, aword T3 // restore IP
| add r4, NR_SPAD // stack alignment
| mov r0, 2 // ZEND_VM_LEAVE
| ret
#endif
}
return 1;
}
static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr)
{
zend_jit_addr ret_addr;
int8_t return_value_used;
ZEND_ASSERT(op_array->type != ZEND_EVAL_CODE && op_array->function_name);
ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF));
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame)) {
if (TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) {
return_value_used = 1;
} else if (TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))) {
return_value_used = 0;
} else {
return_value_used = -1;
}
} else {
return_value_used = -1;
}
if (ZEND_OBSERVER_ENABLED) {
if (Z_MODE(op1_addr) == IS_REG) {
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
if (!zend_jit_spill_store(Dst, op1_addr, dst, op1_info, 1)) {
return 0;
}
op1_addr = dst;
}
| LOAD_ZVAL_ADDR FCARG2a, op1_addr
| mov FCARG1a, FP
| SET_EX_OPLINE opline, r0
| EXT_CALL zend_observer_fcall_end, r0
}
// if (!EX(return_value))
if (Z_MODE(op1_addr) == IS_REG && Z_REG(op1_addr) == ZREG_R1) {
if (return_value_used != 0) {
| mov r2, EX->return_value
}
if (return_value_used == -1) {
| test r2, r2
}
ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 0);
} else {
if (return_value_used != 0) {
| mov r1, EX->return_value
}
if (return_value_used == -1) {
| test r1, r1
}
ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R1, 0);
}
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if (return_value_used == -1) {
| jz >1
|.cold_code
|1:
}
if (return_value_used != 1) {
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if (jit_return_label >= 0) {
| IF_NOT_ZVAL_REFCOUNTED op1_addr, =>jit_return_label
} else {
| IF_NOT_ZVAL_REFCOUNTED op1_addr, >9
}
}
| GET_ZVAL_PTR FCARG1a, op1_addr
| GC_DELREF FCARG1a
if (RC_MAY_BE_1(op1_info)) {
if (RC_MAY_BE_N(op1_info)) {
if (jit_return_label >= 0) {
| jnz =>jit_return_label
} else {
| jnz >9
}
}
| //SAVE_OPLINE()
| ZVAL_DTOR_FUNC op1_info, opline
| //????mov r1, EX->return_value // reload ???
}
if (return_value_used == -1) {
if (jit_return_label >= 0) {
| jmp =>jit_return_label
} else {
| jmp >9
}
|.code
}
}
} else if (return_value_used == -1) {
if (jit_return_label >= 0) {
| jz =>jit_return_label
} else {
| jz >9
}
}
if (return_value_used == 0) {
|9:
return 1;
}
if (opline->op1_type == IS_CONST) {
zval *zv = RT_CONSTANT(opline, opline->op1);
| ZVAL_COPY_CONST ret_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_R0
if (Z_REFCOUNTED_P(zv)) {
| ADDREF_CONST zv, r0
}
} else if (opline->op1_type == IS_TMP_VAR) {
| ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2
} else if (opline->op1_type == IS_CV) {
if (op1_info & MAY_BE_REF) {
| LOAD_ZVAL_ADDR r0, op1_addr
| ZVAL_DEREF r0, op1_info
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0);
}
| ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2
if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
(op1_info & (MAY_BE_REF|MAY_BE_OBJECT)) ||
!op_array->function_name) {
| TRY_ADDREF op1_info, ah, r2
} else if (return_value_used != 1) {
| // if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr);
| SET_ZVAL_TYPE_INFO op1_addr, IS_NULL
}
}
} else {
if (op1_info & MAY_BE_REF) {
zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, offsetof(zend_reference, val));
| IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1
|.cold_code
|1:
| // zend_refcounted *ref = Z_COUNTED_P(retval_ptr);
| GET_ZVAL_PTR r0, op1_addr
| // ZVAL_COPY_VALUE(return_value, &ref->value);
| ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_R2, ZREG_R2
| GC_DELREF r0
| je >2
| // if (IS_REFCOUNTED())
if (jit_return_label >= 0) {
| IF_NOT_REFCOUNTED dh, =>jit_return_label
} else {
| IF_NOT_REFCOUNTED dh, >9
}
| // ADDREF
| GET_ZVAL_PTR r2, ret_addr // reload
| GC_ADDREF r2
if (jit_return_label >= 0) {
| jmp =>jit_return_label
} else {
| jmp >9
}
|2:
| EFREE_REFERENCE r0
if (jit_return_label >= 0) {
| jmp =>jit_return_label
} else {
| jmp >9
}
|.code
}
| ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2
}
|9:
return 1;
}
static int zend_jit_zval_copy_deref(dasm_State **Dst, zend_jit_addr res_addr, zend_jit_addr val_addr, zend_reg type_reg)
{
ZEND_ASSERT(type_reg == ZREG_R2);
|.if not(X64)
|| if (Z_REG(val_addr) == ZREG_R1) {
| GET_ZVAL_W2 r0, val_addr
|| }
|.endif
| GET_ZVAL_PTR r1, val_addr
|.if not(X64)
|| if (Z_REG(val_addr) != ZREG_R1) {
| GET_ZVAL_W2 r0, val_addr
|| }
|.endif
| IF_NOT_REFCOUNTED dh, >2
| IF_NOT_TYPE dl, IS_REFERENCE, >1
| GET_Z_TYPE_INFO edx, r1+offsetof(zend_reference, val)
|.if not(X64)
| GET_Z_W2 r0, r1+offsetof(zend_reference, val)
|.endif
| GET_Z_PTR r1, r1+offsetof(zend_reference, val)
| IF_NOT_REFCOUNTED dh, >2
|1:
| GC_ADDREF r1
|2:
| SET_ZVAL_PTR res_addr, r1
|.if not(X64)
| SET_ZVAL_W2 res_addr, r0
|.endif
| SET_ZVAL_TYPE_INFO res_addr, edx
return 1;
}
static zend_bool zend_jit_may_avoid_refcounting(const zend_op *opline, uint32_t op1_info)
{
switch (opline->opcode) {
case ZEND_FETCH_OBJ_FUNC_ARG:
if (!JIT_G(current_frame) ||
!JIT_G(current_frame)->call->func ||
!TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
return 0;
}
/* break missing intentionally */
case ZEND_FETCH_OBJ_R:
case ZEND_FETCH_OBJ_IS:
if ((op1_info & MAY_BE_OBJECT)
&& opline->op2_type == IS_CONST
&& Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_STRING
&& Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] != '\0') {
return 1;
}
break;
case ZEND_FETCH_DIM_FUNC_ARG:
if (!JIT_G(current_frame) ||
!JIT_G(current_frame)->call->func ||
!TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
return 0;
}
/* break missing intentionally */
case ZEND_FETCH_DIM_R:
case ZEND_FETCH_DIM_IS:
return 1;
case ZEND_ISSET_ISEMPTY_DIM_OBJ:
if (!(opline->extended_value & ZEND_ISEMPTY)) {
return 1;
}
break;
}
return 0;
}
static int zend_jit_fetch_dim_read(dasm_State **Dst,
const zend_op *opline,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
uint32_t op1_info,
zend_jit_addr op1_addr,
zend_bool op1_avoid_refcounting,
uint32_t op2_info,
uint32_t res_info,
zend_jit_addr res_addr,
int may_throw)
{
zend_jit_addr orig_op1_addr, op2_addr;
const void *exit_addr = NULL;
const void *not_found_exit_addr = NULL;
const void *res_exit_addr = NULL;
zend_bool result_avoid_refcounting = 0;
uint32_t may_be_string = (opline->opcode != ZEND_FETCH_LIST_R) ? MAY_BE_STRING : 0;
orig_op1_addr = OP1_ADDR();
op2_addr = OP2_ADDR();
if (opline->opcode != ZEND_FETCH_DIM_IS
&& JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
}
if ((res_info & MAY_BE_GUARD)
&& JIT_G(current_frame)
&& (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
uint32_t flags = 0;
uint32_t old_op1_info = 0;
uint32_t old_info;
zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
int32_t exit_point;
if (opline->opcode != ZEND_FETCH_LIST_R
&& (opline->op1_type & (IS_VAR|IS_TMP_VAR))
&& !op1_avoid_refcounting) {
flags |= ZEND_JIT_EXIT_FREE_OP1;
}
if ((opline->op2_type & (IS_VAR|IS_TMP_VAR))
&& (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
flags |= ZEND_JIT_EXIT_FREE_OP2;
}
if ((opline->result_type & (IS_VAR|IS_TMP_VAR))
&& !(flags & ZEND_JIT_EXIT_FREE_OP1)
&& (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))
&& (ssa_op+1)->op1_use == ssa_op->result_def
&& !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG)))
&& zend_jit_may_avoid_refcounting(opline+1, res_info)) {
result_avoid_refcounting = 1;
ssa->var_info[ssa_op->result_def].avoid_refcounting = 1;
}
if (op1_avoid_refcounting) {
old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
}
if (!(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG)))) {
old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_R0);
exit_point = zend_jit_trace_get_exit_point(opline+1, flags);
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info);
res_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!res_exit_addr) {
return 0;
}
res_info &= ~MAY_BE_GUARD;
ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
}
if (opline->opcode == ZEND_FETCH_DIM_IS
&& !(res_info & MAY_BE_NULL)) {
old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_NULL, 0);
SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NULL);
exit_point = zend_jit_trace_get_exit_point(opline+1, flags);
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info);
not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!not_found_exit_addr) {
return 0;
}
}
if (op1_avoid_refcounting) {
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info);
}
}
if (op1_info & MAY_BE_REF) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
| ZVAL_DEREF FCARG1a, op1_info
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
}
if (op1_info & MAY_BE_ARRAY) {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
if (exit_addr && !(op1_info & (MAY_BE_OBJECT|may_be_string))) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, &exit_addr
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7
}
}
| GET_ZVAL_LVAL ZREG_FCARG1a, op1_addr
if (!zend_jit_fetch_dimension_address_inner(Dst, opline, (opline->opcode != ZEND_FETCH_DIM_IS) ? BP_VAR_R : BP_VAR_IS, op1_info, op2_info, res_exit_addr, not_found_exit_addr, exit_addr)) {
return 0;
}
}
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) {
if (op1_info & MAY_BE_ARRAY) {
|.cold_code
|7:
}
if (opline->opcode != ZEND_FETCH_LIST_R && (op1_info & MAY_BE_STRING)) {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING))) {
if (exit_addr && !(op1_info & MAY_BE_OBJECT)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &exit_addr
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6
}
}
| SET_EX_OPLINE opline, r0
| GET_ZVAL_LVAL ZREG_FCARG1a, op1_addr
if (opline->opcode != ZEND_FETCH_DIM_IS) {
if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG) {
| GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr
| EXT_CALL zend_jit_fetch_dim_str_offset_r_helper, r0
} else {
| LOAD_ZVAL_ADDR FCARG2a, op2_addr
| EXT_CALL zend_jit_fetch_dim_str_r_helper, r0
}
| SET_ZVAL_PTR res_addr, r0
| SET_ZVAL_TYPE_INFO res_addr, IS_STRING
} else {
| LOAD_ZVAL_ADDR FCARG2a, op2_addr
|.if X64
| LOAD_ZVAL_ADDR CARG3, res_addr
|.else
| sub r4, 12
| PUSH_ZVAL_ADDR res_addr, r0
|.endif
| EXT_CALL zend_jit_fetch_dim_str_is_helper, r0
|.if not(X64)
| add r4, 12
|.endif
}
if ((op1_info & MAY_BE_ARRAY) ||
(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING)))) {
| jmp >9 // END
}
|6:
}
if (op1_info & MAY_BE_OBJECT) {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) {
if (exit_addr) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >6
}
}
| SET_EX_OPLINE opline, r0
if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
| LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1)
} else {
| LOAD_ZVAL_ADDR FCARG2a, op2_addr
}
|.if X64
| LOAD_ZVAL_ADDR CARG3, res_addr
|.else
| sub r4, 12
| PUSH_ZVAL_ADDR res_addr, r0
|.endif
if (opline->opcode != ZEND_FETCH_DIM_IS) {
| EXT_CALL zend_jit_fetch_dim_obj_r_helper, r0
} else {
| EXT_CALL zend_jit_fetch_dim_obj_is_helper, r0
}
|.if not(X64)
| add r4, 12
|.endif
if ((op1_info & MAY_BE_ARRAY) ||
(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) {
| jmp >9 // END
}
|6:
}
if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))
&& (!exit_addr || !(op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) {
if ((opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || (op2_info & MAY_BE_UNDEF)) {
| SET_EX_OPLINE opline, r0
if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1
| // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
| mov FCARG1d, opline->op1.var
| EXT_CALL zend_jit_undefined_op_helper, r0
|1:
}
if (op2_info & MAY_BE_UNDEF) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1
| mov FCARG1d, opline->op2.var
| EXT_CALL zend_jit_undefined_op_helper, r0
|1:
}
}
if (opline->opcode != ZEND_FETCH_DIM_IS && opline->opcode != ZEND_FETCH_LIST_R) {
if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
| LOAD_ZVAL_ADDR FCARG1a, orig_op1_addr
} else {
| SET_EX_OPLINE opline, r0
if (Z_MODE(op1_addr) != IS_MEM_ZVAL ||
Z_REG(op1_addr) != ZREG_FCARG1a ||
Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
}
| EXT_CALL zend_jit_invalid_array_access, r0
}
| SET_ZVAL_TYPE_INFO res_addr, IS_NULL
if (op1_info & MAY_BE_ARRAY) {
| jmp >9 // END
}
}
if (op1_info & MAY_BE_ARRAY) {
|.code
}
}
if (op1_info & MAY_BE_ARRAY) {
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0);
|8:
if (res_exit_addr) {
zend_uchar type = concrete_type(res_info);
if (op1_info & MAY_BE_ARRAY_OF_REF) {
| ZVAL_DEREF r0, MAY_BE_REF
}
if (type < IS_STRING) {
| IF_NOT_ZVAL_TYPE val_addr, type, &res_exit_addr
} else {
| GET_ZVAL_TYPE_INFO edx, val_addr
| IF_NOT_TYPE dl, type, &res_exit_addr
}
| // ZVAL_COPY
|7:
| ZVAL_COPY_VALUE_V res_addr, -1, val_addr, res_info, ZREG_R0, ZREG_R1
if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
if (type < IS_STRING) {
if (Z_REG(res_addr) != ZREG_FP ||
JIT_G(current_frame) == NULL ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(res_addr))) != type) {
| SET_ZVAL_TYPE_INFO res_addr, type
}
} else {
| SET_ZVAL_TYPE_INFO res_addr, edx
if (!result_avoid_refcounting) {
| TRY_ADDREF res_info, dh, r1
}
}
} else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) {
return 0;
}
} else if (op1_info & MAY_BE_ARRAY_OF_REF) {
| // ZVAL_COPY_DEREF
| GET_ZVAL_TYPE_INFO Rd(ZREG_R2), val_addr
if (!zend_jit_zval_copy_deref(Dst, res_addr, val_addr, ZREG_R2)) {
return 0;
}
} else {
| // ZVAL_COPY
| ZVAL_COPY_VALUE res_addr, -1, val_addr, res_info, ZREG_R1, ZREG_R2
| TRY_ADDREF res_info, ch, r2
}
}
|9: // END
#ifdef ZEND_JIT_USE_RC_INFERENCE
if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) {
/* Magic offsetGet() may increase refcount of the key */
op2_info |= MAY_BE_RCN;
}
#endif
| FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline
if (opline->opcode != ZEND_FETCH_LIST_R && !op1_avoid_refcounting) {
| FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline
}
if (may_throw) {
if (!zend_jit_check_exception(Dst)) {
return 0;
}
}
return 1;
}
static int zend_jit_fetch_dim(dasm_State **Dst,
const zend_op *opline,
uint32_t op1_info,
zend_jit_addr op1_addr,
uint32_t op2_info,
zend_jit_addr res_addr,
int may_throw)
{
zend_jit_addr op2_addr;
op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0;
if (op1_info & MAY_BE_REF) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
| IF_NOT_Z_TYPE FCARG1a, IS_REFERENCE, >1
| GET_Z_PTR FCARG2a, FCARG1a
| IF_NOT_TYPE byte [FCARG2a + offsetof(zend_reference, val) + offsetof(zval, u1.v.type)], IS_ARRAY, >2
| lea FCARG1a, [FCARG2a + offsetof(zend_reference, val)]
| jmp >3
|.cold_code
|2:
| SET_EX_OPLINE opline, r0
| EXT_CALL zend_jit_prepare_assign_dim_ref, r0
| test r0, r0
| mov FCARG1a, r0
| jne >1
| jmp ->exception_handler_undef
|.code
|1:
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
}
if (op1_info & MAY_BE_ARRAY) {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7
}
|3:
| SEPARATE_ARRAY op1_addr, op1_info, 1
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) {
if (op1_info & MAY_BE_ARRAY) {
|.cold_code
|7:
}
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) {
| CMP_ZVAL_TYPE op1_addr, IS_FALSE
| jg >7
}
if (Z_REG(op1_addr) != ZREG_FP) {
| mov T1, Ra(Z_REG(op1_addr)) // save
}
if ((op1_info & MAY_BE_UNDEF)
&& opline->opcode == ZEND_FETCH_DIM_RW) {
if (op1_info & (MAY_BE_NULL|MAY_BE_FALSE)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1
}
| SET_EX_OPLINE opline, r0
| mov FCARG1a, opline->op1.var
| EXT_CALL zend_jit_undefined_op_helper, r0
|1:
}
| // ZVAL_ARR(container, zend_new_array(8));
| EXT_CALL _zend_new_array_0, r0
if (Z_REG(op1_addr) != ZREG_FP) {
| mov Ra(Z_REG(op1_addr)), T1 // restore
}
| SET_ZVAL_LVAL op1_addr, r0
| SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX
| mov FCARG1a, r0
if (op1_info & MAY_BE_ARRAY) {
| jmp >1
|.code
|1:
}
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) {
|6:
if (opline->op2_type == IS_UNUSED) {
| // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval));
| LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval
| EXT_CALL zend_hash_next_index_insert, r0
| // if (UNEXPECTED(!var_ptr)) {
| test r0, r0
| jz >1
|.cold_code
|1:
| // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied");
| CANNOT_ADD_ELEMENT opline
| SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF
| //ZEND_VM_C_GOTO(assign_dim_op_ret_null);
| jmp >8
|.code
| SET_ZVAL_PTR res_addr, r0
| SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT
} else {
uint32_t type;
switch (opline->opcode) {
case ZEND_FETCH_DIM_W:
case ZEND_FETCH_LIST_W:
type = BP_VAR_W;
break;
case ZEND_FETCH_DIM_RW:
type = BP_VAR_RW;
break;
case ZEND_FETCH_DIM_UNSET:
type = BP_VAR_UNSET;
break;
default:
ZEND_UNREACHABLE();
}
if (!zend_jit_fetch_dimension_address_inner(Dst, opline, type, op1_info, op2_info, NULL, NULL, NULL)) {
return 0;
}
|8:
| SET_ZVAL_PTR res_addr, r0
| SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT
if (type == BP_VAR_RW || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) {
|.cold_code
|9:
| SET_ZVAL_TYPE_INFO res_addr, IS_NULL
| jmp >8
|.code
}
}
}
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) {
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) {
|.cold_code
|7:
}
| SET_EX_OPLINE opline, r0
if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
if (opline->op2_type == IS_UNUSED) {
| xor FCARG2a, FCARG2a
} else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
| LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1)
} else {
| LOAD_ZVAL_ADDR FCARG2a, op2_addr
}
|.if X64
| LOAD_ZVAL_ADDR CARG3, res_addr
|.else
| sub r4, 12
| PUSH_ZVAL_ADDR res_addr, r0
|.endif
switch (opline->opcode) {
case ZEND_FETCH_DIM_W:
case ZEND_FETCH_LIST_W:
| EXT_CALL zend_jit_fetch_dim_obj_w_helper, r0
break;
case ZEND_FETCH_DIM_RW:
| EXT_CALL zend_jit_fetch_dim_obj_rw_helper, r0
break;
// case ZEND_FETCH_DIM_UNSET:
// | EXT_CALL zend_jit_fetch_dim_obj_unset_helper, r0
// break;
default:
ZEND_UNREACHABLE();
}
|.if not(X64)
| add r4, 12
|.endif
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) {
| jmp >8 // END
|.code
}
}
#ifdef ZEND_JIT_USE_RC_INFERENCE
if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) {
/* ASSIGN_DIM may increase refcount of the key */
op2_info |= MAY_BE_RCN;
}
#endif
|8:
| FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline
if (may_throw) {
if (!zend_jit_check_exception(Dst)) {
return 0;
}
}
return 1;
}
static int zend_jit_isset_isempty_dim(dasm_State **Dst,
const zend_op *opline,
uint32_t op1_info,
zend_jit_addr op1_addr,
zend_bool op1_avoid_refcounting,
uint32_t op2_info,
int may_throw,
zend_uchar smart_branch_opcode,
uint32_t target_label,
uint32_t target_label2,
const void *exit_addr)
{
zend_jit_addr op2_addr, res_addr;
// TODO: support for empty() ???
ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY));
op2_addr = OP2_ADDR();
res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
if (op1_info & MAY_BE_REF) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
| ZVAL_DEREF FCARG1a, op1_info
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
}
if (op1_info & MAY_BE_ARRAY) {
const void *found_exit_addr = NULL;
const void *not_found_exit_addr = NULL;
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7
}
| GET_ZVAL_LVAL ZREG_FCARG1a, op1_addr
if (exit_addr
&& !(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY))
&& !may_throw
&& (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || op1_avoid_refcounting)
&& (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)))) {
if (smart_branch_opcode == ZEND_JMPNZ) {
found_exit_addr = exit_addr;
} else {
not_found_exit_addr = exit_addr;
}
}
if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_JIT_IS, op1_info, op2_info, found_exit_addr, not_found_exit_addr, NULL)) {
return 0;
}
if (found_exit_addr) {
|9:
return 1;
} else if (not_found_exit_addr) {
|8:
return 1;
}
}
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) {
if (op1_info & MAY_BE_ARRAY) {
|.cold_code
|7:
}
if (op1_info & (MAY_BE_STRING|MAY_BE_OBJECT)) {
| SET_EX_OPLINE opline, r0
if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
| LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1)
} else {
| LOAD_ZVAL_ADDR FCARG2a, op2_addr
}
| EXT_CALL zend_jit_isset_dim_helper, r0
| test r0, r0
| jz >9
if (op1_info & MAY_BE_ARRAY) {
| jmp >8
|.code
}
} else {
if (op2_info & MAY_BE_UNDEF) {
if (op2_info & MAY_BE_ANY) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1
}
| SET_EX_OPLINE opline, r0
| mov FCARG1d, opline->op2.var
| EXT_CALL zend_jit_undefined_op_helper, r0
|1:
}
if (op1_info & MAY_BE_ARRAY) {
| jmp >9
|.code
}
}
}
#ifdef ZEND_JIT_USE_RC_INFERENCE
if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) {
/* Magic offsetExists() may increase refcount of the key */
op2_info |= MAY_BE_RCN;
}
#endif
if (op1_info & (MAY_BE_ARRAY|MAY_BE_STRING|MAY_BE_OBJECT)) {
|8:
| FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline
if (!op1_avoid_refcounting) {
| FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline
}
if (may_throw) {
if (!zend_jit_check_exception_undef_result(Dst, opline)) {
return 0;
}
}
if (!(opline->extended_value & ZEND_ISEMPTY)) {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
| jmp &exit_addr
} else {
| jmp >8
}
} else if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ) {
| jmp =>target_label2
} else if (smart_branch_opcode == ZEND_JMPNZ) {
| jmp =>target_label
} else if (smart_branch_opcode == ZEND_JMPZNZ) {
| jmp =>target_label2
} else {
ZEND_UNREACHABLE();
}
} else {
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
| jmp >8
}
} else {
| //????
| int3
}
}
|9: // not found
| FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline
if (!op1_avoid_refcounting) {
| FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline
}
if (may_throw) {
if (!zend_jit_check_exception_undef_result(Dst, opline)) {
return 0;
}
}
if (!(opline->extended_value & ZEND_ISEMPTY)) {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
| jmp &exit_addr
}
} else if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ) {
| jmp =>target_label
} else if (smart_branch_opcode == ZEND_JMPNZ) {
} else if (smart_branch_opcode == ZEND_JMPZNZ) {
| jmp =>target_label
} else {
ZEND_UNREACHABLE();
}
} else {
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
}
} else {
| //????
| int3
}
|8:
return 1;
}
static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, uint32_t op1_info)
{
zend_jit_addr op1_addr = OP1_ADDR();
zend_string *varname = Z_STR_P(RT_CONSTANT(opline, opline->op2));
| // idx = (uint32_t)(uintptr_t)CACHED_PTR(opline->extended_value) - 1;
| mov r0, EX->run_time_cache
| mov r0, aword [r0 + opline->extended_value]
| sub r0, 1
| // if (EXPECTED(idx < EG(symbol_table).nNumUsed * sizeof(Bucket)))
| MEM_OP2_2_ZTS mov, ecx, dword, executor_globals, symbol_table.nNumUsed, r1
|.if X64
| shl r1, 5
|.else
| imul r1, sizeof(Bucket)
|.endif
| cmp r0, r1
| jae >9
| // Bucket *p = (Bucket*)((char*)EG(symbol_table).arData + idx);
| MEM_OP2_2_ZTS add, r0, aword, executor_globals, symbol_table.arData, r1
| IF_NOT_Z_TYPE r0, IS_REFERENCE, >9
| // (EXPECTED(p->key == varname))
| ADDR_OP2_2 cmp, aword [r0 + offsetof(Bucket, key)], varname, r1
| jne >9
| GET_Z_PTR r0, r0
| GC_ADDREF r0
|1:
if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| // if (UNEXPECTED(Z_REFCOUNTED_P(variable_ptr)))
| IF_ZVAL_REFCOUNTED op1_addr, >2
|.cold_code
|2:
}
| // zend_refcounted *garbage = Z_COUNTED_P(variable_ptr);
| GET_ZVAL_PTR FCARG1a, op1_addr
| // ZVAL_REF(variable_ptr, ref)
| SET_ZVAL_PTR op1_addr, r0
| SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX
| // if (GC_DELREF(garbage) == 0)
| GC_DELREF FCARG1a
if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) {
| jnz >3
} else {
| jnz >5
}
| ZVAL_DTOR_FUNC op1_info, opline
| jmp >5
if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) {
|3:
| // GC_ZVAL_CHECK_POSSIBLE_ROOT(variable_ptr)
| IF_GC_MAY_NOT_LEAK FCARG1a, >5
| EXT_CALL gc_possible_root, r1
| jmp >5
}
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
|.code
}
}
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| // ZVAL_REF(variable_ptr, ref)
| SET_ZVAL_PTR op1_addr, r0
| SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX
}
|5:
//END of handler
|.cold_code
|9:
| LOAD_ADDR FCARG1a, (ptrdiff_t)varname
| mov FCARG2a, EX->run_time_cache
if (opline->extended_value) {
| add FCARG2a, opline->extended_value
}
| EXT_CALL zend_jit_fetch_global_helper, r0
| jmp <1
|.code
return 1;
}
static int zend_jit_verify_arg_type(dasm_State **Dst, const zend_op *opline, zend_arg_info *arg_info, zend_bool check_exception)
{
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
zend_bool in_cold = 0;
uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY;
zend_reg tmp_reg = (type_mask == 0 || is_power_of_two(type_mask)) ? ZREG_FCARG1a : ZREG_R0;
if (ZEND_ARG_SEND_MODE(arg_info)) {
if (opline->opcode == ZEND_RECV_INIT) {
| LOAD_ZVAL_ADDR Ra(tmp_reg), res_addr
| ZVAL_DEREF Ra(tmp_reg), MAY_BE_REF
res_addr = ZEND_ADDR_MEM_ZVAL(tmp_reg, 0);
} else {
| GET_ZVAL_PTR Ra(tmp_reg), res_addr
res_addr = ZEND_ADDR_MEM_ZVAL(tmp_reg, offsetof(zend_reference, val));
}
}
if (type_mask != 0) {
if (is_power_of_two(type_mask)) {
uint32_t type_code = concrete_type(type_mask);
| IF_NOT_ZVAL_TYPE res_addr, type_code, >1
} else {
| mov edx, 1
| mov cl, byte [Ra(Z_REG(res_addr))+Z_OFFSET(res_addr)+offsetof(zval, u1.v.type)]
| shl edx, cl
| test edx, type_mask
| je >1
}
|.cold_code
|1:
in_cold = 1;
}
if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, res_addr
}
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
| SET_EX_OPLINE opline, r0
} else {
| ADDR_OP2_2 mov, aword EX->opline, opline, r0
}
| LOAD_ADDR FCARG2a, (ptrdiff_t)arg_info
| EXT_CALL zend_jit_verify_arg_slow, r0
if (check_exception) {
| test al, al
if (in_cold) {
| jnz >1
| jmp ->exception_handler
|.code
|1:
} else {
| jz ->exception_handler
}
} else if (in_cold) {
| jmp >1
|.code
|1:
}
return 1;
}
static int zend_jit_recv(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array)
{
uint32_t arg_num = opline->op1.num;
zend_arg_info *arg_info = NULL;
if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
if (EXPECTED(arg_num <= op_array->num_args)) {
arg_info = &op_array->arg_info[arg_num-1];
} else if (UNEXPECTED(op_array->fn_flags & ZEND_ACC_VARIADIC)) {
arg_info = &op_array->arg_info[op_array->num_args];
}
if (arg_info && !ZEND_TYPE_IS_SET(arg_info->type)) {
arg_info = NULL;
}
}
if (arg_info || (opline+1)->opcode != ZEND_RECV) {
| cmp dword EX->This.u2.num_args, arg_num
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| jb &exit_addr
} else {
| jb >1
|.cold_code
|1:
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
| SET_EX_OPLINE opline, r0
} else {
| ADDR_OP2_2 mov, aword EX->opline, opline, r0
}
| mov FCARG1a, FP
| EXT_CALL zend_missing_arg_error, r0
| jmp ->exception_handler
|.code
}
}
if (arg_info) {
if (!zend_jit_verify_arg_type(Dst, opline, arg_info, 1)) {
return 0;
}
}
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
if ((opline+1)->opcode != ZEND_RECV && (opline+1)->opcode != ZEND_RECV_INIT) {
| LOAD_IP_ADDR (opline + 1)
zend_jit_set_last_valid_opline(opline + 1);
}
}
return 1;
}
static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_bool is_last, int may_throw)
{
uint32_t arg_num = opline->op1.num;
zval *zv = RT_CONSTANT(opline, opline->op2);
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
| cmp dword EX->This.u2.num_args, arg_num
| jae >5
}
| ZVAL_COPY_CONST res_addr, -1, -1, zv, ZREG_R0
if (Z_REFCOUNTED_P(zv)) {
| ADDREF_CONST zv, r0
}
if (Z_CONSTANT_P(zv)) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
| SET_EX_OPLINE opline, r0
} else {
| ADDR_OP2_2 mov, aword EX->opline, opline, r0
}
| LOAD_ZVAL_ADDR FCARG1a, res_addr
| mov r0, EX->func
| mov FCARG2a, [r0 + offsetof(zend_op_array, scope)]
| .if X64
| EXT_CALL zval_update_constant_ex, r0
| .else
||#if (PHP_VERSION_ID < 80100) && (SIZEOF_SIZE_T == 4)
| EXT_CALL zval_jit_update_constant_ex, r0
||#else
| EXT_CALL zval_update_constant_ex, r0
||#endif
| .endif
| test al, al
| jnz >1
|.cold_code
|1:
| ZVAL_PTR_DTOR res_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, 0, opline
| SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF
| jmp ->exception_handler
|.code
}
|5:
if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
do {
zend_arg_info *arg_info;
if (arg_num <= op_array->num_args) {
arg_info = &op_array->arg_info[arg_num-1];
} else if (op_array->fn_flags & ZEND_ACC_VARIADIC) {
arg_info = &op_array->arg_info[op_array->num_args];
} else {
break;
}
if (!ZEND_TYPE_IS_SET(arg_info->type)) {
break;
}
if (!zend_jit_verify_arg_type(Dst, opline, arg_info, may_throw)) {
return 0;
}
} while (0);
}
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
if (is_last) {
| LOAD_IP_ADDR (opline + 1)
zend_jit_set_last_valid_opline(opline + 1);
}
}
return 1;
}
static zend_property_info* zend_get_known_property_info(zend_class_entry *ce, zend_string *member, zend_bool on_this, zend_string *filename)
{
zend_property_info *info = NULL;
if (!ce ||
!(ce->ce_flags & ZEND_ACC_LINKED) ||
(ce->ce_flags & ZEND_ACC_TRAIT) ||
ce->create_object) {
return NULL;
}
if (!(ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
if (ce->info.user.filename != filename) {
/* class declaration might be changed independently */
return NULL;
}
if (ce->parent) {
zend_class_entry *parent = ce->parent;
do {
if (parent->type == ZEND_INTERNAL_CLASS) {
break;
} else if (parent->info.user.filename != filename) {
/* some of parents class declarations might be changed independently */
/* TODO: this check may be not enough, because even
* in the same it's possible to conditionally define
* few classes with the same name, and "parent" may
* change from request to request.
*/
return NULL;
}
parent = parent->parent;
} while (parent);
}
}
info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member);
if (info == NULL ||
!IS_VALID_PROPERTY_OFFSET(info->offset) ||
(info->flags & ZEND_ACC_STATIC)) {
return NULL;
}
if (!(info->flags & ZEND_ACC_PUBLIC) &&
(!on_this || info->ce != ce)) {
return NULL;
}
return info;
}
static zend_bool zend_may_be_dynamic_property(zend_class_entry *ce, zend_string *member, zend_bool on_this, zend_string *filename)
{
zend_property_info *info;
if (!ce || (ce->ce_flags & ZEND_ACC_TRAIT)) {
return 1;
}
if (!(ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
if (ce->info.user.filename != filename) {
/* class declaration might be changed independently */
return 1;
}
}
info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member);
if (info == NULL ||
!IS_VALID_PROPERTY_OFFSET(info->offset) ||
(info->flags & ZEND_ACC_STATIC)) {
return 1;
}
if (!(info->flags & ZEND_ACC_PUBLIC) &&
(!on_this || info->ce != ce)) {
return 1;
}
return 0;
}
static int zend_jit_class_guard(dasm_State **Dst, const zend_op *opline, zend_class_entry *ce)
{
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
|.if X64
|| if (!IS_SIGNED_32BIT(ce)) {
| mov64 r0, ((ptrdiff_t)ce)
| cmp aword [FCARG1a + offsetof(zend_object, ce)], r0
|| } else {
| cmp aword [FCARG1a + offsetof(zend_object, ce)], ce
|| }
|.else
| cmp aword [FCARG1a + offsetof(zend_object, ce)], ce
|.endif
| jne &exit_addr
return 1;
}
static int zend_jit_fetch_obj(dasm_State **Dst,
const zend_op *opline,
const zend_op_array *op_array,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
uint32_t op1_info,
zend_jit_addr op1_addr,
zend_bool op1_indirect,
zend_class_entry *ce,
zend_bool ce_is_instanceof,
zend_bool use_this,
zend_bool op1_avoid_refcounting,
zend_class_entry *trace_ce,
int may_throw)
{
zval *member;
zend_property_info *prop_info;
zend_bool may_be_dynamic = 1;
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
zend_jit_addr prop_addr;
uint32_t res_info = RES_INFO();
ZEND_ASSERT(opline->op2_type == IS_CONST);
ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
member = RT_CONSTANT(opline, opline->op2);
ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0');
prop_info = zend_get_known_property_info(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename);
if (opline->op1_type == IS_UNUSED || use_this) {
| GET_ZVAL_PTR FCARG1a, this_addr
} else {
if (opline->op1_type == IS_VAR
&& opline->opcode == ZEND_FETCH_OBJ_W
&& (op1_info & MAY_BE_INDIRECT)
&& Z_REG(op1_addr) == ZREG_FP) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
| IF_NOT_Z_TYPE FCARG1a, IS_INDIRECT, >1
| GET_Z_PTR FCARG1a, FCARG1a
|1:
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
}
if (op1_info & MAY_BE_REF) {
if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
| ZVAL_DEREF FCARG1a, op1_info
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
}
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >7
}
}
| GET_ZVAL_PTR FCARG1a, op1_addr
}
if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
prop_info = zend_get_known_property_info(trace_ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename);
if (prop_info) {
ce = trace_ce;
ce_is_instanceof = 0;
if (!(op1_info & MAY_BE_CLASS_GUARD)) {
if (!zend_jit_class_guard(Dst, opline, trace_ce)) {
return 0;
}
if (ssa->var_info && ssa_op->op1_use >= 0) {
ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op1_use].ce = ce;
ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof;
}
}
}
}
if (!prop_info) {
| mov r0, EX->run_time_cache
| mov r2, aword [r0 + (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS)]
| cmp r2, aword [FCARG1a + offsetof(zend_object, ce)]
| jne >5
| mov r0, aword [r0 + (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)]
may_be_dynamic = zend_may_be_dynamic_property(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename);
if (may_be_dynamic) {
| test r0, r0
if (opline->opcode == ZEND_FETCH_OBJ_W) {
| jl >5
} else {
| jl >8 // dynamic property
}
}
| mov edx, dword [FCARG1a + r0 + 8]
| IF_UNDEF dl, >5
| add FCARG1a, r0
prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
if (opline->opcode == ZEND_FETCH_OBJ_W
&& (opline->extended_value & ZEND_FETCH_OBJ_FLAGS)
&& (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS))) {
uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS;
| mov r0, EX->run_time_cache
| mov FCARG2a, aword [r0 + (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2]
| test FCARG2a, FCARG2a
| jnz >1
|.cold_code
|1:
if (flags == ZEND_FETCH_DIM_WRITE) {
| SET_EX_OPLINE opline, r0
| EXT_CALL zend_jit_check_array_promotion, r0
| jmp >9
} else if (flags == ZEND_FETCH_REF) {
|.if X64
| LOAD_ZVAL_ADDR CARG3, res_addr
|.else
| sub r4, 12
| PUSH_ZVAL_ADDR res_addr, r0
|.endif
| EXT_CALL zend_jit_create_typed_ref, r0
|.if not(X64)
| add r4, 12
|.endif
| jmp >9
} else {
ZEND_UNREACHABLE();
}
|.code
}
} else {
prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, prop_info->offset);
| mov edx, dword [FCARG1a + prop_info->offset + 8]
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
if (opline->opcode == ZEND_FETCH_OBJ_W || !(res_info & MAY_BE_GUARD) || !JIT_G(current_frame)) {
/* perform IS_UNDEF check only after result type guard (during deoptimization) */
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_UNDEF dl, &exit_addr
}
} else {
| IF_UNDEF dl, >5
}
if (opline->opcode == ZEND_FETCH_OBJ_W
&& (opline->extended_value & ZEND_FETCH_OBJ_FLAGS)
&& ZEND_TYPE_IS_SET(prop_info->type)) {
uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS;
if (flags == ZEND_FETCH_DIM_WRITE) {
if ((ZEND_TYPE_FULL_MASK(prop_info->type) & (MAY_BE_ITERABLE|MAY_BE_ARRAY)) == 0) {
| cmp dl, IS_FALSE
| jle >1
|.cold_code
|1:
if (Z_REG(prop_addr) != ZREG_FCARG1a || Z_OFFSET(prop_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, prop_addr
}
| LOAD_ADDR FCARG2a, prop_info
| SET_EX_OPLINE opline, r0
| EXT_CALL zend_jit_check_array_promotion, r0
| jmp >9
|.code
}
} else if (flags == ZEND_FETCH_REF) {
| IF_TYPE dl, IS_REFERENCE, >1
if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) {
| LOAD_ADDR FCARG2a, prop_info
} else {
int prop_info_offset =
(((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*));
| mov r0, aword [FCARG1a + offsetof(zend_object, ce)]
| mov r0, aword [r0 + offsetof(zend_class_entry, properties_info_table)]
| mov FCARG2a, aword[r0 + prop_info_offset]
}
if (Z_REG(prop_addr) != ZREG_FCARG1a || Z_OFFSET(prop_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, prop_addr
}
|.if X64
| LOAD_ZVAL_ADDR CARG3, res_addr
|.else
| sub r4, 12
| PUSH_ZVAL_ADDR res_addr, r0
|.endif
| EXT_CALL zend_jit_create_typed_ref, r0
|.if not(X64)
| add r4, 12
|.endif
| jmp >9
|1:
} else {
ZEND_UNREACHABLE();
}
}
}
if (op1_avoid_refcounting) {
SET_STACK_REG(JIT_G(current_frame)->stack,
EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
}
if (opline->opcode == ZEND_FETCH_OBJ_W) {
if (Z_REG(prop_addr) != ZREG_FCARG1a || Z_OFFSET(prop_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, prop_addr
}
| SET_ZVAL_PTR res_addr, FCARG1a
| SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && prop_info) {
ssa->var_info[ssa_op->result_def].indirect_reference = 1;
}
} else {
zend_bool result_avoid_refcounting = 0;
if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame) && prop_info) {
uint32_t flags = 0;
uint32_t old_info;
zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
int32_t exit_point;
const void *exit_addr;
zend_uchar type;
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0);
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
&& !use_this
&& !op1_avoid_refcounting) {
flags = ZEND_JIT_EXIT_FREE_OP1;
}
| LOAD_ZVAL_ADDR r0, prop_addr
if ((opline->result_type & (IS_VAR|IS_TMP_VAR))
&& !(flags & ZEND_JIT_EXIT_FREE_OP1)
&& (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))
&& (ssa_op+1)->op1_use == ssa_op->result_def
&& zend_jit_may_avoid_refcounting(opline+1, res_info)) {
result_avoid_refcounting = 1;
ssa->var_info[ssa_op->result_def].avoid_refcounting = 1;
}
old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_R0);
exit_point = zend_jit_trace_get_exit_point(opline+1, flags);
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
res_info &= ~MAY_BE_GUARD;
ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
type = concrete_type(res_info);
| // ZVAL_DEREF()
| IF_NOT_TYPE dl, IS_REFERENCE, >1
| GET_Z_PTR r0, r0
| add r0, offsetof(zend_reference, val)
if (type < IS_STRING) {
|1:
| IF_NOT_ZVAL_TYPE val_addr, type, &exit_addr
} else {
| GET_ZVAL_TYPE_INFO edx, val_addr
|1:
| IF_NOT_TYPE dl, type, &exit_addr
}
| // ZVAL_COPY
| ZVAL_COPY_VALUE_V res_addr, -1, val_addr, res_info, ZREG_R0, ZREG_R1
if (type < IS_STRING) {
if (Z_REG(res_addr) != ZREG_FP ||
JIT_G(current_frame) == NULL ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(res_addr))) != type) {
| SET_ZVAL_TYPE_INFO res_addr, type
}
} else {
| SET_ZVAL_TYPE_INFO res_addr, edx
if (!result_avoid_refcounting) {
| TRY_ADDREF res_info, dh, r1
}
}
} else {
if (!zend_jit_zval_copy_deref(Dst, res_addr, prop_addr, ZREG_R2)) {
return 0;
}
}
}
|.cold_code
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || !prop_info) {
|5:
| SET_EX_OPLINE opline, r0
if (opline->opcode == ZEND_FETCH_OBJ_W) {
| EXT_CALL zend_jit_fetch_obj_w_slow, r0
} else if (opline->opcode != ZEND_FETCH_OBJ_IS) {
| EXT_CALL zend_jit_fetch_obj_r_slow, r0
} else {
| EXT_CALL zend_jit_fetch_obj_is_slow, r0
}
| jmp >9
}
if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)- MAY_BE_OBJECT)) && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
|7:
if (opline->opcode != ZEND_FETCH_OBJ_IS) {
| SET_EX_OPLINE opline, r0
if (opline->opcode != ZEND_FETCH_OBJ_W
&& (op1_info & MAY_BE_UNDEF)) {
zend_jit_addr orig_op1_addr = OP1_ADDR();
if (op1_info & MAY_BE_ANY) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1
}
| mov FCARG1d, opline->op1.var
| EXT_CALL zend_jit_undefined_op_helper, r0
|1:
| LOAD_ZVAL_ADDR FCARG1a, orig_op1_addr
} else if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
| LOAD_ADDR FCARG2a, Z_STRVAL_P(member)
if (opline->opcode == ZEND_FETCH_OBJ_W) {
| EXT_CALL zend_jit_invalid_property_write, r0
| SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR
} else {
| EXT_CALL zend_jit_invalid_property_read, r0
| SET_ZVAL_TYPE_INFO res_addr, IS_NULL
}
| jmp >9
} else {
| SET_ZVAL_TYPE_INFO res_addr, IS_NULL
| jmp >9
}
}
if (!prop_info
&& may_be_dynamic
&& opline->opcode != ZEND_FETCH_OBJ_W) {
|8:
| mov FCARG2a, r0
| SET_EX_OPLINE opline, r0
if (opline->opcode != ZEND_FETCH_OBJ_IS) {
| EXT_CALL zend_jit_fetch_obj_r_dynamic, r0
} else {
| EXT_CALL zend_jit_fetch_obj_is_dynamic, r0
}
| jmp >9
}
|.code;
|9: // END
if (opline->op1_type != IS_UNUSED && !use_this && !op1_indirect) {
if (opline->op1_type == IS_VAR
&& opline->opcode == ZEND_FETCH_OBJ_W
&& (op1_info & MAY_BE_RC1)) {
zend_jit_addr orig_op1_addr = OP1_ADDR();
| IF_NOT_ZVAL_REFCOUNTED orig_op1_addr, >1
| GET_ZVAL_PTR FCARG1a, orig_op1_addr
| GC_DELREF FCARG1a
| jnz >1
| SET_EX_OPLINE opline, r0
| EXT_CALL zend_jit_extract_helper, r0
|1:
} else if (!op1_avoid_refcounting) {
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline
}
}
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& prop_info
&& (opline->opcode != ZEND_FETCH_OBJ_W ||
!(opline->extended_value & ZEND_FETCH_OBJ_FLAGS) ||
!ZEND_TYPE_IS_SET(prop_info->type))
&& opline->op1_type != IS_VAR
&& opline->op1_type != IS_TMP_VAR) {
may_throw = 0;
}
if (may_throw) {
if (!zend_jit_check_exception(Dst)) {
return 0;
}
}
return 1;
}
static int zend_jit_incdec_obj(dasm_State **Dst,
const zend_op *opline,
const zend_op_array *op_array,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
uint32_t op1_info,
zend_jit_addr op1_addr,
zend_bool op1_indirect,
zend_class_entry *ce,
zend_bool ce_is_instanceof,
zend_bool use_this,
zend_class_entry *trace_ce,
int may_throw)
{
zval *member;
zend_string *name;
zend_property_info *prop_info;
zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
zend_jit_addr res_addr = 0;
zend_jit_addr prop_addr;
zend_bool needs_slow_path = 0;
ZEND_ASSERT(opline->op2_type == IS_CONST);
ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
if (opline->result_type != IS_UNUSED) {
res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
}
member = RT_CONSTANT(opline, opline->op2);
ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0');
name = Z_STR_P(member);
prop_info = zend_get_known_property_info(ce, name, opline->op1_type == IS_UNUSED, op_array->filename);
if (opline->op1_type == IS_UNUSED || use_this) {
| GET_ZVAL_PTR FCARG1a, this_addr
} else {
if (opline->op1_type == IS_VAR
&& (op1_info & MAY_BE_INDIRECT)
&& Z_REG(op1_addr) == ZREG_FP) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
| IF_NOT_Z_TYPE FCARG1a, IS_INDIRECT, >1
| GET_Z_PTR FCARG1a, FCARG1a
|1:
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
}
if (op1_info & MAY_BE_REF) {
if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
| ZVAL_DEREF FCARG1a, op1_info
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
}
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1
|.cold_code
|1:
| SET_EX_OPLINE opline, r0
if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
| LOAD_ADDR FCARG2a, ZSTR_VAL(name)
| EXT_CALL zend_jit_invalid_property_incdec, r0
| jmp ->exception_handler
|.code
}
}
| GET_ZVAL_PTR FCARG1a, op1_addr
}
if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
prop_info = zend_get_known_property_info(trace_ce, name, opline->op1_type == IS_UNUSED, op_array->filename);
if (prop_info) {
ce = trace_ce;
ce_is_instanceof = 0;
if (!(op1_info & MAY_BE_CLASS_GUARD)) {
if (!zend_jit_class_guard(Dst, opline, trace_ce)) {
return 0;
}
if (ssa->var_info && ssa_op->op1_use >= 0) {
ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op1_use].ce = ce;
ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof;
}
if (ssa->var_info && ssa_op->op1_def >= 0) {
ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op1_def].ce = ce;
ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof;
}
}
}
}
if (!prop_info) {
needs_slow_path = 1;
| mov r0, EX->run_time_cache
| mov r2, aword [r0 + opline->extended_value]
| cmp r2, aword [FCARG1a + offsetof(zend_object, ce)]
| jne >7
if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
| cmp aword [r0 + opline->extended_value + sizeof(void*) * 2], 0
| jnz >7
}
| mov r0, aword [r0 + opline->extended_value + sizeof(void*)]
| test r0, r0
| jl >7
| IF_TYPE byte [FCARG1a + r0 + 8], IS_UNDEF, >7
| add FCARG1a, r0
prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
} else {
prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, prop_info->offset);
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, &exit_addr
} else {
| IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, >7
needs_slow_path = 1;
}
if (ZEND_TYPE_IS_SET(prop_info->type)) {
| SET_EX_OPLINE opline, r0
if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) {
| LOAD_ADDR FCARG2a, prop_info
} else {
int prop_info_offset =
(((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*));
| mov r0, aword [FCARG1a + offsetof(zend_object, ce)]
| mov r0, aword [r0 + offsetof(zend_class_entry, properties_info_table)]
| mov FCARG2a, aword[r0 + prop_info_offset]
}
| LOAD_ZVAL_ADDR FCARG1a, prop_addr
if (opline->result_type == IS_UNUSED) {
switch (opline->opcode) {
case ZEND_PRE_INC_OBJ:
case ZEND_POST_INC_OBJ:
| EXT_CALL zend_jit_inc_typed_prop, r0
break;
case ZEND_PRE_DEC_OBJ:
case ZEND_POST_DEC_OBJ:
| EXT_CALL zend_jit_dec_typed_prop, r0
break;
default:
ZEND_UNREACHABLE();
}
} else {
|.if X64
| LOAD_ZVAL_ADDR CARG3, res_addr
|.else
| sub r4, 12
| PUSH_ZVAL_ADDR res_addr, r0
|.endif
switch (opline->opcode) {
case ZEND_PRE_INC_OBJ:
| EXT_CALL zend_jit_pre_inc_typed_prop, r0
break;
case ZEND_PRE_DEC_OBJ:
| EXT_CALL zend_jit_pre_dec_typed_prop, r0
break;
case ZEND_POST_INC_OBJ:
| EXT_CALL zend_jit_post_inc_typed_prop, r0
break;
case ZEND_POST_DEC_OBJ:
| EXT_CALL zend_jit_post_dec_typed_prop, r0
break;
default:
ZEND_UNREACHABLE();
}
|.if not(X64)
| add r4, 12
|.endif
}
}
}
if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) {
zend_jit_addr var_addr = prop_addr;
var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
if (Z_REG(prop_addr) != ZREG_FCARG1a || Z_OFFSET(prop_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, prop_addr
}
| IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2
| GET_ZVAL_PTR FCARG1a, var_addr
| cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0
| jnz >1
| lea FCARG1a, aword [FCARG1a + offsetof(zend_reference, val)]
|.cold_code
|1:
if (opline) {
| SET_EX_OPLINE opline, r0
}
if (opline->result_type == IS_UNUSED) {
| xor FCARG2a, FCARG2a
} else {
| LOAD_ZVAL_ADDR FCARG2a, res_addr
}
switch (opline->opcode) {
case ZEND_PRE_INC_OBJ:
| EXT_CALL zend_jit_pre_inc_typed_ref, r0
break;
case ZEND_PRE_DEC_OBJ:
| EXT_CALL zend_jit_pre_dec_typed_ref, r0
break;
case ZEND_POST_INC_OBJ:
| EXT_CALL zend_jit_post_inc_typed_ref, r0
break;
case ZEND_POST_DEC_OBJ:
| EXT_CALL zend_jit_post_dec_typed_ref, r0
break;
default:
ZEND_UNREACHABLE();
}
| jmp >9
|.code
|2:
| IF_NOT_ZVAL_TYPE var_addr, IS_LONG, >2
if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) {
if (opline->result_type != IS_UNUSED) {
| ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_LONG, ZREG_R1, ZREG_R2
}
}
if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) {
| LONG_OP_WITH_32BIT_CONST add, var_addr, Z_L(1)
} else {
| LONG_OP_WITH_32BIT_CONST sub, var_addr, Z_L(1)
}
| jo >3
if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ) {
if (opline->result_type != IS_UNUSED) {
| ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_LONG, ZREG_R0, ZREG_R2
}
}
|.cold_code
|2:
if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) {
| ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_ANY, ZREG_R0, ZREG_R2
| TRY_ADDREF MAY_BE_ANY, ah, r2
}
if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) {
if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) {
| LOAD_ZVAL_ADDR FCARG2a, res_addr
| EXT_CALL zend_jit_pre_inc, r0
} else {
| EXT_CALL increment_function, r0
}
} else {
if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) {
| LOAD_ZVAL_ADDR FCARG2a, res_addr
| EXT_CALL zend_jit_pre_dec, r0
} else {
| EXT_CALL decrement_function, r0
}
}
| jmp >4
|3:
if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) {
|.if X64
| mov64 rax, 0x43e0000000000000
| SET_ZVAL_LVAL var_addr, rax
| SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE
if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) {
| SET_ZVAL_LVAL res_addr, rax
| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE
}
|.else
| SET_ZVAL_LVAL var_addr, 0
| SET_ZVAL_W2 var_addr, 0x41e00000
| SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE
if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) {
| SET_ZVAL_LVAL res_addr, 0
| SET_ZVAL_W2 res_addr, 0x41e00000
| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE
}
|.endif
} else {
|.if X64
| mov64 rax, 0xc3e0000000000000
| SET_ZVAL_LVAL var_addr, rax
| SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE
if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) {
| SET_ZVAL_LVAL res_addr, rax
| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE
}
|.else
| SET_ZVAL_LVAL var_addr, 0x00200000
| SET_ZVAL_W2 var_addr, 0xc1e00000
| SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE
if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) {
| SET_ZVAL_LVAL res_addr, 0x00200000
| SET_ZVAL_W2 res_addr, 0xc1e00000
| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE
}
|.endif
}
| jmp >4
|.code
|4:
}
if (needs_slow_path) {
|.cold_code
|7:
| SET_EX_OPLINE opline, r0
| // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value));
| LOAD_ADDR FCARG2a, name
|.if X64
| mov CARG3, EX->run_time_cache
| add CARG3, opline->extended_value
if (opline->result_type == IS_UNUSED) {
| xor CARG4, CARG4
} else {
| LOAD_ZVAL_ADDR CARG4, res_addr
}
|.else
| sub r4, 8
if (opline->result_type == IS_UNUSED) {
| push 0
} else {
| PUSH_ZVAL_ADDR res_addr, r0
}
| mov r0, EX->run_time_cache
| add r0, opline->extended_value
| push r0
|.endif
switch (opline->opcode) {
case ZEND_PRE_INC_OBJ:
| EXT_CALL zend_jit_pre_inc_obj_helper, r0
break;
case ZEND_PRE_DEC_OBJ:
| EXT_CALL zend_jit_pre_dec_obj_helper, r0
break;
case ZEND_POST_INC_OBJ:
| EXT_CALL zend_jit_post_inc_obj_helper, r0
break;
case ZEND_POST_DEC_OBJ:
| EXT_CALL zend_jit_post_dec_obj_helper, r0
break;
default:
ZEND_UNREACHABLE();
}
|.if not(X64)
| add r4, 8
|.endif
| jmp >9
|.code
}
|9:
if (opline->op1_type != IS_UNUSED && !use_this && !op1_indirect) {
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline
}
if (may_throw) {
if (!zend_jit_check_exception(Dst)) {
return 0;
}
}
return 1;
}
static int zend_jit_assign_obj_op(dasm_State **Dst,
const zend_op *opline,
const zend_op_array *op_array,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
uint32_t op1_info,
zend_jit_addr op1_addr,
uint32_t val_info,
zend_ssa_range *val_range,
zend_bool op1_indirect,
zend_class_entry *ce,
zend_bool ce_is_instanceof,
zend_bool use_this,
zend_class_entry *trace_ce,
int may_throw)
{
zval *member;
zend_string *name;
zend_property_info *prop_info;
zend_jit_addr val_addr = OP1_DATA_ADDR();
zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
zend_jit_addr prop_addr;
zend_bool needs_slow_path = 0;
binary_op_type binary_op = get_binary_op(opline->extended_value);
ZEND_ASSERT(opline->op2_type == IS_CONST);
ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
ZEND_ASSERT(opline->result_type == IS_UNUSED);
member = RT_CONSTANT(opline, opline->op2);
ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0');
name = Z_STR_P(member);
prop_info = zend_get_known_property_info(ce, name, opline->op1_type == IS_UNUSED, op_array->filename);
if (opline->op1_type == IS_UNUSED || use_this) {
| GET_ZVAL_PTR FCARG1a, this_addr
} else {
if (opline->op1_type == IS_VAR
&& (op1_info & MAY_BE_INDIRECT)
&& Z_REG(op1_addr) == ZREG_FP) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
| IF_NOT_Z_TYPE FCARG1a, IS_INDIRECT, >1
| GET_Z_PTR FCARG1a, FCARG1a
|1:
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
}
if (op1_info & MAY_BE_REF) {
if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
| ZVAL_DEREF FCARG1a, op1_info
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
}
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1
|.cold_code
|1:
| SET_EX_OPLINE opline, r0
if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
| LOAD_ADDR FCARG2a, ZSTR_VAL(name)
if (op1_info & MAY_BE_UNDEF) {
| EXT_CALL zend_jit_invalid_property_assign_op, r0
} else {
| EXT_CALL zend_jit_invalid_property_assign, r0
}
may_throw = 1;
if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR))
&& (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| jmp >8
} else {
| jmp >9
}
|.code
}
}
| GET_ZVAL_PTR FCARG1a, op1_addr
}
if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
prop_info = zend_get_known_property_info(trace_ce, name, opline->op1_type == IS_UNUSED, op_array->filename);
if (prop_info) {
ce = trace_ce;
ce_is_instanceof = 0;
if (!(op1_info & MAY_BE_CLASS_GUARD)) {
if (!zend_jit_class_guard(Dst, opline, trace_ce)) {
return 0;
}
if (ssa->var_info && ssa_op->op1_use >= 0) {
ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op1_use].ce = ce;
ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof;
}
if (ssa->var_info && ssa_op->op1_def >= 0) {
ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op1_def].ce = ce;
ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof;
}
}
}
}
if (!prop_info) {
needs_slow_path = 1;
| mov r0, EX->run_time_cache
| mov r2, aword [r0 + (opline+1)->extended_value]
| cmp r2, aword [FCARG1a + offsetof(zend_object, ce)]
| jne >7
if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
| cmp aword [r0 + ((opline+1)->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2], 0
| jnz >7
}
| mov r0, aword [r0 + (opline+1)->extended_value + sizeof(void*)]
| test r0, r0
| jl >7
| IF_TYPE byte [FCARG1a + r0 + 8], IS_UNDEF, >7
| add FCARG1a, r0
prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
} else {
prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, prop_info->offset);
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, &exit_addr
} else {
| IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, >7
needs_slow_path = 1;
}
if (ZEND_TYPE_IS_SET(prop_info->type)) {
uint32_t info = val_info;
if (opline) {
| SET_EX_OPLINE opline, r0
}
| IF_ZVAL_TYPE prop_addr, IS_REFERENCE, >1
|.cold_code
|1:
| GET_ZVAL_PTR FCARG1a, prop_addr
if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2a || Z_OFFSET(val_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG2a, val_addr
}
|.if X64
| LOAD_ADDR CARG3, binary_op
|.else
| sub r4, 12
| PUSH_ADDR binary_op, r0
|.endif
if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR))
&& (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, r0
} else {
| EXT_CALL zend_jit_assign_op_to_typed_ref, r0
}
|.if not(X64)
| add r4, 12
|.endif
| jmp >9
|.code
| // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC);
if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) {
| LOAD_ADDR FCARG2a, prop_info
} else {
int prop_info_offset =
(((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*));
| mov r0, aword [FCARG1a + offsetof(zend_object, ce)]
| mov r0, aword [r0 + offsetof(zend_class_entry, properties_info_table)]
| mov FCARG2a, aword[r0 + prop_info_offset]
}
| LOAD_ZVAL_ADDR FCARG1a, prop_addr
|.if X64
| LOAD_ZVAL_ADDR CARG3, val_addr
| LOAD_ADDR CARG4, binary_op
|.else
| sub r4, 8
| PUSH_ADDR binary_op, r0
| PUSH_ZVAL_ADDR val_addr, r0
|.endif
| EXT_CALL zend_jit_assign_op_to_typed_prop, r0
|.if not(X64)
| add r4, 8
|.endif
if (info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
info |= MAY_BE_RC1|MAY_BE_RCN;
}
| FREE_OP (opline+1)->op1_type, (opline+1)->op1, info, 0, opline
}
}
if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) {
zend_jit_addr var_addr = prop_addr;
uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN;
uint32_t var_def_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN;
var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0);
| LOAD_ZVAL_ADDR r0, prop_addr
| IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2
| GET_ZVAL_PTR FCARG1a, var_addr
| cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0
| jnz >1
| lea r0, aword [FCARG1a + offsetof(zend_reference, val)]
|.cold_code
|1:
if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2a || Z_OFFSET(val_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG2a, val_addr
}
if (opline) {
| SET_EX_OPLINE opline, r0
}
|.if X64
| LOAD_ADDR CARG3, binary_op
|.else
| sub r4, 12
| PUSH_ADDR binary_op, r0
|.endif
if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR))
&& (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, r0
} else {
| EXT_CALL zend_jit_assign_op_to_typed_ref, r0
}
|.if not(X64)
| add r4, 12
|.endif
| jmp >9
|.code
|2:
switch (opline->extended_value) {
case ZEND_ADD:
case ZEND_SUB:
case ZEND_MUL:
case ZEND_DIV:
if (!zend_jit_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, 0, var_addr, var_def_info, var_info,
1 /* may overflow */, 0)) {
return 0;
}
break;
case ZEND_BW_OR:
case ZEND_BW_AND:
case ZEND_BW_XOR:
case ZEND_SL:
case ZEND_SR:
case ZEND_MOD:
if (!zend_jit_long_math_helper(Dst, opline, opline->extended_value,
IS_CV, opline->op1, var_addr, var_info, NULL,
(opline+1)->op1_type, (opline+1)->op1, val_addr, val_info,
val_range,
0, var_addr, var_def_info, var_info, 0)) {
return 0;
}
break;
case ZEND_CONCAT:
if (!zend_jit_concat_helper(Dst, opline, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, var_addr,
0)) {
return 0;
}
break;
default:
ZEND_UNREACHABLE();
}
}
if (needs_slow_path) {
|.cold_code
|7:
| SET_EX_OPLINE opline, r0
| // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value));
| LOAD_ADDR FCARG2a, name
|.if X64
| LOAD_ZVAL_ADDR CARG3, val_addr
| mov CARG4, EX->run_time_cache
| add CARG4, (opline+1)->extended_value
|.if X64WIN
| LOAD_ADDR r0, binary_op
| mov aword A5, r0
|.else
| LOAD_ADDR CARG5, binary_op
|.endif
|.else
| sub r4, 4
| PUSH_ADDR binary_op, r0
| mov r0, EX->run_time_cache
| add r0, (opline+1)->extended_value
| push r0
| PUSH_ZVAL_ADDR val_addr, r0
|.endif
| EXT_CALL zend_jit_assign_obj_op_helper, r0
|.if not(X64)
| add r4, 4
|.endif
if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
val_info |= MAY_BE_RC1|MAY_BE_RCN;
}
|8:
| // FREE_OP_DATA();
| FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline
| jmp >9
|.code
}
|9:
if (opline->op1_type != IS_UNUSED && !use_this && !op1_indirect) {
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline
}
if (may_throw) {
if (!zend_jit_check_exception(Dst)) {
return 0;
}
}
return 1;
}
static int zend_jit_assign_obj(dasm_State **Dst,
const zend_op *opline,
const zend_op_array *op_array,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
uint32_t op1_info,
zend_jit_addr op1_addr,
uint32_t val_info,
zend_bool op1_indirect,
zend_class_entry *ce,
zend_bool ce_is_instanceof,
zend_bool use_this,
zend_class_entry *trace_ce,
int may_throw)
{
zval *member;
zend_string *name;
zend_property_info *prop_info;
zend_jit_addr val_addr = OP1_DATA_ADDR();
zend_jit_addr res_addr = 0;
zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
zend_jit_addr prop_addr;
zend_bool needs_slow_path = 0;
zend_bool needs_val_dtor = 0;
if (RETURN_VALUE_USED(opline)) {
res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
}
ZEND_ASSERT(opline->op2_type == IS_CONST);
ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
member = RT_CONSTANT(opline, opline->op2);
ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0');
name = Z_STR_P(member);
prop_info = zend_get_known_property_info(ce, name, opline->op1_type == IS_UNUSED, op_array->filename);
if (opline->op1_type == IS_UNUSED || use_this) {
| GET_ZVAL_PTR FCARG1a, this_addr
} else {
if (opline->op1_type == IS_VAR
&& (op1_info & MAY_BE_INDIRECT)
&& Z_REG(op1_addr) == ZREG_FP) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
| IF_NOT_Z_TYPE FCARG1a, IS_INDIRECT, >1
| GET_Z_PTR FCARG1a, FCARG1a
|1:
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
}
if (op1_info & MAY_BE_REF) {
if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
| ZVAL_DEREF FCARG1a, op1_info
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
}
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1
|.cold_code
|1:
| SET_EX_OPLINE opline, r0
if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
| LOAD_ADDR FCARG2a, ZSTR_VAL(name)
| EXT_CALL zend_jit_invalid_property_assign, r0
if (RETURN_VALUE_USED(opline)) {
| SET_ZVAL_TYPE_INFO res_addr, IS_NULL
}
if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR))
&& (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
needs_val_dtor = 1;
| jmp >7
} else {
| jmp >9
}
|.code
}
}
| GET_ZVAL_PTR FCARG1a, op1_addr
}
if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
prop_info = zend_get_known_property_info(trace_ce, name, opline->op1_type == IS_UNUSED, op_array->filename);
if (prop_info) {
ce = trace_ce;
ce_is_instanceof = 0;
if (!(op1_info & MAY_BE_CLASS_GUARD)) {
if (!zend_jit_class_guard(Dst, opline, trace_ce)) {
return 0;
}
if (ssa->var_info && ssa_op->op1_use >= 0) {
ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op1_use].ce = ce;
ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof;
}
if (ssa->var_info && ssa_op->op1_def >= 0) {
ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op1_def].ce = ce;
ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof;
}
}
}
}
if (!prop_info) {
needs_slow_path = 1;
| mov r0, EX->run_time_cache
| mov r2, aword [r0 + opline->extended_value]
| cmp r2, aword [FCARG1a + offsetof(zend_object, ce)]
| jne >5
if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
| mov FCARG2a, aword [r0 + (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2]
}
| mov r0, aword [r0 + opline->extended_value + sizeof(void*)]
| test r0, r0
| jl >5
| IF_TYPE byte [FCARG1a + r0 + 8], IS_UNDEF, >5
| add FCARG1a, r0
prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
| test FCARG2a, FCARG2a
| jnz >1
|.cold_code
|1:
| // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC);
| SET_EX_OPLINE opline, r0
|.if X64
| LOAD_ZVAL_ADDR CARG3, val_addr
if (RETURN_VALUE_USED(opline)) {
| LOAD_ZVAL_ADDR CARG4, res_addr
} else {
| xor CARG4, CARG4
}
|.else
| sub r4, 8
if (RETURN_VALUE_USED(opline)) {
| PUSH_ZVAL_ADDR res_addr, r0
} else {
| push 0
}
| PUSH_ZVAL_ADDR val_addr, r0
|.endif
| EXT_CALL zend_jit_assign_to_typed_prop, r0
|.if not(X64)
| add r4, 8
|.endif
if ((opline+1)->op1_type == IS_CONST) {
| // TODO: ???
| // if (Z_TYPE_P(value) == orig_type) {
| // CACHE_PTR_EX(cache_slot + 2, NULL);
}
if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR))
&& (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| jmp >7
} else {
| jmp >9
}
|.code
}
} else {
prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, prop_info->offset);
if (!ce || ce_is_instanceof || !(ce->ce_flags & ZEND_ACC_IMMUTABLE) || ce->__get || ce->__set) {
// Undefined property with magic __get()/__set()
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, &exit_addr
} else {
| IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, >5
needs_slow_path = 1;
}
}
if (ZEND_TYPE_IS_SET(prop_info->type)) {
uint32_t info = val_info;
| // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC);
| SET_EX_OPLINE opline, r0
if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) {
| LOAD_ADDR FCARG2a, prop_info
} else {
int prop_info_offset =
(((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*));
| mov r0, aword [FCARG1a + offsetof(zend_object, ce)]
| mov r0, aword [r0 + offsetof(zend_class_entry, properties_info_table)]
| mov FCARG2a, aword[r0 + prop_info_offset]
}
| LOAD_ZVAL_ADDR FCARG1a, prop_addr
|.if X64
| LOAD_ZVAL_ADDR CARG3, val_addr
if (RETURN_VALUE_USED(opline)) {
| LOAD_ZVAL_ADDR CARG4, res_addr
} else {
| xor CARG4, CARG4
}
|.else
| sub r4, 8
if (RETURN_VALUE_USED(opline)) {
| PUSH_ZVAL_ADDR res_addr, r0
} else {
| push 0
}
| PUSH_ZVAL_ADDR val_addr, r0
|.endif
| EXT_CALL zend_jit_assign_to_typed_prop, r0
|.if not(X64)
| add r4, 8
|.endif
if (info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
info |= MAY_BE_RC1|MAY_BE_RCN;
}
| FREE_OP (opline+1)->op1_type, (opline+1)->op1, info, 0, opline
}
}
if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) {
// value = zend_assign_to_variable(property_val, value, OP_DATA_TYPE, EX_USES_STRICT_TYPES());
if (opline->result_type == IS_UNUSED) {
if (!zend_jit_assign_to_variable_call(Dst, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) {
return 0;
}
} else {
if (!zend_jit_assign_to_variable(Dst, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) {
return 0;
}
}
}
if (needs_slow_path) {
|.cold_code
|5:
| SET_EX_OPLINE opline, r0
| // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value));
| LOAD_ADDR FCARG2a, name
|.if X64
| LOAD_ZVAL_ADDR CARG3, val_addr
| mov CARG4, EX->run_time_cache
| add CARG4, opline->extended_value
if (RETURN_VALUE_USED(opline)) {
|.if X64WIN
| LOAD_ZVAL_ADDR r0, res_addr
| mov aword A5, r0
|.else
| LOAD_ZVAL_ADDR CARG5, res_addr
|.endif
} else {
|.if X64WIN
| mov aword A5, 0
|.else
| xor CARG5, CARG5
|.endif
}
|.else
| sub r4, 4
if (RETURN_VALUE_USED(opline)) {
| PUSH_ZVAL_ADDR res_addr, r0
} else {
| push 0
}
| mov r0, EX->run_time_cache
| add r0, opline->extended_value
| push r0
| PUSH_ZVAL_ADDR val_addr, r0
|.endif
| EXT_CALL zend_jit_assign_obj_helper, r0
|.if not(X64)
| add r4, 4
|.endif
if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
val_info |= MAY_BE_RC1|MAY_BE_RCN;
}
|7:
| // FREE_OP_DATA();
| FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline
| jmp >9
|.code
} else if (needs_val_dtor) {
|.cold_code
|7:
| // FREE_OP_DATA();
| FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline
| jmp >9
|.code
}
|9:
if (opline->op1_type != IS_UNUSED && !use_this && !op1_indirect) {
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline
}
if (may_throw) {
if (!zend_jit_check_exception(Dst)) {
return 0;
}
}
return 1;
}
static int zend_jit_free(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, int may_throw)
{
zend_jit_addr op1_addr = OP1_ADDR();
if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
if (may_throw) {
| SET_EX_OPLINE opline, r0
}
if (opline->opcode == ZEND_FE_FREE && (op1_info & (MAY_BE_OBJECT|MAY_BE_REF))) {
if (op1_info & MAY_BE_ARRAY) {
| IF_ZVAL_TYPE op1_addr, IS_ARRAY, >7
}
| mov FCARG1d, dword [FP + opline->op1.var + offsetof(zval, u2.fe_iter_idx)]
| cmp FCARG1d, -1
| je >7
| EXT_CALL zend_hash_iterator_del, r0
|7:
}
| ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, opline
if (may_throw) {
if (!zend_jit_check_exception(Dst)) {
return 0;
}
}
}
return 1;
}
static int zend_jit_echo(dasm_State **Dst, const zend_op *opline, uint32_t op1_info)
{
if (opline->op1_type == IS_CONST) {
zval *zv;
size_t len;
zv = RT_CONSTANT(opline, opline->op1);
ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING);
len = Z_STRLEN_P(zv);
if (len > 0) {
const char *str = Z_STRVAL_P(zv);
| SET_EX_OPLINE opline, r0
|.if X64
| LOAD_ADDR CARG1, str
| LOAD_ADDR CARG2, len
| EXT_CALL zend_write, r0
|.else
| mov aword A2, len
| mov aword A1, str
| EXT_CALL zend_write, r0
|.endif
if (!zend_jit_check_exception(Dst)) {
return 0;
}
}
} else {
zend_jit_addr op1_addr = OP1_ADDR();
ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING);
| SET_EX_OPLINE opline, r0
| GET_ZVAL_PTR r0, op1_addr
|.if X64
| lea CARG1, aword [r0 + offsetof(zend_string, val)]
| mov CARG2, aword [r0 + offsetof(zend_string, len)]
| EXT_CALL zend_write, r0
|.else
| add r0, offsetof(zend_string, val)
| mov aword A1, r0
| mov r0, aword [r0 + (offsetof(zend_string, len)-offsetof(zend_string, val))]
| mov aword A2, r0
| EXT_CALL zend_write, r0
|.endif
if (opline->op1_type & (IS_VAR|IS_TMP_VAR)) {
| ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, opline
}
if (!zend_jit_check_exception(Dst)) {
return 0;
}
}
return 1;
}
static int zend_jit_strlen(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr)
{
zend_jit_addr res_addr = RES_ADDR();
if (opline->op1_type == IS_CONST) {
zval *zv;
size_t len;
zv = RT_CONSTANT(opline, opline->op1);
ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING);
len = Z_STRLEN_P(zv);
| SET_ZVAL_LVAL res_addr, len
| SET_ZVAL_TYPE_INFO res_addr, IS_LONG
} else {
ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING);
| GET_ZVAL_PTR r0, op1_addr
| mov r0, aword [r0 + offsetof(zend_string, len)]
| SET_ZVAL_LVAL res_addr, r0
| SET_ZVAL_TYPE_INFO res_addr, IS_LONG
| FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline
}
return 1;
}
static int zend_jit_load_this(dasm_State **Dst, uint32_t var)
{
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
| mov FCARG1a, aword EX->This.value.ptr
| SET_ZVAL_PTR var_addr, FCARG1a
| SET_ZVAL_TYPE_INFO var_addr, IS_OBJECT_EX
| GC_ADDREF FCARG1a
return 1;
}
static int zend_jit_fetch_this(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_bool check_only)
{
if (!op_array->scope || (op_array->fn_flags & ZEND_ACC_STATIC)) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
if (!JIT_G(current_frame) ||
!TRACE_FRAME_IS_THIS_CHECKED(JIT_G(current_frame))) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
| cmp byte EX->This.u1.v.type, IS_OBJECT
| jne &exit_addr
if (JIT_G(current_frame)) {
TRACE_FRAME_SET_THIS_CHECKED(JIT_G(current_frame));
}
}
} else {
| cmp byte EX->This.u1.v.type, IS_OBJECT
| jne >1
|.cold_code
|1:
| SET_EX_OPLINE opline, r0
| jmp ->invalid_this
|.code
}
}
if (!check_only) {
if (!zend_jit_load_this(Dst, opline->result.var)) {
return 0;
}
}
return 1;
}
static int zend_jit_hash_jmp(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, HashTable *jumptable, int default_b, const void *default_label, const zend_op *next_opline, zend_jit_trace_info *trace_info)
{
uint32_t count;
Bucket *p;
const zend_op *target;
int b;
int32_t exit_point;
const void *exit_addr;
| test r0, r0
if (default_label) {
| jz &default_label
} else if (next_opline) {
| jz >3
} else {
| jz =>default_b
}
| LOAD_ADDR FCARG1a, jumptable
| sub r0, aword [FCARG1a + offsetof(HashTable, arData)]
| mov FCARG1a, (sizeof(Bucket) / sizeof(void*))
|.if X64
| cqo
|.else
| cdq
|.endif
| idiv FCARG1a
|.if X64
if (!IS_32BIT(dasm_end)) {
| lea FCARG1a, aword [>4]
| jmp aword [FCARG1a + r0]
} else {
| jmp aword [r0 + >4]
}
|.else
| jmp aword [r0 + >4]
|.endif
|.jmp_table
|.align aword
|4:
if (trace_info) {
trace_info->jmp_table_size += zend_hash_num_elements(jumptable);
}
count = jumptable->nNumUsed;
p = jumptable->arData;
do {
if (Z_TYPE(p->val) == IS_UNDEF) {
if (default_label) {
| .aword &default_label
} else if (next_opline) {
| .aword >3
} else {
| .aword =>default_b
}
} else {
target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val));
if (!next_opline) {
b = ssa->cfg.map[target - op_array->opcodes];
| .aword =>b
} else if (next_opline == target) {
| .aword >3
} else {
exit_point = zend_jit_trace_get_exit_point(target, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
| .aword &exit_addr
}
}
p++;
count--;
} while (count);
|.code
return 1;
}
static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, zend_jit_trace_rec *trace, zend_jit_trace_info *trace_info)
{
HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2));
const zend_op *next_opline = NULL;
if (trace) {
ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END);
ZEND_ASSERT(trace->opline != NULL);
next_opline = trace->opline;
}
if (opline->op1_type == IS_CONST) {
zval *zv = RT_CONSTANT(opline, opline->op1);
zval *jump_zv = NULL;
int b;
if (opline->opcode == ZEND_SWITCH_LONG) {
if (Z_TYPE_P(zv) == IS_LONG) {
jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv));
}
} else if (opline->opcode == ZEND_SWITCH_STRING) {
if (Z_TYPE_P(zv) == IS_STRING) {
jump_zv = zend_hash_find_ex(jumptable, Z_STR_P(zv), 1);
}
} else if (opline->opcode == ZEND_MATCH) {
if (Z_TYPE_P(zv) == IS_LONG) {
jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv));
} else if (Z_TYPE_P(zv) == IS_STRING) {
jump_zv = zend_hash_find_ex(jumptable, Z_STR_P(zv), 1);
}
} else {
ZEND_UNREACHABLE();
}
if (next_opline) {
const zend_op *target;
if (jump_zv != NULL) {
target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv));
} else {
target = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value);
}
ZEND_ASSERT(target == next_opline);
} else {
if (jump_zv != NULL) {
b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)) - op_array->opcodes];
} else {
b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes];
}
| jmp =>b
}
} else {
zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
uint32_t op1_info = OP1_INFO();
zend_jit_addr op1_addr = OP1_ADDR();
const zend_op *default_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value);
const zend_op *target;
int default_b = next_opline ? -1 : ssa->cfg.map[default_opline - op_array->opcodes];
int b;
int32_t exit_point;
const void *fallback_label = NULL;
const void *default_label = NULL;
const void *exit_addr;
if (next_opline) {
if (next_opline != opline + 1) {
exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
fallback_label = zend_jit_trace_get_exit_addr(exit_point);
}
if (next_opline != default_opline) {
exit_point = zend_jit_trace_get_exit_point(default_opline, 0);
default_label = zend_jit_trace_get_exit_addr(exit_point);
}
}
if (opline->opcode == ZEND_SWITCH_LONG) {
if (op1_info & MAY_BE_LONG) {
if (op1_info & MAY_BE_REF) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >1
| GET_ZVAL_LVAL ZREG_FCARG2a, op1_addr
|.cold_code
|1:
| // ZVAL_DEREF(op)
if (fallback_label) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3
}
| GET_ZVAL_PTR FCARG2a, op1_addr
if (fallback_label) {
| IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_LONG, &fallback_label
} else {
| IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_LONG, >3
}
| mov FCARG2a, aword [FCARG2a + offsetof(zend_reference, val.value.lval)]
| jmp >2
|.code
|2:
} else {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
if (fallback_label) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &fallback_label
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3
}
}
| GET_ZVAL_LVAL ZREG_FCARG2a, op1_addr
}
if (HT_IS_PACKED(jumptable)) {
uint32_t count = jumptable->nNumUsed;
Bucket *p = jumptable->arData;
| cmp FCARG2a, jumptable->nNumUsed
if (default_label) {
| jae &default_label
} else if (next_opline) {
| jae >3
} else {
| jae =>default_b
}
|.if X64
if (!IS_32BIT(dasm_end)) {
| lea r0, aword [>4]
| jmp aword [r0 + FCARG2a * 8]
} else {
| jmp aword [FCARG2a * 8 + >4]
}
|.else
| jmp aword [FCARG2a * 4 + >4]
|.endif
|.jmp_table
|.align aword
|4:
if (trace_info) {
trace_info->jmp_table_size += count;
}
p = jumptable->arData;
do {
if (Z_TYPE(p->val) == IS_UNDEF) {
if (default_label) {
| .aword &default_label
} else if (next_opline) {
| .aword >3
} else {
| .aword =>default_b
}
} else {
target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val));
if (!next_opline) {
b = ssa->cfg.map[target - op_array->opcodes];
| .aword =>b
} else if (next_opline == target) {
| .aword >3
} else {
exit_point = zend_jit_trace_get_exit_point(target, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
| .aword &exit_addr
}
}
p++;
count--;
} while (count);
|.code
|3:
} else {
| LOAD_ADDR FCARG1a, jumptable
| EXT_CALL zend_hash_index_find, r0
if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) {
return 0;
}
|3:
}
}
} else if (opline->opcode == ZEND_SWITCH_STRING) {
if (op1_info & MAY_BE_STRING) {
if (op1_info & MAY_BE_REF) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >1
| GET_ZVAL_PTR FCARG2a, op1_addr
|.cold_code
|1:
| // ZVAL_DEREF(op)
if (fallback_label) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3
}
| GET_ZVAL_PTR FCARG2a, op1_addr
if (fallback_label) {
| IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_STRING, &fallback_label
} else {
| IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_STRING, >3
}
| mov FCARG2a, aword [FCARG2a + offsetof(zend_reference, val.value.ptr)]
| jmp >2
|.code
|2:
} else {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_STRING)) {
if (fallback_label) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &fallback_label
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3
}
}
| GET_ZVAL_PTR FCARG2a, op1_addr
}
| LOAD_ADDR FCARG1a, jumptable
| EXT_CALL zend_hash_find, r0
if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) {
return 0;
}
|3:
}
} else if (opline->opcode == ZEND_MATCH) {
if (op1_info & (MAY_BE_LONG|MAY_BE_STRING)) {
if (op1_info & MAY_BE_REF) {
| LOAD_ZVAL_ADDR FCARG2a, op1_addr
| ZVAL_DEREF FCARG2a, op1_info
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2a, 0);
}
| LOAD_ADDR FCARG1a, jumptable
if (op1_info & MAY_BE_LONG) {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
if (op1_info & MAY_BE_STRING) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >5
} else if (op1_info & MAY_BE_UNDEF) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6
} else if (default_label) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &default_label
} else if (next_opline) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, =>default_b
}
}
| GET_ZVAL_LVAL ZREG_FCARG2a, op1_addr
| EXT_CALL zend_hash_index_find, r0
if (op1_info & MAY_BE_STRING) {
| jmp >2
}
}
if (op1_info & MAY_BE_STRING) {
|5:
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_STRING))) {
if (op1_info & MAY_BE_UNDEF) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6
} else if (default_label) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &default_label
} else if (next_opline) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, =>default_b
}
}
| GET_ZVAL_PTR FCARG2a, op1_addr
| EXT_CALL zend_hash_find, r0
}
|2:
if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) {
return 0;
}
}
if (op1_info & MAY_BE_UNDEF) {
|6:
if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_STRING))) {
if (default_label) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, &default_label
} else if (next_opline) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >3
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, =>default_b
}
}
| // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
| SET_EX_OPLINE opline, r0
| mov FCARG1d, opline->op1.var
| EXT_CALL zend_jit_undefined_op_helper, r0
if (!zend_jit_check_exception_undef_result(Dst, opline)) {
return 0;
}
}
if (default_label) {
| jmp &default_label
} else if (next_opline) {
| jmp >3
} else {
| jmp =>default_b
}
|3:
} else {
ZEND_UNREACHABLE();
}
}
return 1;
}
static zend_bool zend_jit_verify_return_type(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info)
{
zend_arg_info *arg_info = &op_array->arg_info[-1];
ZEND_ASSERT(ZEND_TYPE_IS_SET(arg_info->type));
zend_jit_addr op1_addr = OP1_ADDR();
zend_bool needs_slow_check = 1;
zend_bool slow_check_in_cold = 1;
uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY;
if (type_mask == 0) {
slow_check_in_cold = 0;
} else {
if (((op1_info & MAY_BE_ANY) & type_mask) == 0) {
slow_check_in_cold = 0;
} else if (((op1_info & MAY_BE_ANY) | type_mask) == type_mask) {
needs_slow_check = 0;
} else if (is_power_of_two(type_mask)) {
uint32_t type_code = concrete_type(type_mask);
| IF_NOT_ZVAL_TYPE op1_addr, type_code, >6
} else {
| mov edx, 1
| GET_ZVAL_TYPE cl, op1_addr
| shl edx, cl
| test edx, type_mask
| je >6
}
}
if (needs_slow_check) {
if (slow_check_in_cold) {
|.cold_code
|6:
}
| SET_EX_OPLINE opline, r1
if (op1_info & MAY_BE_UNDEF) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >7
| mov FCARG1a, opline->op1.var
| EXT_CALL zend_jit_undefined_op_helper, FCARG2a
| test r0, r0
| jz ->exception_handler
| LOAD_ADDR_ZTS FCARG1a, executor_globals, uninitialized_zval
| jmp >8
}
|7:
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
|8:
| mov FCARG2a, EX->func
|.if X64
| LOAD_ADDR CARG3, (ptrdiff_t)arg_info
| mov r0, EX->run_time_cache
| lea CARG4, aword [r0+opline->op2.num]
| EXT_CALL zend_jit_verify_return_slow, r0
|.else
| sub r4, 8
| mov r0, EX->run_time_cache
| add r0, opline->op2.num
| push r0
| push (ptrdiff_t)arg_info
| EXT_CALL zend_jit_verify_return_slow, r0
| add r4, 8
|.endif
if (!zend_jit_check_exception(Dst)) {
return 0;
}
if (slow_check_in_cold) {
| jmp >9
|.code
}
}
|9:
return 1;
}
static int zend_jit_isset_isempty_cv(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
// TODO: support for empty() ???
ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY));
if (op1_info & MAY_BE_REF) {
if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
}
| ZVAL_DEREF FCARG1a, op1_info
|1:
}
if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) {
if (exit_addr) {
ZEND_ASSERT(smart_branch_opcode == ZEND_JMPZ);
} else if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPNZ) {
| jmp =>target_label
} else if (smart_branch_opcode == ZEND_JMPZNZ) {
| jmp =>target_label2
}
} else {
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
}
} else if (!(op1_info & (MAY_BE_ANY - MAY_BE_NULL))) {
if (exit_addr) {
ZEND_ASSERT(smart_branch_opcode == ZEND_JMPNZ);
} else if (smart_branch_opcode) {
if (smart_branch_opcode != ZEND_JMPNZ) {
| jmp =>target_label
}
} else {
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
}
} else {
ZEND_ASSERT(Z_MODE(op1_addr) == IS_MEM_ZVAL);
| cmp byte [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)+offsetof(zval, u1.v.type)], IS_NULL
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
| jg &exit_addr
} else {
| jle &exit_addr
}
} else if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ) {
| jle =>target_label
} else if (smart_branch_opcode == ZEND_JMPNZ) {
| jg =>target_label
} else if (smart_branch_opcode == ZEND_JMPZNZ) {
| jle =>target_label
| jmp =>target_label2
} else {
ZEND_UNREACHABLE();
}
} else {
| setg al
| movzx eax, al
| lea eax, [eax + IS_FALSE]
| SET_ZVAL_TYPE_INFO res_addr, eax
}
}
return 1;
}
static int zend_jit_fe_reset(dasm_State **Dst, const zend_op *opline, uint32_t op1_info)
{
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
if (opline->op1_type == IS_CONST) {
zval *zv = RT_CONSTANT(opline, opline->op1);
| ZVAL_COPY_CONST res_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_R0
if (Z_REFCOUNTED_P(zv)) {
| ADDREF_CONST zv, r0
}
} else {
zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
| // ZVAL_COPY(res, value);
| ZVAL_COPY_VALUE res_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_FCARG1a
if (opline->op1_type == IS_CV) {
| TRY_ADDREF op1_info, ah, FCARG1a
}
}
| // Z_FE_POS_P(res) = 0;
| mov dword [FP + opline->result.var + offsetof(zval, u2.fe_pos)], 0
return 1;
}
static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, unsigned int target_label, zend_uchar exit_opcode, const void *exit_addr)
{
zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
| // array = EX_VAR(opline->op1.var);
| // fe_ht = Z_ARRVAL_P(array);
| GET_ZVAL_PTR FCARG2a, op1_addr
| // pos = Z_FE_POS_P(array);
| mov FCARG1d, dword [FP + opline->op1.var + offsetof(zval, u2.fe_pos)]
| // p = fe_ht->arData + pos;
|.if X64
|| ZEND_ASSERT(sizeof(Bucket) == 32);
| mov eax, FCARG1d
| shl r0, 5
|.else
| imul r0, FCARG1a, sizeof(Bucket)
|.endif
| add r0, aword [FCARG2a + offsetof(zend_array, arData)]
|1:
| // if (UNEXPECTED(pos >= fe_ht->nNumUsed)) {
| cmp dword [FCARG2a + offsetof(zend_array, nNumUsed)], FCARG1d
| // ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value);
| // ZEND_VM_CONTINUE();
if (exit_addr) {
if (exit_opcode == ZEND_JMP) {
| jbe &exit_addr
} else {
| jbe >3
}
} else {
| jbe =>target_label
}
| // pos++;
| add FCARG1d, 1
| // value_type = Z_TYPE_INFO_P(value);
| // if (EXPECTED(value_type != IS_UNDEF)) {
| IF_Z_TYPE r0, IS_UNDEF, >2
if (!exit_addr || exit_opcode == ZEND_JMP) {
| IF_NOT_Z_TYPE r0, IS_INDIRECT, >3
} else {
| IF_NOT_Z_TYPE r0, IS_INDIRECT, &exit_addr
}
| // value = Z_INDIRECT_P(value);
| GET_Z_PTR FCARG2a, r0
| // value_type = Z_TYPE_INFO_P(value);
| // if (EXPECTED(value_type != IS_UNDEF)) {
if (!exit_addr || exit_opcode == ZEND_JMP) {
| IF_NOT_Z_TYPE FCARG2a, IS_UNDEF, >4
} else {
| IF_NOT_Z_TYPE r0, IS_UNDEF, &exit_addr
}
| GET_ZVAL_PTR FCARG2a, op1_addr // reload
|2:
| // p++;
| add r0, sizeof(Bucket)
| jmp <1
|3:
if (!exit_addr || exit_opcode == ZEND_JMP) {
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2a, 0);
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
uint32_t val_info;
| mov FCARG2a, r0
|4:
| // Z_FE_POS_P(array) = pos + 1;
| mov dword [FP + opline->op1.var + offsetof(zval, u2.fe_pos)], FCARG1d
if (RETURN_VALUE_USED(opline)) {
zend_jit_addr res_addr = RES_ADDR();
if ((op1_info & MAY_BE_ARRAY_KEY_LONG)
&& (op1_info & MAY_BE_ARRAY_KEY_STRING)) {
| // if (!p->key) {
| cmp aword [r0 + offsetof(Bucket, key)], 0
| jz >2
}
if (op1_info & MAY_BE_ARRAY_KEY_STRING) {
| // ZVAL_STR_COPY(EX_VAR(opline->result.var), p->key);
| mov FCARG1a, aword [r0 + offsetof(Bucket, key)]
| SET_ZVAL_PTR res_addr, FCARG1a
| test dword [FCARG1a + offsetof(zend_refcounted, gc.u.type_info)], IS_STR_INTERNED
| jz >1
| SET_ZVAL_TYPE_INFO res_addr, IS_STRING
| jmp >3
|1:
| GC_ADDREF FCARG1a
| SET_ZVAL_TYPE_INFO res_addr, IS_STRING_EX
if (op1_info & MAY_BE_ARRAY_KEY_LONG) {
| jmp >3
|2:
}
}
if (op1_info & MAY_BE_ARRAY_KEY_LONG) {
| // ZVAL_LONG(EX_VAR(opline->result.var), p->h);
| mov FCARG1a, aword [r0 + offsetof(Bucket, h)]
| SET_ZVAL_LVAL res_addr, FCARG1a
| SET_ZVAL_TYPE_INFO res_addr, IS_LONG
}
|3:
}
val_info = ((op1_info & MAY_BE_ARRAY_OF_ANY) >> MAY_BE_ARRAY_SHIFT);
if (val_info & MAY_BE_ARRAY) {
val_info |= MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
}
if (op1_info & MAY_BE_ARRAY_OF_REF) {
val_info |= MAY_BE_REF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_ANY |
MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
} else if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
val_info |= MAY_BE_RC1 | MAY_BE_RCN;
}
if (opline->op2_type == IS_CV) {
| // zend_assign_to_variable(variable_ptr, value, IS_CV, EX_USES_STRICT_TYPES());
if (!zend_jit_assign_to_variable(Dst, opline, var_addr, var_addr, op2_info, -1, IS_CV, val_addr, val_info, 0, 1)) {
return 0;
}
} else {
| // ZVAL_COPY(res, value);
| ZVAL_COPY_VALUE var_addr, -1, val_addr, val_info, ZREG_R0, ZREG_FCARG1a
| TRY_ADDREF val_info, ah, FCARG1a
}
}
return 1;
}
static int zend_jit_fetch_constant(dasm_State **Dst,
const zend_op *opline,
const zend_op_array *op_array,
zend_ssa *ssa,
const zend_ssa_op *ssa_op)
{
zval *zv = RT_CONSTANT(opline, opline->op2) + 1;
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
zend_jit_addr const_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0);
uint32_t res_info = RES_INFO();
| // c = CACHED_PTR(opline->extended_value);
| mov FCARG1a, EX->run_time_cache
| mov r0, aword [FCARG1a + opline->extended_value]
| // if (c != NULL)
| test r0, r0
| jz >9
| // if (!IS_SPECIAL_CACHE_VAL(c))
| test r0, CACHE_SPECIAL
| jnz >9
|8:
if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame)) {
zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
int32_t exit_point;
const void *exit_addr = NULL;
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_R0);
exit_point = zend_jit_trace_get_exit_point(opline+1, 0);
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
res_info &= ~MAY_BE_GUARD;
ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
zend_uchar type = concrete_type(res_info);
if (type < IS_STRING) {
| IF_NOT_ZVAL_TYPE const_addr, type, &exit_addr
} else {
| GET_ZVAL_TYPE_INFO edx, const_addr
| IF_NOT_TYPE dl, type, &exit_addr
}
| ZVAL_COPY_VALUE_V res_addr, -1, const_addr, res_info, ZREG_R0, ZREG_R1
if (type < IS_STRING) {
| SET_ZVAL_TYPE_INFO res_addr, type
} else {
| SET_ZVAL_TYPE_INFO res_addr, edx
| TRY_ADDREF res_info, dh, r1
}
} else {
| // ZVAL_COPY_OR_DUP(EX_VAR(opline->result.var), &c->value); (no dup)
| ZVAL_COPY_VALUE res_addr, MAY_BE_ANY, const_addr, MAY_BE_ANY, ZREG_R0, ZREG_R1
| TRY_ADDREF MAY_BE_ANY, ah, r1
}
|.cold_code
|9:
| // SAVE_OPLINE();
| SET_EX_OPLINE opline, r0
| // zend_quick_get_constant(RT_CONSTANT(opline, opline->op2) + 1, opline->op1.num OPLINE_CC EXECUTE_DATA_CC);
| LOAD_ADDR FCARG1a, zv
| mov FCARG2a, opline->op1.num
| EXT_CALL zend_jit_get_constant, r0
| // ZEND_VM_NEXT_OPCODE_CHECK_EXCEPTION();
| test r0, r0
| jnz <8
| jmp ->exception_handler
|.code
return 1;
}
static int zend_jit_in_array(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
HashTable *ht = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2));
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
ZEND_ASSERT(opline->op1_type != IS_VAR && opline->op1_type != IS_TMP_VAR);
ZEND_ASSERT((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == MAY_BE_STRING);
| // result = zend_hash_find_ex(ht, Z_STR_P(op1), OP1_TYPE == IS_CONST);
| LOAD_ADDR FCARG1a, ht
if (opline->op1_type != IS_CONST) {
| GET_ZVAL_PTR FCARG2a, op1_addr
| EXT_CALL zend_hash_find, r0
} else {
zend_string *str = Z_STR_P(RT_CONSTANT(opline, opline->op1));
| LOAD_ADDR FCARG2a, str
| EXT_CALL _zend_hash_find_known_hash, r0
}
| test r0, r0
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
| jz &exit_addr
} else {
| jnz &exit_addr
}
} else if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ) {
| jz =>target_label
} else if (smart_branch_opcode == ZEND_JMPNZ) {
| jnz =>target_label
} else if (smart_branch_opcode == ZEND_JMPZNZ) {
| jz =>target_label
| jmp =>target_label2
} else {
ZEND_UNREACHABLE();
}
} else {
| setnz al
| movzx eax, al
| lea eax, [eax + IS_FALSE]
| SET_ZVAL_TYPE_INFO res_addr, eax
}
return 1;
}
static zend_bool zend_jit_noref_guard(dasm_State **Dst, const zend_op *opline, zend_jit_addr var_addr)
{
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr
return 1;
}
static zend_bool zend_jit_fetch_reference(dasm_State **Dst, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, zend_bool add_ref_guard, zend_bool add_type_guard)
{
zend_jit_addr var_addr = *var_addr_ptr;
uint32_t var_info = *var_info_ptr;
const void *exit_addr = NULL;
if (add_ref_guard || add_type_guard) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
}
if (add_ref_guard) {
| IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr
}
if (opline->opcode == ZEND_INIT_METHOD_CALL && opline->op1_type == IS_VAR) {
/* Hack: Convert reference to regular value to simplify JIT code for INIT_METHOD_CALL */
if (Z_REG(var_addr) != ZREG_FCARG1a || Z_OFFSET(var_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, var_addr
}
| EXT_CALL zend_jit_unref_helper, r0
} else {
| GET_ZVAL_PTR FCARG1a, var_addr
var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, offsetof(zend_reference, val));
*var_addr_ptr = var_addr;
}
if (var_type != IS_UNKNOWN) {
var_type &= ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED);
}
if (add_type_guard
&& var_type != IS_UNKNOWN
&& (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) {
| IF_NOT_ZVAL_TYPE var_addr, var_type, &exit_addr
ZEND_ASSERT(var_info & (1 << var_type));
if (var_type < IS_STRING) {
var_info = (1 << var_type);
} else if (var_type != IS_ARRAY) {
var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN));
} else {
var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN));
}
*var_info_ptr = var_info;
} else {
var_info &= ~MAY_BE_REF;
*var_info_ptr = var_info;
}
*var_info_ptr |= MAY_BE_GUARD; /* prevent generation of specialized zval dtor */
return 1;
}
static zend_bool zend_jit_fetch_indirect_var(dasm_State **Dst, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, zend_bool add_indirect_guard)
{
zend_jit_addr var_addr = *var_addr_ptr;
uint32_t var_info = *var_info_ptr;
int32_t exit_point;
const void *exit_addr;
if (add_indirect_guard) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_NOT_ZVAL_TYPE var_addr, IS_INDIRECT, &exit_addr
| GET_ZVAL_PTR FCARG1a, var_addr
} else {
/* May be already loaded into FCARG1a or RAX by previus FETCH_OBJ_W/DIM_W */
if (opline->op1_type != IS_VAR ||
(opline-1)->result_type != IS_VAR ||
(opline-1)->result.var != opline->op1.var ||
(opline-1)->op2_type == IS_VAR ||
(opline-1)->op2_type == IS_TMP_VAR) {
| GET_ZVAL_PTR FCARG1a, var_addr
} else if ((opline-1)->opcode == ZEND_FETCH_DIM_W || (opline-1)->opcode == ZEND_FETCH_DIM_RW) {
| mov FCARG1a, r0
}
}
*var_info_ptr &= ~MAY_BE_INDIRECT;
var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
*var_addr_ptr = var_addr;
if (var_type != IS_UNKNOWN) {
var_type &= ~(IS_TRACE_INDIRECT|IS_TRACE_PACKED);
}
if (!(var_type & IS_TRACE_REFERENCE)
&& var_type != IS_UNKNOWN
&& (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) {
exit_point = zend_jit_trace_get_exit_point(opline, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_NOT_Z_TYPE FCARG1a, var_type, &exit_addr
//var_info = zend_jit_trace_type_to_info_ex(var_type, var_info);
ZEND_ASSERT(var_info & (1 << var_type));
if (var_type < IS_STRING) {
var_info = (1 << var_type);
} else if (var_type != IS_ARRAY) {
var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN));
} else {
var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN));
}
*var_info_ptr = var_info;
}
return 1;
}
static zend_bool zend_jit_may_reuse_reg(const zend_op *opline, const zend_ssa_op *ssa_op, zend_ssa *ssa, int def_var, int use_var)
{
if ((ssa->var_info[def_var].type & ~MAY_BE_GUARD) != (ssa->var_info[use_var].type & ~MAY_BE_GUARD)) {
return 0;
}
switch (opline->opcode) {
case ZEND_QM_ASSIGN:
case ZEND_SEND_VAR:
case ZEND_ASSIGN:
case ZEND_PRE_INC:
case ZEND_PRE_DEC:
case ZEND_POST_INC:
case ZEND_POST_DEC:
return 1;
case ZEND_ADD:
case ZEND_SUB:
case ZEND_MUL:
case ZEND_BW_OR:
case ZEND_BW_AND:
case ZEND_BW_XOR:
if (def_var == ssa_op->result_def &&
use_var == ssa_op->op1_use) {
return 1;
}
break;
default:
break;
}
return 0;
}
static zend_bool zend_jit_opline_supports_reg(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *opline, const zend_ssa_op *ssa_op, zend_jit_trace_rec *trace)
{
uint32_t op1_info, op2_info;
switch (opline->opcode) {
case ZEND_SEND_VAR:
case ZEND_SEND_VAL:
case ZEND_SEND_VAL_EX:
return (opline->op2_type != IS_CONST);
case ZEND_QM_ASSIGN:
case ZEND_IS_SMALLER:
case ZEND_IS_SMALLER_OR_EQUAL:
case ZEND_IS_EQUAL:
case ZEND_IS_NOT_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_IS_NOT_IDENTICAL:
case ZEND_CASE:
return 1;
case ZEND_RETURN:
return (op_array->type != ZEND_EVAL_CODE && op_array->function_name);
case ZEND_ASSIGN:
op1_info = OP1_INFO();
op2_info = OP2_INFO();
return
opline->op1_type == IS_CV &&
!(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_RESOURCE|MAY_BE_OBJECT|MAY_BE_REF)) &&
!(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)));
case ZEND_ADD:
case ZEND_SUB:
case ZEND_MUL:
op1_info = OP1_INFO();
op2_info = OP2_INFO();
return !((op1_info | op2_info) & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_DOUBLE)));
case ZEND_BW_OR:
case ZEND_BW_AND:
case ZEND_BW_XOR:
case ZEND_SL:
case ZEND_SR:
case ZEND_MOD:
op1_info = OP1_INFO();
op2_info = OP2_INFO();
return !((op1_info | op2_info) & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - MAY_BE_LONG));
case ZEND_PRE_INC:
case ZEND_PRE_DEC:
case ZEND_POST_INC:
case ZEND_POST_DEC:
op1_info = OP1_INFO();
return opline->op1_type == IS_CV && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - MAY_BE_LONG));
case ZEND_JMPZ:
case ZEND_JMPNZ:
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
if (!ssa->cfg.map) {
return 0;
}
if (opline > op_array->opcodes + ssa->cfg.blocks[ssa->cfg.map[opline-op_array->opcodes]].start &&
((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
return 0;
}
}
/* break missing intentionally */
case ZEND_BOOL:
case ZEND_BOOL_NOT:
case ZEND_JMPZNZ:
case ZEND_JMPZ_EX:
case ZEND_JMPNZ_EX:
return 1;
case ZEND_FETCH_DIM_R:
op1_info = OP1_INFO();
op2_info = OP2_INFO();
if (trace
&& trace->op1_type != IS_UNKNOWN
&& (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED)) == IS_ARRAY) {
op1_info &= ~((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY);
}
return ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) &&
(!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || !(op1_info & MAY_BE_RC1)) &&
(((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) ||
(((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_STRING) &&
(!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & MAY_BE_RC1))));
}
return 0;
}
static zend_bool zend_jit_var_supports_reg(zend_ssa *ssa, int var)
{
if (ssa->vars[var].no_val) {
/* we don't need the value */
return 0;
}
if (!(JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL)) {
/* Disable global register allocation,
* register allocation for SSA variables connected through Phi functions
*/
if (ssa->vars[var].definition_phi) {
return 0;
}
if (ssa->vars[var].phi_use_chain) {
zend_ssa_phi *phi = ssa->vars[var].phi_use_chain;
do {
if (!ssa->vars[phi->ssa_var].no_val) {
return 0;
}
phi = zend_ssa_next_use_phi(ssa, var, phi);
} while (phi);
}
}
if (((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) &&
((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG)) {
/* bad type */
return 0;
}
return 1;
}
static zend_bool zend_jit_may_be_in_reg(const zend_op_array *op_array, zend_ssa *ssa, int var)
{
if (!zend_jit_var_supports_reg(ssa, var)) {
return 0;
}
if (ssa->vars[var].definition >= 0) {
uint32_t def = ssa->vars[var].definition;
if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + def, ssa->ops + def, NULL)) {
return 0;
}
}
if (ssa->vars[var].use_chain >= 0) {
int use = ssa->vars[var].use_chain;
do {
if (!zend_ssa_is_no_val_use(op_array->opcodes + use, ssa->ops + use, var) &&
!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + use, ssa->ops + use, NULL)) {
return 0;
}
use = zend_ssa_next_use(ssa->ops, var, use);
} while (use >= 0);
}
return 1;
}
static zend_bool zend_needs_extra_reg_for_const(const zend_op *opline, zend_uchar op_type, znode_op op)
{
|.if X64
|| if (op_type == IS_CONST) {
|| zval *zv = RT_CONSTANT(opline, op);
|| if (Z_TYPE_P(zv) == IS_DOUBLE && Z_DVAL_P(zv) != 0 && !IS_SIGNED_32BIT(zv)) {
|| return 1;
|| } else if (Z_TYPE_P(zv) == IS_LONG && !IS_SIGNED_32BIT(Z_LVAL_P(zv))) {
|| return 1;
|| }
|| }
|.endif
return 0;
}
static zend_regset zend_jit_get_def_scratch_regset(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, int current_var, zend_bool last_use)
{
uint32_t op1_info, op2_info;
switch (opline->opcode) {
case ZEND_FETCH_DIM_R:
op1_info = OP1_INFO();
op2_info = OP2_INFO();
if (((opline->op1_type & (IS_TMP_VAR|IS_VAR)) &&
(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) ||
((opline->op2_type & (IS_TMP_VAR|IS_VAR)) &&
(op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)))) {
return ZEND_REGSET(ZREG_FCARG1a);
}
break;
default:
break;
}
return ZEND_REGSET_EMPTY;
}
static zend_regset zend_jit_get_scratch_regset(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, int current_var, zend_bool last_use)
{
uint32_t op1_info, op2_info, res_info;
zend_regset regset = ZEND_REGSET_SCRATCH;
switch (opline->opcode) {
case ZEND_NOP:
case ZEND_OP_DATA:
case ZEND_JMP:
case ZEND_RETURN:
regset = ZEND_REGSET_EMPTY;
break;
case ZEND_QM_ASSIGN:
if (ssa_op->op1_def == current_var ||
ssa_op->result_def == current_var) {
regset = ZEND_REGSET_EMPTY;
break;
}
/* break missing intentionally */
case ZEND_SEND_VAL:
case ZEND_SEND_VAL_EX:
if (opline->op2_type == IS_CONST) {
break;
}
if (ssa_op->op1_use == current_var) {
regset = ZEND_REGSET(ZREG_R0);
break;
}
op1_info = OP1_INFO();
if (!(op1_info & MAY_BE_UNDEF)) {
if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) {
regset = ZEND_REGSET(ZREG_XMM0);
} else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) {
regset = ZEND_REGSET(ZREG_R0);
} else {
regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_R0), ZEND_REGSET(ZREG_R2));
}
}
break;
case ZEND_SEND_VAR:
if (opline->op2_type == IS_CONST) {
break;
}
if (ssa_op->op1_use == current_var ||
ssa_op->op1_def == current_var) {
regset = ZEND_REGSET_EMPTY;
break;
}
op1_info = OP1_INFO();
if (!(op1_info & MAY_BE_UNDEF)) {
if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) {
regset = ZEND_REGSET(ZREG_XMM0);
} else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) {
} else {
regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_R0), ZEND_REGSET(ZREG_R2));
if (op1_info & MAY_BE_REF) {
ZEND_REGSET_INCL(regset, ZREG_R1);
}
}
}
break;
case ZEND_ASSIGN:
if (ssa_op->op2_use == current_var ||
ssa_op->op2_def == current_var ||
ssa_op->op1_def == current_var ||
ssa_op->result_def == current_var) {
regset = ZEND_REGSET_EMPTY;
break;
}
op1_info = OP1_INFO();
op2_info = OP2_INFO();
if (opline->op1_type == IS_CV
&& !(op2_info & MAY_BE_UNDEF)
&& !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) {
if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) {
regset = ZEND_REGSET(ZREG_XMM0);
} else if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) {
regset = ZEND_REGSET(ZREG_R0);
} else {
regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_R0), ZEND_REGSET(ZREG_R2));
}
}
break;
case ZEND_PRE_INC:
case ZEND_PRE_DEC:
case ZEND_POST_INC:
case ZEND_POST_DEC:
if (ssa_op->op1_use == current_var ||
ssa_op->op1_def == current_var ||
ssa_op->result_def == current_var) {
regset = ZEND_REGSET_EMPTY;
break;
}
op1_info = OP1_INFO();
if (opline->op1_type == IS_CV
&& (op1_info & MAY_BE_LONG)
&& !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) {
regset = ZEND_REGSET_EMPTY;
if (op1_info & MAY_BE_DOUBLE) {
regset = ZEND_REGSET(ZREG_XMM0);
}
if (opline->result_type != IS_UNUSED && (op1_info & MAY_BE_LONG)) {
ZEND_REGSET_INCL(regset, ZREG_R1);
}
}
break;
case ZEND_ADD:
case ZEND_SUB:
case ZEND_MUL:
op1_info = OP1_INFO();
op2_info = OP2_INFO();
if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) &&
!(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) {
regset = ZEND_REGSET_EMPTY;
if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) {
if (ssa_op->result_def != current_var &&
(ssa_op->op1_use != current_var || !last_use)) {
ZEND_REGSET_INCL(regset, ZREG_R0);
}
res_info = RES_INFO();
if (res_info & MAY_BE_DOUBLE) {
ZEND_REGSET_INCL(regset, ZREG_R0);
ZEND_REGSET_INCL(regset, ZREG_XMM0);
ZEND_REGSET_INCL(regset, ZREG_XMM1);
} else if (res_info & MAY_BE_GUARD) {
ZEND_REGSET_INCL(regset, ZREG_R0);
}
}
if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) {
if (ssa_op->result_def != current_var) {
ZEND_REGSET_INCL(regset, ZREG_XMM0);
}
}
if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) {
if (zend_is_commutative(opline->opcode)) {
if (ssa_op->result_def != current_var) {
ZEND_REGSET_INCL(regset, ZREG_XMM0);
}
} else {
ZEND_REGSET_INCL(regset, ZREG_XMM0);
if (ssa_op->result_def != current_var &&
(ssa_op->op1_use != current_var || !last_use)) {
ZEND_REGSET_INCL(regset, ZREG_XMM1);
}
}
}
if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) {
if (ssa_op->result_def != current_var &&
(ssa_op->op1_use != current_var || !last_use) &&
(!zend_is_commutative(opline->opcode) || ssa_op->op2_use != current_var || !last_use)) {
ZEND_REGSET_INCL(regset, ZREG_XMM0);
}
}
if (zend_needs_extra_reg_for_const(opline, opline->op1_type, opline->op1) ||
zend_needs_extra_reg_for_const(opline, opline->op2_type, opline->op2)) {
if (!ZEND_REGSET_IN(regset, ZREG_R0)) {
ZEND_REGSET_INCL(regset, ZREG_R0);
} else {
ZEND_REGSET_INCL(regset, ZREG_R1);
}
}
}
break;
case ZEND_BW_OR:
case ZEND_BW_AND:
case ZEND_BW_XOR:
op1_info = OP1_INFO();
op2_info = OP2_INFO();
if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) &&
!(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) {
regset = ZEND_REGSET_EMPTY;
if (ssa_op->result_def != current_var &&
(ssa_op->op1_use != current_var || !last_use)) {
ZEND_REGSET_INCL(regset, ZREG_R0);
}
if (zend_needs_extra_reg_for_const(opline, opline->op1_type, opline->op1) ||
zend_needs_extra_reg_for_const(opline, opline->op2_type, opline->op2)) {
if (!ZEND_REGSET_IN(regset, ZREG_R0)) {
ZEND_REGSET_INCL(regset, ZREG_R0);
} else {
ZEND_REGSET_INCL(regset, ZREG_R1);
}
}
}
break;
case ZEND_SL:
case ZEND_SR:
op1_info = OP1_INFO();
op2_info = OP2_INFO();
if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) &&
!(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) {
regset = ZEND_REGSET_EMPTY;
if (ssa_op->result_def != current_var &&
(ssa_op->op1_use != current_var || !last_use)) {
ZEND_REGSET_INCL(regset, ZREG_R0);
}
if (opline->op2_type != IS_CONST && ssa_op->op2_use != current_var) {
ZEND_REGSET_INCL(regset, ZREG_R1);
}
}
break;
case ZEND_MOD:
op1_info = OP1_INFO();
op2_info = OP2_INFO();
if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) &&
!(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) {
regset = ZEND_REGSET_EMPTY;
if (opline->op2_type == IS_CONST &&
opline->op1_type != IS_CONST &&
Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG &&
zend_long_is_power_of_two(Z_LVAL_P(RT_CONSTANT(opline, opline->op2))) &&
OP1_HAS_RANGE() &&
OP1_MIN_RANGE() >= 0) {
if (ssa_op->result_def != current_var &&
(ssa_op->op1_use != current_var || !last_use)) {
ZEND_REGSET_INCL(regset, ZREG_R0);
}
if (sizeof(void*) == 8
&& !IS_SIGNED_32BIT(Z_LVAL_P(RT_CONSTANT(opline, opline->op2)) - 1)) {
if (!ZEND_REGSET_IN(regset, ZREG_R0)) {
ZEND_REGSET_INCL(regset, ZREG_R0);
} else {
ZEND_REGSET_INCL(regset, ZREG_R1);
}
}
} else {
ZEND_REGSET_INCL(regset, ZREG_R0);
ZEND_REGSET_INCL(regset, ZREG_R2);
if (opline->op2_type == IS_CONST) {
ZEND_REGSET_INCL(regset, ZREG_R1);
}
}
}
break;
case ZEND_IS_SMALLER:
case ZEND_IS_SMALLER_OR_EQUAL:
case ZEND_IS_EQUAL:
case ZEND_IS_NOT_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_IS_NOT_IDENTICAL:
case ZEND_CASE:
op1_info = OP1_INFO();
op2_info = OP2_INFO();
if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) &&
!(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) {
regset = ZEND_REGSET_EMPTY;
if (!(opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ))) {
ZEND_REGSET_INCL(regset, ZREG_R0);
}
if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) &&
opline->op1_type != IS_CONST && opline->op2_type != IS_CONST) {
if (ssa_op->op1_use != current_var &&
ssa_op->op2_use != current_var) {
ZEND_REGSET_INCL(regset, ZREG_R0);
}
}
if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) {
ZEND_REGSET_INCL(regset, ZREG_XMM0);
}
if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) {
ZEND_REGSET_INCL(regset, ZREG_XMM0);
}
if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) {
if (ssa_op->op1_use != current_var &&
ssa_op->op2_use != current_var) {
ZEND_REGSET_INCL(regset, ZREG_XMM0);
}
}
if (zend_needs_extra_reg_for_const(opline, opline->op1_type, opline->op1) ||
zend_needs_extra_reg_for_const(opline, opline->op2_type, opline->op2)) {
ZEND_REGSET_INCL(regset, ZREG_R0);
}
}
break;
case ZEND_BOOL:
case ZEND_BOOL_NOT:
case ZEND_JMPZ:
case ZEND_JMPNZ:
case ZEND_JMPZNZ:
case ZEND_JMPZ_EX:
case ZEND_JMPNZ_EX:
op1_info = OP1_INFO();
if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)))) {
regset = ZEND_REGSET_EMPTY;
if (op1_info & MAY_BE_DOUBLE) {
ZEND_REGSET_INCL(regset, ZREG_XMM0);
}
if (opline->opcode == ZEND_BOOL ||
opline->opcode == ZEND_BOOL_NOT ||
opline->opcode == ZEND_JMPZ_EX ||
opline->opcode == ZEND_JMPNZ_EX) {
ZEND_REGSET_INCL(regset, ZREG_R0);
}
}
break;
case ZEND_DO_UCALL:
case ZEND_DO_FCALL:
case ZEND_DO_FCALL_BY_NAME:
case ZEND_INCLUDE_OR_EVAL:
case ZEND_GENERATOR_CREATE:
case ZEND_YIELD:
case ZEND_YIELD_FROM:
regset = ZEND_REGSET_UNION(ZEND_REGSET_GP, ZEND_REGSET_FP);
break;
default:
break;
}
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
if (ssa_op == ssa->ops
&& JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].op == ZEND_JIT_TRACE_INIT_CALL
&& (JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
ZEND_REGSET_INCL(regset, ZREG_R0);
ZEND_REGSET_INCL(regset, ZREG_R1);
}
}
/* %r0 is used to check EG(vm_interrupt) */
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
if (ssa_op == ssa->ops
&& (JIT_G(current_trace)->stop == ZEND_JIT_TRACE_STOP_LOOP ||
JIT_G(current_trace)->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL)) {
#if ZTS
ZEND_REGSET_INCL(regset, ZREG_R0);
#else
if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(vm_interrupt)))) {
ZEND_REGSET_INCL(regset, ZREG_R0);
}
#endif
}
} else {
uint32_t b = ssa->cfg.map[ssa_op - ssa->ops];
if ((ssa->cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) != 0
&& ssa->cfg.blocks[b].start == ssa_op - ssa->ops) {
#if ZTS
ZEND_REGSET_INCL(regset, ZREG_R0);
#else
if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(vm_interrupt)))) {
ZEND_REGSET_INCL(regset, ZREG_R0);
}
#endif
}
}
return regset;
}
#if defined(__clang__)
# pragma clang diagnostic pop
#endif
/*
* Local variables:
* tab-width: 4
* c-basic-offset: 4
* indent-tabs-mode: t
* End:
*/
Reference in New Issue View Git Blame Copy Permalink