/* * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under the BSD-style license found in the * LICENSE file in the root directory of this source tree. */ #include "./FbgemmFP16UKernelsAvx512.h" #include "./InlineAsmDefines.h" namespace fbgemm { void NOINLINE gemmkernel_1x2_Avx512_fp16_fA0fB0fC0(GemmParamsFP16* gp) { asm volatile( #if FBGEMM_USE_CLANG_INTEL_SYNTAX_ASM_HACK "mov %[gp], %%r14\t\n" ".intel_syntax noprefix\t\n" #else "mov r14, %[gp]\t\n" #endif // Copy parameters // k "mov r8, [r14 + 0]\t\n" "dec r8\t\n" // A "mov r9, [r14 + 8]\t\n" // B "mov r10, [r14 + 16]\t\n" // beta "lea r15, [r14 + 24]\t\n" // C "mov r12, [r14 + 32]\t\n" // ldc "mov r13, [r14 + 40]\t\n" // b_block_cols "mov rdi, [r14 + 48]\t\n" // b_block_size "mov rsi, [r14 + 56]\t\n" // Make copies of A and C "mov rax, r9\t\n" "mov rcx, r12\t\n" "xor ebx, ebx\t\n" "loop_outter%=:\t\n" "mov r14, r8\t\n" "vbroadcastss zmm31,DWORD PTR [r15]\t\n" "vcvtph2ps zmm3,YMMWORD PTR [r10 + 0]\t\n" "vcvtph2ps zmm4,YMMWORD PTR [r10 + 32]\t\n" "vxorps xmm0, xmm0, xmm0\t\n" "vcomiss xmm31, xmm0\t\n" "jz zero_regs%=\t\n" // Setup values with beta multiplication "vmulps zmm0, zmm31, [r12 + 0]\t\n" "vmulps zmm1, zmm31, [r12 + 64]\t\n" "test r14,r14\t\n" "jz skip_preload%=\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "skip_preload%=:\t\n" "vbroadcastss zmm2,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm3,zmm2\t\n" "vfmadd231ps zmm1,zmm4,zmm2\t\n" "test r14,r14\t\n" "jnz next_inner%=\t\n" "add r10,64\t\n" "jmp dump_C%=\t\n" "zero_regs%=:\t\n" "test r14,r14\t\n" "jz skip_preload_b_zero%=\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "skip_preload_b_zero%=:\t\n" "vbroadcastss zmm2,DWORD PTR [r9+0]\t\n" "vmulps zmm0,zmm3,zmm2\t\n" "vmulps zmm1,zmm4,zmm2\t\n" "test r14,r14\t\n" "jnz next_inner%=\t\n" "add r10,64\t\n" "jmp dump_C%=\t\n" "loop_inner%=:\t\n" "vmovaps zmm3,zmm31\t\n" "vcvtph2ps zmm4,YMMWORD PTR [r10 + 32]\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "vbroadcastss zmm2,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm3,zmm2\t\n" "vfmadd231ps zmm1,zmm4,zmm2\t\n" "next_inner%=:\t\n" "add r9,4\t\n" "add r10,64\t\n" "dec r14\t\n" "jnz loop_inner%=\t\n" "vmovaps zmm3,zmm31\t\n" "vcvtph2ps zmm4,YMMWORD PTR [r10 + 32]\t\n" "vbroadcastss zmm2,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm3,zmm2\t\n" "vfmadd231ps zmm1,zmm4,zmm2\t\n" "add r9,4\t\n" "add r10,64\t\n" // Dump C "dump_C%=:\t\n" "vmovups zmmword PTR [r12 + 0], zmm0\t\n" "vmovups zmmword PTR [r12 + 64], zmm1\t\n" // next outer iteration "add rcx, 128\t\n" "mov r12, rcx\t\n" "mov r9, rax\t\n" "inc rbx\t\n" "cmp rbx, rdi\t\n" "jl loop_outter%=\t\n" : : [gp] "rm"(gp) : "r8", "r9", "r10", "r11", "r13", "r14", "rax", "rcx", "rsi", "rdi", "rbx", "r12", "r15", "memory"); } void NOINLINE gemmkernel_2x2_Avx512_fp16_fA0fB0fC0(GemmParamsFP16* gp) { asm volatile( #if FBGEMM_USE_CLANG_INTEL_SYNTAX_ASM_HACK "mov %[gp], %%r14\t\n" ".intel_syntax noprefix\t\n" #else "mov r14, %[gp]\t\n" #endif // Copy parameters // k "mov r8, [r14 + 0]\t\n" "dec r8\t\n" // A "mov r9, [r14 + 8]\t\n" // B "mov r10, [r14 + 16]\t\n" // beta "lea r15, [r14 + 24]\t\n" // C "mov r12, [r14 + 32]\t\n" // ldc "mov r13, [r14 + 40]\t\n" // b_block_cols "mov rdi, [r14 + 48]\t\n" // b_block_size "mov rsi, [r14 + 56]\t\n" // Make copies of A and C "mov rax, r9\t\n" "mov rcx, r12\t\n" "xor ebx, ebx\t\n" "loop_outter%=:\t\n" "mov r14, r8\t\n" "vbroadcastss zmm31,DWORD PTR [r15]\t\n" "vcvtph2ps zmm5,YMMWORD PTR [r10 + 0]\t\n" "vcvtph2ps zmm6,YMMWORD PTR [r10 + 32]\t\n" "vxorps xmm0, xmm0, xmm0\t\n" "vcomiss xmm31, xmm0\t\n" "jz zero_regs%=\t\n" // Setup values with beta multiplication "vmulps zmm0, zmm31, [r12 + 0]\t\n" "vmulps zmm1, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm2, zmm31, [r12 + 0]\t\n" "vmulps zmm3, zmm31, [r12 + 64]\t\n" "test r14,r14\t\n" "jz skip_preload%=\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "skip_preload%=:\t\n" "vbroadcastss zmm4,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm5,zmm4\t\n" "vfmadd231ps zmm1,zmm6,zmm4\t\n" "vbroadcastss zmm4,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm5,zmm4\t\n" "vfmadd231ps zmm3,zmm6,zmm4\t\n" "mov r12, rcx\t\n" "test r14,r14\t\n" "jnz next_inner%=\t\n" "add r10,64\t\n" "jmp dump_C%=\t\n" "zero_regs%=:\t\n" "test r14,r14\t\n" "jz skip_preload_b_zero%=\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "skip_preload_b_zero%=:\t\n" "vbroadcastss zmm4,DWORD PTR [r9+0]\t\n" "vmulps zmm0,zmm5,zmm4\t\n" "vmulps zmm1,zmm6,zmm4\t\n" "add r12, r13\t\n" "vbroadcastss zmm4,DWORD PTR [r9+4]\t\n" "vmulps zmm2,zmm5,zmm4\t\n" "vmulps zmm3,zmm6,zmm4\t\n" "mov r12, rcx\t\n" "test r14,r14\t\n" "jnz next_inner%=\t\n" "add r10,64\t\n" "jmp dump_C%=\t\n" "loop_inner%=:\t\n" "vmovaps zmm5,zmm31\t\n" "vcvtph2ps zmm6,YMMWORD PTR [r10 + 32]\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "vbroadcastss zmm4,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm5,zmm4\t\n" "vfmadd231ps zmm1,zmm6,zmm4\t\n" "vbroadcastss zmm4,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm5,zmm4\t\n" "vfmadd231ps zmm3,zmm6,zmm4\t\n" "next_inner%=:\t\n" "add r9,8\t\n" "add r10,64\t\n" "dec r14\t\n" "jnz loop_inner%=\t\n" "vmovaps zmm5,zmm31\t\n" "vcvtph2ps zmm6,YMMWORD PTR [r10 + 32]\t\n" "vbroadcastss zmm4,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm5,zmm4\t\n" "vfmadd231ps zmm1,zmm6,zmm4\t\n" "vbroadcastss zmm4,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm5,zmm4\t\n" "vfmadd231ps zmm3,zmm6,zmm4\t\n" "add r9,8\t\n" "add r10,64\t\n" // Dump C "dump_C%=:\t\n" "vmovups zmmword PTR [r12 + 0], zmm0\t\n" "vmovups zmmword PTR [r12 + 64], zmm1\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm2\t\n" "vmovups zmmword PTR [r12 + 64], zmm3\t\n" // next outer iteration "add rcx, 128\t\n" "mov r12, rcx\t\n" "mov r9, rax\t\n" "inc rbx\t\n" "cmp rbx, rdi\t\n" "jl loop_outter%=\t\n" : : [gp] "rm"(gp) : "r8", "r9", "r10", "r11", "r13", "r14", "rax", "rcx", "rsi", "rdi", "rbx", "r12", "r15", "memory"); } void NOINLINE gemmkernel_3x2_Avx512_fp16_fA0fB0fC0(GemmParamsFP16* gp) { asm volatile( #if FBGEMM_USE_CLANG_INTEL_SYNTAX_ASM_HACK "mov %[gp], %%r14\t\n" ".intel_syntax noprefix\t\n" #else "mov r14, %[gp]\t\n" #endif // Copy parameters // k "mov r8, [r14 + 0]\t\n" "dec r8\t\n" // A "mov r9, [r14 + 8]\t\n" // B "mov r10, [r14 + 16]\t\n" // beta "lea r15, [r14 + 24]\t\n" // C "mov r12, [r14 + 32]\t\n" // ldc "mov r13, [r14 + 40]\t\n" // b_block_cols "mov rdi, [r14 + 48]\t\n" // b_block_size "mov rsi, [r14 + 56]\t\n" // Make copies of A and C "mov rax, r9\t\n" "mov rcx, r12\t\n" "xor ebx, ebx\t\n" "loop_outter%=:\t\n" "mov r14, r8\t\n" "vbroadcastss zmm31,DWORD PTR [r15]\t\n" "vcvtph2ps zmm7,YMMWORD PTR [r10 + 0]\t\n" "vcvtph2ps zmm8,YMMWORD PTR [r10 + 32]\t\n" "vxorps xmm0, xmm0, xmm0\t\n" "vcomiss xmm31, xmm0\t\n" "jz zero_regs%=\t\n" // Setup values with beta multiplication "vmulps zmm0, zmm31, [r12 + 0]\t\n" "vmulps zmm1, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm2, zmm31, [r12 + 0]\t\n" "vmulps zmm3, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm4, zmm31, [r12 + 0]\t\n" "vmulps zmm5, zmm31, [r12 + 64]\t\n" "test r14,r14\t\n" "jz skip_preload%=\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "skip_preload%=:\t\n" "vbroadcastss zmm6,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm7,zmm6\t\n" "vfmadd231ps zmm1,zmm8,zmm6\t\n" "vbroadcastss zmm6,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm7,zmm6\t\n" "vfmadd231ps zmm3,zmm8,zmm6\t\n" "vbroadcastss zmm6,DWORD PTR [r9+8]\t\n" "vfmadd231ps zmm4,zmm7,zmm6\t\n" "vfmadd231ps zmm5,zmm8,zmm6\t\n" "mov r12, rcx\t\n" "test r14,r14\t\n" "jnz next_inner%=\t\n" "add r10,64\t\n" "jmp dump_C%=\t\n" "zero_regs%=:\t\n" "test r14,r14\t\n" "jz skip_preload_b_zero%=\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "skip_preload_b_zero%=:\t\n" "vbroadcastss zmm6,DWORD PTR [r9+0]\t\n" "vmulps zmm0,zmm7,zmm6\t\n" "vmulps zmm1,zmm8,zmm6\t\n" "add r12, r13\t\n" "vbroadcastss zmm6,DWORD PTR [r9+4]\t\n" "vmulps zmm2,zmm7,zmm6\t\n" "vmulps zmm3,zmm8,zmm6\t\n" "add r12, r13\t\n" "vbroadcastss zmm6,DWORD PTR [r9+8]\t\n" "vmulps zmm4,zmm7,zmm6\t\n" "vmulps zmm5,zmm8,zmm6\t\n" "mov r12, rcx\t\n" "test r14,r14\t\n" "jnz next_inner%=\t\n" "add r10,64\t\n" "jmp dump_C%=\t\n" "loop_inner%=:\t\n" "vmovaps zmm7,zmm31\t\n" "vcvtph2ps zmm8,YMMWORD PTR [r10 + 32]\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "vbroadcastss zmm6,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm7,zmm6\t\n" "vfmadd231ps zmm1,zmm8,zmm6\t\n" "vbroadcastss zmm6,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm7,zmm6\t\n" "vfmadd231ps zmm3,zmm8,zmm6\t\n" "vbroadcastss zmm6,DWORD PTR [r9+8]\t\n" "vfmadd231ps zmm4,zmm7,zmm6\t\n" "vfmadd231ps zmm5,zmm8,zmm6\t\n" "next_inner%=:\t\n" "add r9,12\t\n" "add r10,64\t\n" "dec r14\t\n" "jnz loop_inner%=\t\n" "vmovaps zmm7,zmm31\t\n" "vcvtph2ps zmm8,YMMWORD PTR [r10 + 32]\t\n" "vbroadcastss zmm6,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm7,zmm6\t\n" "vfmadd231ps zmm1,zmm8,zmm6\t\n" "vbroadcastss zmm6,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm7,zmm6\t\n" "vfmadd231ps zmm3,zmm8,zmm6\t\n" "vbroadcastss zmm6,DWORD PTR [r9+8]\t\n" "vfmadd231ps zmm4,zmm7,zmm6\t\n" "vfmadd231ps zmm5,zmm8,zmm6\t\n" "add r9,12\t\n" "add r10,64\t\n" // Dump C "dump_C%=:\t\n" "vmovups zmmword PTR [r12 + 0], zmm0\t\n" "vmovups zmmword PTR [r12 + 64], zmm1\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm2\t\n" "vmovups zmmword PTR [r12 + 64], zmm3\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm4\t\n" "vmovups zmmword PTR [r12 + 64], zmm5\t\n" // next outer iteration "add rcx, 128\t\n" "mov r12, rcx\t\n" "mov r9, rax\t\n" "inc rbx\t\n" "cmp rbx, rdi\t\n" "jl loop_outter%=\t\n" : : [gp] "rm"(gp) : "r8", "r9", "r10", "r11", "r13", "r14", "rax", "rcx", "rsi", "rdi", "rbx", "r12", "r15", "memory"); } void NOINLINE gemmkernel_4x2_Avx512_fp16_fA0fB0fC0(GemmParamsFP16* gp) { asm volatile( #if FBGEMM_USE_CLANG_INTEL_SYNTAX_ASM_HACK "mov %[gp], %%r14\t\n" ".intel_syntax noprefix\t\n" #else "mov r14, %[gp]\t\n" #endif // Copy parameters // k "mov r8, [r14 + 0]\t\n" "dec r8\t\n" // A "mov r9, [r14 + 8]\t\n" // B "mov r10, [r14 + 16]\t\n" // beta "lea r15, [r14 + 24]\t\n" // C "mov r12, [r14 + 32]\t\n" // ldc "mov r13, [r14 + 40]\t\n" // b_block_cols "mov rdi, [r14 + 48]\t\n" // b_block_size "mov rsi, [r14 + 56]\t\n" // Make copies of A and C "mov rax, r9\t\n" "mov rcx, r12\t\n" "xor ebx, ebx\t\n" "loop_outter%=:\t\n" "mov r14, r8\t\n" "vbroadcastss zmm31,DWORD PTR [r15]\t\n" "vcvtph2ps zmm9,YMMWORD PTR [r10 + 0]\t\n" "vcvtph2ps zmm10,YMMWORD PTR [r10 + 32]\t\n" "vxorps xmm0, xmm0, xmm0\t\n" "vcomiss xmm31, xmm0\t\n" "jz zero_regs%=\t\n" // Setup values with beta multiplication "vmulps zmm0, zmm31, [r12 + 0]\t\n" "vmulps zmm1, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm2, zmm31, [r12 + 0]\t\n" "vmulps zmm3, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm4, zmm31, [r12 + 0]\t\n" "vmulps zmm5, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm6, zmm31, [r12 + 0]\t\n" "vmulps zmm7, zmm31, [r12 + 64]\t\n" "test r14,r14\t\n" "jz skip_preload%=\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "skip_preload%=:\t\n" "vbroadcastss zmm8,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm9,zmm8\t\n" "vfmadd231ps zmm1,zmm10,zmm8\t\n" "vbroadcastss zmm8,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm9,zmm8\t\n" "vfmadd231ps zmm3,zmm10,zmm8\t\n" "vbroadcastss zmm8,DWORD PTR [r9+8]\t\n" "vfmadd231ps zmm4,zmm9,zmm8\t\n" "vfmadd231ps zmm5,zmm10,zmm8\t\n" "vbroadcastss zmm8,DWORD PTR [r9+12]\t\n" "vfmadd231ps zmm6,zmm9,zmm8\t\n" "vfmadd231ps zmm7,zmm10,zmm8\t\n" "mov r12, rcx\t\n" "test r14,r14\t\n" "jnz next_inner%=\t\n" "add r10,64\t\n" "jmp dump_C%=\t\n" "zero_regs%=:\t\n" "test r14,r14\t\n" "jz skip_preload_b_zero%=\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "skip_preload_b_zero%=:\t\n" "vbroadcastss zmm8,DWORD PTR [r9+0]\t\n" "vmulps zmm0,zmm9,zmm8\t\n" "vmulps zmm1,zmm10,zmm8\t\n" "add r12, r13\t\n" "vbroadcastss zmm8,DWORD PTR [r9+4]\t\n" "vmulps zmm2,zmm9,zmm8\t\n" "vmulps zmm3,zmm10,zmm8\t\n" "add r12, r13\t\n" "vbroadcastss zmm8,DWORD PTR [r9+8]\t\n" "vmulps zmm4,zmm9,zmm8\t\n" "vmulps zmm5,zmm10,zmm8\t\n" "add r12, r13\t\n" "vbroadcastss zmm8,DWORD PTR [r9+12]\t\n" "vmulps zmm6,zmm9,zmm8\t\n" "vmulps zmm7,zmm10,zmm8\t\n" "mov r12, rcx\t\n" "test r14,r14\t\n" "jnz next_inner%=\t\n" "add r10,64\t\n" "jmp dump_C%=\t\n" "loop_inner%=:\t\n" "vmovaps zmm9,zmm31\t\n" "vcvtph2ps zmm10,YMMWORD PTR [r10 + 32]\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "vbroadcastss zmm8,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm9,zmm8\t\n" "vfmadd231ps zmm1,zmm10,zmm8\t\n" "vbroadcastss zmm8,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm9,zmm8\t\n" "vfmadd231ps zmm3,zmm10,zmm8\t\n" "vbroadcastss zmm8,DWORD PTR [r9+8]\t\n" "vfmadd231ps zmm4,zmm9,zmm8\t\n" "vfmadd231ps zmm5,zmm10,zmm8\t\n" "vbroadcastss zmm8,DWORD PTR [r9+12]\t\n" "vfmadd231ps zmm6,zmm9,zmm8\t\n" "vfmadd231ps zmm7,zmm10,zmm8\t\n" "next_inner%=:\t\n" "add r9,16\t\n" "add r10,64\t\n" "dec r14\t\n" "jnz loop_inner%=\t\n" "vmovaps zmm9,zmm31\t\n" "vcvtph2ps zmm10,YMMWORD PTR [r10 + 32]\t\n" "vbroadcastss zmm8,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm9,zmm8\t\n" "vfmadd231ps zmm1,zmm10,zmm8\t\n" "vbroadcastss zmm8,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm9,zmm8\t\n" "vfmadd231ps zmm3,zmm10,zmm8\t\n" "vbroadcastss zmm8,DWORD PTR [r9+8]\t\n" "vfmadd231ps zmm4,zmm9,zmm8\t\n" "vfmadd231ps zmm5,zmm10,zmm8\t\n" "vbroadcastss zmm8,DWORD PTR [r9+12]\t\n" "vfmadd231ps zmm6,zmm9,zmm8\t\n" "vfmadd231ps zmm7,zmm10,zmm8\t\n" "add r9,16\t\n" "add r10,64\t\n" // Dump C "dump_C%=:\t\n" "vmovups zmmword PTR [r12 + 0], zmm0\t\n" "vmovups zmmword PTR [r12 + 64], zmm1\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm2\t\n" "vmovups zmmword PTR [r12 + 64], zmm3\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm4\t\n" "vmovups zmmword PTR [r12 + 64], zmm5\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm6\t\n" "vmovups zmmword PTR [r12 + 64], zmm7\t\n" // next outer iteration "add rcx, 128\t\n" "mov r12, rcx\t\n" "mov r9, rax\t\n" "inc rbx\t\n" "cmp rbx, rdi\t\n" "jl loop_outter%=\t\n" : : [gp] "rm"(gp) : "r8", "r9", "r10", "r11", "r13", "r14", "rax", "rcx", "rsi", "rdi", "rbx", "r12", "r15", "memory"); } void NOINLINE gemmkernel_5x2_Avx512_fp16_fA0fB0fC0(GemmParamsFP16* gp) { asm volatile( #if FBGEMM_USE_CLANG_INTEL_SYNTAX_ASM_HACK "mov %[gp], %%r14\t\n" ".intel_syntax noprefix\t\n" #else "mov r14, %[gp]\t\n" #endif // Copy parameters // k "mov r8, [r14 + 0]\t\n" "dec r8\t\n" // A "mov r9, [r14 + 8]\t\n" // B "mov r10, [r14 + 16]\t\n" // beta "lea r15, [r14 + 24]\t\n" // C "mov r12, [r14 + 32]\t\n" // ldc "mov r13, [r14 + 40]\t\n" // b_block_cols "mov rdi, [r14 + 48]\t\n" // b_block_size "mov rsi, [r14 + 56]\t\n" // Make copies of A and C "mov rax, r9\t\n" "mov rcx, r12\t\n" "xor ebx, ebx\t\n" "loop_outter%=:\t\n" "mov r14, r8\t\n" "vbroadcastss zmm31,DWORD PTR [r15]\t\n" "vcvtph2ps zmm11,YMMWORD PTR [r10 + 0]\t\n" "vcvtph2ps zmm12,YMMWORD PTR [r10 + 32]\t\n" "vxorps xmm0, xmm0, xmm0\t\n" "vcomiss xmm31, xmm0\t\n" "jz zero_regs%=\t\n" // Setup values with beta multiplication "vmulps zmm0, zmm31, [r12 + 0]\t\n" "vmulps zmm1, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm2, zmm31, [r12 + 0]\t\n" "vmulps zmm3, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm4, zmm31, [r12 + 0]\t\n" "vmulps zmm5, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm6, zmm31, [r12 + 0]\t\n" "vmulps zmm7, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm8, zmm31, [r12 + 0]\t\n" "vmulps zmm9, zmm31, [r12 + 64]\t\n" "test r14,r14\t\n" "jz skip_preload%=\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "skip_preload%=:\t\n" "vbroadcastss zmm10,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm11,zmm10\t\n" "vfmadd231ps zmm1,zmm12,zmm10\t\n" "vbroadcastss zmm10,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm11,zmm10\t\n" "vfmadd231ps zmm3,zmm12,zmm10\t\n" "vbroadcastss zmm10,DWORD PTR [r9+8]\t\n" "vfmadd231ps zmm4,zmm11,zmm10\t\n" "vfmadd231ps zmm5,zmm12,zmm10\t\n" "vbroadcastss zmm10,DWORD PTR [r9+12]\t\n" "vfmadd231ps zmm6,zmm11,zmm10\t\n" "vfmadd231ps zmm7,zmm12,zmm10\t\n" "vbroadcastss zmm10,DWORD PTR [r9+16]\t\n" "vfmadd231ps zmm8,zmm11,zmm10\t\n" "vfmadd231ps zmm9,zmm12,zmm10\t\n" "mov r12, rcx\t\n" "test r14,r14\t\n" "jnz next_inner%=\t\n" "add r10,64\t\n" "jmp dump_C%=\t\n" "zero_regs%=:\t\n" "test r14,r14\t\n" "jz skip_preload_b_zero%=\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "skip_preload_b_zero%=:\t\n" "vbroadcastss zmm10,DWORD PTR [r9+0]\t\n" "vmulps zmm0,zmm11,zmm10\t\n" "vmulps zmm1,zmm12,zmm10\t\n" "add r12, r13\t\n" "vbroadcastss zmm10,DWORD PTR [r9+4]\t\n" "vmulps zmm2,zmm11,zmm10\t\n" "vmulps zmm3,zmm12,zmm10\t\n" "add r12, r13\t\n" "vbroadcastss zmm10,DWORD PTR [r9+8]\t\n" "vmulps zmm4,zmm11,zmm10\t\n" "vmulps zmm5,zmm12,zmm10\t\n" "add r12, r13\t\n" "vbroadcastss zmm10,DWORD PTR [r9+12]\t\n" "vmulps zmm6,zmm11,zmm10\t\n" "vmulps zmm7,zmm12,zmm10\t\n" "add r12, r13\t\n" "vbroadcastss zmm10,DWORD PTR [r9+16]\t\n" "vmulps zmm8,zmm11,zmm10\t\n" "vmulps zmm9,zmm12,zmm10\t\n" "mov r12, rcx\t\n" "test r14,r14\t\n" "jnz next_inner%=\t\n" "add r10,64\t\n" "jmp dump_C%=\t\n" "loop_inner%=:\t\n" "vmovaps zmm11,zmm31\t\n" "vcvtph2ps zmm12,YMMWORD PTR [r10 + 32]\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "vbroadcastss zmm10,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm11,zmm10\t\n" "vfmadd231ps zmm1,zmm12,zmm10\t\n" "vbroadcastss zmm10,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm11,zmm10\t\n" "vfmadd231ps zmm3,zmm12,zmm10\t\n" "vbroadcastss zmm10,DWORD PTR [r9+8]\t\n" "vfmadd231ps zmm4,zmm11,zmm10\t\n" "vfmadd231ps zmm5,zmm12,zmm10\t\n" "vbroadcastss zmm10,DWORD PTR [r9+12]\t\n" "vfmadd231ps zmm6,zmm11,zmm10\t\n" "vfmadd231ps zmm7,zmm12,zmm10\t\n" "vbroadcastss zmm10,DWORD PTR [r9+16]\t\n" "vfmadd231ps zmm8,zmm11,zmm10\t\n" "vfmadd231ps zmm9,zmm12,zmm10\t\n" "next_inner%=:\t\n" "add r9,20\t\n" "add r10,64\t\n" "dec r14\t\n" "jnz loop_inner%=\t\n" "vmovaps zmm11,zmm31\t\n" "vcvtph2ps zmm12,YMMWORD PTR [r10 + 32]\t\n" "vbroadcastss zmm10,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm11,zmm10\t\n" "vfmadd231ps zmm1,zmm12,zmm10\t\n" "vbroadcastss zmm10,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm11,zmm10\t\n" "vfmadd231ps zmm3,zmm12,zmm10\t\n" "vbroadcastss zmm10,DWORD PTR [r9+8]\t\n" "vfmadd231ps zmm4,zmm11,zmm10\t\n" "vfmadd231ps zmm5,zmm12,zmm10\t\n" "vbroadcastss zmm10,DWORD PTR [r9+12]\t\n" "vfmadd231ps zmm6,zmm11,zmm10\t\n" "vfmadd231ps zmm7,zmm12,zmm10\t\n" "vbroadcastss zmm10,DWORD PTR [r9+16]\t\n" "vfmadd231ps zmm8,zmm11,zmm10\t\n" "vfmadd231ps zmm9,zmm12,zmm10\t\n" "add r9,20\t\n" "add r10,64\t\n" // Dump C "dump_C%=:\t\n" "vmovups zmmword PTR [r12 + 0], zmm0\t\n" "vmovups zmmword PTR [r12 + 64], zmm1\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm2\t\n" "vmovups zmmword PTR [r12 + 64], zmm3\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm4\t\n" "vmovups zmmword PTR [r12 + 64], zmm5\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm6\t\n" "vmovups zmmword PTR [r12 + 64], zmm7\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm8\t\n" "vmovups zmmword PTR [r12 + 64], zmm9\t\n" // next outer iteration "add rcx, 128\t\n" "mov r12, rcx\t\n" "mov r9, rax\t\n" "inc rbx\t\n" "cmp rbx, rdi\t\n" "jl loop_outter%=\t\n" : : [gp] "rm"(gp) : "r8", "r9", "r10", "r11", "r13", "r14", "rax", "rcx", "rsi", "rdi", "rbx", "r12", "r15", "memory"); } void NOINLINE gemmkernel_6x2_Avx512_fp16_fA0fB0fC0(GemmParamsFP16* gp) { asm volatile( #if FBGEMM_USE_CLANG_INTEL_SYNTAX_ASM_HACK "mov %[gp], %%r14\t\n" ".intel_syntax noprefix\t\n" #else "mov r14, %[gp]\t\n" #endif // Copy parameters // k "mov r8, [r14 + 0]\t\n" "dec r8\t\n" // A "mov r9, [r14 + 8]\t\n" // B "mov r10, [r14 + 16]\t\n" // beta "lea r15, [r14 + 24]\t\n" // C "mov r12, [r14 + 32]\t\n" // ldc "mov r13, [r14 + 40]\t\n" // b_block_cols "mov rdi, [r14 + 48]\t\n" // b_block_size "mov rsi, [r14 + 56]\t\n" // Make copies of A and C "mov rax, r9\t\n" "mov rcx, r12\t\n" "xor ebx, ebx\t\n" "loop_outter%=:\t\n" "mov r14, r8\t\n" "vbroadcastss zmm31,DWORD PTR [r15]\t\n" "vcvtph2ps zmm13,YMMWORD PTR [r10 + 0]\t\n" "vcvtph2ps zmm14,YMMWORD PTR [r10 + 32]\t\n" "vxorps xmm0, xmm0, xmm0\t\n" "vcomiss xmm31, xmm0\t\n" "jz zero_regs%=\t\n" // Setup values with beta multiplication "vmulps zmm0, zmm31, [r12 + 0]\t\n" "vmulps zmm1, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm2, zmm31, [r12 + 0]\t\n" "vmulps zmm3, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm4, zmm31, [r12 + 0]\t\n" "vmulps zmm5, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm6, zmm31, [r12 + 0]\t\n" "vmulps zmm7, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm8, zmm31, [r12 + 0]\t\n" "vmulps zmm9, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm10, zmm31, [r12 + 0]\t\n" "vmulps zmm11, zmm31, [r12 + 64]\t\n" "test r14,r14\t\n" "jz skip_preload%=\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "skip_preload%=:\t\n" "vbroadcastss zmm12,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm13,zmm12\t\n" "vfmadd231ps zmm1,zmm14,zmm12\t\n" "vbroadcastss zmm12,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm13,zmm12\t\n" "vfmadd231ps zmm3,zmm14,zmm12\t\n" "vbroadcastss zmm12,DWORD PTR [r9+8]\t\n" "vfmadd231ps zmm4,zmm13,zmm12\t\n" "vfmadd231ps zmm5,zmm14,zmm12\t\n" "vbroadcastss zmm12,DWORD PTR [r9+12]\t\n" "vfmadd231ps zmm6,zmm13,zmm12\t\n" "vfmadd231ps zmm7,zmm14,zmm12\t\n" "vbroadcastss zmm12,DWORD PTR [r9+16]\t\n" "vfmadd231ps zmm8,zmm13,zmm12\t\n" "vfmadd231ps zmm9,zmm14,zmm12\t\n" "vbroadcastss zmm12,DWORD PTR [r9+20]\t\n" "vfmadd231ps zmm10,zmm13,zmm12\t\n" "vfmadd231ps zmm11,zmm14,zmm12\t\n" "mov r12, rcx\t\n" "test r14,r14\t\n" "jnz next_inner%=\t\n" "add r10,64\t\n" "jmp dump_C%=\t\n" "zero_regs%=:\t\n" "test r14,r14\t\n" "jz skip_preload_b_zero%=\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "skip_preload_b_zero%=:\t\n" "vbroadcastss zmm12,DWORD PTR [r9+0]\t\n" "vmulps zmm0,zmm13,zmm12\t\n" "vmulps zmm1,zmm14,zmm12\t\n" "add r12, r13\t\n" "vbroadcastss zmm12,DWORD PTR [r9+4]\t\n" "vmulps zmm2,zmm13,zmm12\t\n" "vmulps zmm3,zmm14,zmm12\t\n" "add r12, r13\t\n" "vbroadcastss zmm12,DWORD PTR [r9+8]\t\n" "vmulps zmm4,zmm13,zmm12\t\n" "vmulps zmm5,zmm14,zmm12\t\n" "add r12, r13\t\n" "vbroadcastss zmm12,DWORD PTR [r9+12]\t\n" "vmulps zmm6,zmm13,zmm12\t\n" "vmulps zmm7,zmm14,zmm12\t\n" "add r12, r13\t\n" "vbroadcastss zmm12,DWORD PTR [r9+16]\t\n" "vmulps zmm8,zmm13,zmm12\t\n" "vmulps zmm9,zmm14,zmm12\t\n" "add r12, r13\t\n" "vbroadcastss zmm12,DWORD PTR [r9+20]\t\n" "vmulps zmm10,zmm13,zmm12\t\n" "vmulps zmm11,zmm14,zmm12\t\n" "mov r12, rcx\t\n" "test r14,r14\t\n" "jnz next_inner%=\t\n" "add r10,64\t\n" "jmp dump_C%=\t\n" "loop_inner%=:\t\n" "vmovaps zmm13,zmm31\t\n" "vcvtph2ps zmm14,YMMWORD PTR [r10 + 32]\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "vbroadcastss zmm12,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm13,zmm12\t\n" "vfmadd231ps zmm1,zmm14,zmm12\t\n" "vbroadcastss zmm12,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm13,zmm12\t\n" "vfmadd231ps zmm3,zmm14,zmm12\t\n" "vbroadcastss zmm12,DWORD PTR [r9+8]\t\n" "vfmadd231ps zmm4,zmm13,zmm12\t\n" "vfmadd231ps zmm5,zmm14,zmm12\t\n" "vbroadcastss zmm12,DWORD PTR [r9+12]\t\n" "vfmadd231ps zmm6,zmm13,zmm12\t\n" "vfmadd231ps zmm7,zmm14,zmm12\t\n" "vbroadcastss zmm12,DWORD PTR [r9+16]\t\n" "vfmadd231ps zmm8,zmm13,zmm12\t\n" "vfmadd231ps zmm9,zmm14,zmm12\t\n" "vbroadcastss zmm12,DWORD PTR [r9+20]\t\n" "vfmadd231ps zmm10,zmm13,zmm12\t\n" "vfmadd231ps zmm11,zmm14,zmm12\t\n" "next_inner%=:\t\n" "add r9,24\t\n" "add r10,64\t\n" "dec r14\t\n" "jnz loop_inner%=\t\n" "vmovaps zmm13,zmm31\t\n" "vcvtph2ps zmm14,YMMWORD PTR [r10 + 32]\t\n" "vbroadcastss zmm12,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm13,zmm12\t\n" "vfmadd231ps zmm1,zmm14,zmm12\t\n" "vbroadcastss zmm12,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm13,zmm12\t\n" "vfmadd231ps zmm3,zmm14,zmm12\t\n" "vbroadcastss zmm12,DWORD PTR [r9+8]\t\n" "vfmadd231ps zmm4,zmm13,zmm12\t\n" "vfmadd231ps zmm5,zmm14,zmm12\t\n" "vbroadcastss zmm12,DWORD PTR [r9+12]\t\n" "vfmadd231ps zmm6,zmm13,zmm12\t\n" "vfmadd231ps zmm7,zmm14,zmm12\t\n" "vbroadcastss zmm12,DWORD PTR [r9+16]\t\n" "vfmadd231ps zmm8,zmm13,zmm12\t\n" "vfmadd231ps zmm9,zmm14,zmm12\t\n" "vbroadcastss zmm12,DWORD PTR [r9+20]\t\n" "vfmadd231ps zmm10,zmm13,zmm12\t\n" "vfmadd231ps zmm11,zmm14,zmm12\t\n" "add r9,24\t\n" "add r10,64\t\n" // Dump C "dump_C%=:\t\n" "vmovups zmmword PTR [r12 + 0], zmm0\t\n" "vmovups zmmword PTR [r12 + 64], zmm1\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm2\t\n" "vmovups zmmword PTR [r12 + 64], zmm3\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm4\t\n" "vmovups zmmword PTR [r12 + 64], zmm5\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm6\t\n" "vmovups zmmword PTR [r12 + 64], zmm7\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm8\t\n" "vmovups zmmword PTR [r12 + 64], zmm9\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm10\t\n" "vmovups zmmword PTR [r12 + 64], zmm11\t\n" // next outer iteration "add rcx, 128\t\n" "mov r12, rcx\t\n" "mov r9, rax\t\n" "inc rbx\t\n" "cmp rbx, rdi\t\n" "jl loop_outter%=\t\n" : : [gp] "rm"(gp) : "r8", "r9", "r10", "r11", "r13", "r14", "rax", "rcx", "rsi", "rdi", "rbx", "r12", "r15", "memory"); } void NOINLINE gemmkernel_7x2_Avx512_fp16_fA0fB0fC0(GemmParamsFP16* gp) { asm volatile( #if FBGEMM_USE_CLANG_INTEL_SYNTAX_ASM_HACK "mov %[gp], %%r14\t\n" ".intel_syntax noprefix\t\n" #else "mov r14, %[gp]\t\n" #endif // Copy parameters // k "mov r8, [r14 + 0]\t\n" "dec r8\t\n" // A "mov r9, [r14 + 8]\t\n" // B "mov r10, [r14 + 16]\t\n" // beta "lea r15, [r14 + 24]\t\n" // C "mov r12, [r14 + 32]\t\n" // ldc "mov r13, [r14 + 40]\t\n" // b_block_cols "mov rdi, [r14 + 48]\t\n" // b_block_size "mov rsi, [r14 + 56]\t\n" // Make copies of A and C "mov rax, r9\t\n" "mov rcx, r12\t\n" "xor ebx, ebx\t\n" "loop_outter%=:\t\n" "mov r14, r8\t\n" "vbroadcastss zmm31,DWORD PTR [r15]\t\n" "vcvtph2ps zmm15,YMMWORD PTR [r10 + 0]\t\n" "vcvtph2ps zmm16,YMMWORD PTR [r10 + 32]\t\n" "vxorps xmm0, xmm0, xmm0\t\n" "vcomiss xmm31, xmm0\t\n" "jz zero_regs%=\t\n" // Setup values with beta multiplication "vmulps zmm0, zmm31, [r12 + 0]\t\n" "vmulps zmm1, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm2, zmm31, [r12 + 0]\t\n" "vmulps zmm3, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm4, zmm31, [r12 + 0]\t\n" "vmulps zmm5, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm6, zmm31, [r12 + 0]\t\n" "vmulps zmm7, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm8, zmm31, [r12 + 0]\t\n" "vmulps zmm9, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm10, zmm31, [r12 + 0]\t\n" "vmulps zmm11, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm12, zmm31, [r12 + 0]\t\n" "vmulps zmm13, zmm31, [r12 + 64]\t\n" "test r14,r14\t\n" "jz skip_preload%=\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "skip_preload%=:\t\n" "vbroadcastss zmm14,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm15,zmm14\t\n" "vfmadd231ps zmm1,zmm16,zmm14\t\n" "vbroadcastss zmm14,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm15,zmm14\t\n" "vfmadd231ps zmm3,zmm16,zmm14\t\n" "vbroadcastss zmm14,DWORD PTR [r9+8]\t\n" "vfmadd231ps zmm4,zmm15,zmm14\t\n" "vfmadd231ps zmm5,zmm16,zmm14\t\n" "vbroadcastss zmm14,DWORD PTR [r9+12]\t\n" "vfmadd231ps zmm6,zmm15,zmm14\t\n" "vfmadd231ps zmm7,zmm16,zmm14\t\n" "vbroadcastss zmm14,DWORD PTR [r9+16]\t\n" "vfmadd231ps zmm8,zmm15,zmm14\t\n" "vfmadd231ps zmm9,zmm16,zmm14\t\n" "vbroadcastss zmm14,DWORD PTR [r9+20]\t\n" "vfmadd231ps zmm10,zmm15,zmm14\t\n" "vfmadd231ps zmm11,zmm16,zmm14\t\n" "vbroadcastss zmm14,DWORD PTR [r9+24]\t\n" "vfmadd231ps zmm12,zmm15,zmm14\t\n" "vfmadd231ps zmm13,zmm16,zmm14\t\n" "mov r12, rcx\t\n" "test r14,r14\t\n" "jnz next_inner%=\t\n" "add r10,64\t\n" "jmp dump_C%=\t\n" "zero_regs%=:\t\n" "test r14,r14\t\n" "jz skip_preload_b_zero%=\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "skip_preload_b_zero%=:\t\n" "vbroadcastss zmm14,DWORD PTR [r9+0]\t\n" "vmulps zmm0,zmm15,zmm14\t\n" "vmulps zmm1,zmm16,zmm14\t\n" "add r12, r13\t\n" "vbroadcastss zmm14,DWORD PTR [r9+4]\t\n" "vmulps zmm2,zmm15,zmm14\t\n" "vmulps zmm3,zmm16,zmm14\t\n" "add r12, r13\t\n" "vbroadcastss zmm14,DWORD PTR [r9+8]\t\n" "vmulps zmm4,zmm15,zmm14\t\n" "vmulps zmm5,zmm16,zmm14\t\n" "add r12, r13\t\n" "vbroadcastss zmm14,DWORD PTR [r9+12]\t\n" "vmulps zmm6,zmm15,zmm14\t\n" "vmulps zmm7,zmm16,zmm14\t\n" "add r12, r13\t\n" "vbroadcastss zmm14,DWORD PTR [r9+16]\t\n" "vmulps zmm8,zmm15,zmm14\t\n" "vmulps zmm9,zmm16,zmm14\t\n" "add r12, r13\t\n" "vbroadcastss zmm14,DWORD PTR [r9+20]\t\n" "vmulps zmm10,zmm15,zmm14\t\n" "vmulps zmm11,zmm16,zmm14\t\n" "add r12, r13\t\n" "vbroadcastss zmm14,DWORD PTR [r9+24]\t\n" "vmulps zmm12,zmm15,zmm14\t\n" "vmulps zmm13,zmm16,zmm14\t\n" "mov r12, rcx\t\n" "test r14,r14\t\n" "jnz next_inner%=\t\n" "add r10,64\t\n" "jmp dump_C%=\t\n" "loop_inner%=:\t\n" "vmovaps zmm15,zmm31\t\n" "vcvtph2ps zmm16,YMMWORD PTR [r10 + 32]\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "vbroadcastss zmm14,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm15,zmm14\t\n" "vfmadd231ps zmm1,zmm16,zmm14\t\n" "vbroadcastss zmm14,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm15,zmm14\t\n" "vfmadd231ps zmm3,zmm16,zmm14\t\n" "vbroadcastss zmm14,DWORD PTR [r9+8]\t\n" "vfmadd231ps zmm4,zmm15,zmm14\t\n" "vfmadd231ps zmm5,zmm16,zmm14\t\n" "vbroadcastss zmm14,DWORD PTR [r9+12]\t\n" "vfmadd231ps zmm6,zmm15,zmm14\t\n" "vfmadd231ps zmm7,zmm16,zmm14\t\n" "vbroadcastss zmm14,DWORD PTR [r9+16]\t\n" "vfmadd231ps zmm8,zmm15,zmm14\t\n" "vfmadd231ps zmm9,zmm16,zmm14\t\n" "vbroadcastss zmm14,DWORD PTR [r9+20]\t\n" "vfmadd231ps zmm10,zmm15,zmm14\t\n" "vfmadd231ps zmm11,zmm16,zmm14\t\n" "vbroadcastss zmm14,DWORD PTR [r9+24]\t\n" "vfmadd231ps zmm12,zmm15,zmm14\t\n" "vfmadd231ps zmm13,zmm16,zmm14\t\n" "next_inner%=:\t\n" "add r9,28\t\n" "add r10,64\t\n" "dec r14\t\n" "jnz loop_inner%=\t\n" "vmovaps zmm15,zmm31\t\n" "vcvtph2ps zmm16,YMMWORD PTR [r10 + 32]\t\n" "vbroadcastss zmm14,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm15,zmm14\t\n" "vfmadd231ps zmm1,zmm16,zmm14\t\n" "vbroadcastss zmm14,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm15,zmm14\t\n" "vfmadd231ps zmm3,zmm16,zmm14\t\n" "vbroadcastss zmm14,DWORD PTR [r9+8]\t\n" "vfmadd231ps zmm4,zmm15,zmm14\t\n" "vfmadd231ps zmm5,zmm16,zmm14\t\n" "vbroadcastss zmm14,DWORD PTR [r9+12]\t\n" "vfmadd231ps zmm6,zmm15,zmm14\t\n" "vfmadd231ps zmm7,zmm16,zmm14\t\n" "vbroadcastss zmm14,DWORD PTR [r9+16]\t\n" "vfmadd231ps zmm8,zmm15,zmm14\t\n" "vfmadd231ps zmm9,zmm16,zmm14\t\n" "vbroadcastss zmm14,DWORD PTR [r9+20]\t\n" "vfmadd231ps zmm10,zmm15,zmm14\t\n" "vfmadd231ps zmm11,zmm16,zmm14\t\n" "vbroadcastss zmm14,DWORD PTR [r9+24]\t\n" "vfmadd231ps zmm12,zmm15,zmm14\t\n" "vfmadd231ps zmm13,zmm16,zmm14\t\n" "add r9,28\t\n" "add r10,64\t\n" // Dump C "dump_C%=:\t\n" "vmovups zmmword PTR [r12 + 0], zmm0\t\n" "vmovups zmmword PTR [r12 + 64], zmm1\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm2\t\n" "vmovups zmmword PTR [r12 + 64], zmm3\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm4\t\n" "vmovups zmmword PTR [r12 + 64], zmm5\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm6\t\n" "vmovups zmmword PTR [r12 + 64], zmm7\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm8\t\n" "vmovups zmmword PTR [r12 + 64], zmm9\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm10\t\n" "vmovups zmmword PTR [r12 + 64], zmm11\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm12\t\n" "vmovups zmmword PTR [r12 + 64], zmm13\t\n" // next outer iteration "add rcx, 128\t\n" "mov r12, rcx\t\n" "mov r9, rax\t\n" "inc rbx\t\n" "cmp rbx, rdi\t\n" "jl loop_outter%=\t\n" : : [gp] "rm"(gp) : "r8", "r9", "r10", "r11", "r13", "r14", "rax", "rcx", "rsi", "rdi", "rbx", "r12", "r15", "memory"); } void NOINLINE gemmkernel_8x2_Avx512_fp16_fA0fB0fC0(GemmParamsFP16* gp) { asm volatile( #if FBGEMM_USE_CLANG_INTEL_SYNTAX_ASM_HACK "mov %[gp], %%r14\t\n" ".intel_syntax noprefix\t\n" #else "mov r14, %[gp]\t\n" #endif // Copy parameters // k "mov r8, [r14 + 0]\t\n" "dec r8\t\n" // A "mov r9, [r14 + 8]\t\n" // B "mov r10, [r14 + 16]\t\n" // beta "lea r15, [r14 + 24]\t\n" // C "mov r12, [r14 + 32]\t\n" // ldc "mov r13, [r14 + 40]\t\n" // b_block_cols "mov rdi, [r14 + 48]\t\n" // b_block_size "mov rsi, [r14 + 56]\t\n" // Make copies of A and C "mov rax, r9\t\n" "mov rcx, r12\t\n" "xor ebx, ebx\t\n" "loop_outter%=:\t\n" "mov r14, r8\t\n" "vbroadcastss zmm31,DWORD PTR [r15]\t\n" "vcvtph2ps zmm17,YMMWORD PTR [r10 + 0]\t\n" "vcvtph2ps zmm18,YMMWORD PTR [r10 + 32]\t\n" "vxorps xmm0, xmm0, xmm0\t\n" "vcomiss xmm31, xmm0\t\n" "jz zero_regs%=\t\n" // Setup values with beta multiplication "vmulps zmm0, zmm31, [r12 + 0]\t\n" "vmulps zmm1, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm2, zmm31, [r12 + 0]\t\n" "vmulps zmm3, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm4, zmm31, [r12 + 0]\t\n" "vmulps zmm5, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm6, zmm31, [r12 + 0]\t\n" "vmulps zmm7, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm8, zmm31, [r12 + 0]\t\n" "vmulps zmm9, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm10, zmm31, [r12 + 0]\t\n" "vmulps zmm11, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm12, zmm31, [r12 + 0]\t\n" "vmulps zmm13, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm14, zmm31, [r12 + 0]\t\n" "vmulps zmm15, zmm31, [r12 + 64]\t\n" "test r14,r14\t\n" "jz skip_preload%=\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "skip_preload%=:\t\n" "vbroadcastss zmm16,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm17,zmm16\t\n" "vfmadd231ps zmm1,zmm18,zmm16\t\n" "vbroadcastss zmm16,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm17,zmm16\t\n" "vfmadd231ps zmm3,zmm18,zmm16\t\n" "vbroadcastss zmm16,DWORD PTR [r9+8]\t\n" "vfmadd231ps zmm4,zmm17,zmm16\t\n" "vfmadd231ps zmm5,zmm18,zmm16\t\n" "vbroadcastss zmm16,DWORD PTR [r9+12]\t\n" "vfmadd231ps zmm6,zmm17,zmm16\t\n" "vfmadd231ps zmm7,zmm18,zmm16\t\n" "vbroadcastss zmm16,DWORD PTR [r9+16]\t\n" "vfmadd231ps zmm8,zmm17,zmm16\t\n" "vfmadd231ps zmm9,zmm18,zmm16\t\n" "vbroadcastss zmm16,DWORD PTR [r9+20]\t\n" "vfmadd231ps zmm10,zmm17,zmm16\t\n" "vfmadd231ps zmm11,zmm18,zmm16\t\n" "vbroadcastss zmm16,DWORD PTR [r9+24]\t\n" "vfmadd231ps zmm12,zmm17,zmm16\t\n" "vfmadd231ps zmm13,zmm18,zmm16\t\n" "vbroadcastss zmm16,DWORD PTR [r9+28]\t\n" "vfmadd231ps zmm14,zmm17,zmm16\t\n" "vfmadd231ps zmm15,zmm18,zmm16\t\n" "mov r12, rcx\t\n" "test r14,r14\t\n" "jnz next_inner%=\t\n" "add r10,64\t\n" "jmp dump_C%=\t\n" "zero_regs%=:\t\n" "test r14,r14\t\n" "jz skip_preload_b_zero%=\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "skip_preload_b_zero%=:\t\n" "vbroadcastss zmm16,DWORD PTR [r9+0]\t\n" "vmulps zmm0,zmm17,zmm16\t\n" "vmulps zmm1,zmm18,zmm16\t\n" "add r12, r13\t\n" "vbroadcastss zmm16,DWORD PTR [r9+4]\t\n" "vmulps zmm2,zmm17,zmm16\t\n" "vmulps zmm3,zmm18,zmm16\t\n" "add r12, r13\t\n" "vbroadcastss zmm16,DWORD PTR [r9+8]\t\n" "vmulps zmm4,zmm17,zmm16\t\n" "vmulps zmm5,zmm18,zmm16\t\n" "add r12, r13\t\n" "vbroadcastss zmm16,DWORD PTR [r9+12]\t\n" "vmulps zmm6,zmm17,zmm16\t\n" "vmulps zmm7,zmm18,zmm16\t\n" "add r12, r13\t\n" "vbroadcastss zmm16,DWORD PTR [r9+16]\t\n" "vmulps zmm8,zmm17,zmm16\t\n" "vmulps zmm9,zmm18,zmm16\t\n" "add r12, r13\t\n" "vbroadcastss zmm16,DWORD PTR [r9+20]\t\n" "vmulps zmm10,zmm17,zmm16\t\n" "vmulps zmm11,zmm18,zmm16\t\n" "add r12, r13\t\n" "vbroadcastss zmm16,DWORD PTR [r9+24]\t\n" "vmulps zmm12,zmm17,zmm16\t\n" "vmulps zmm13,zmm18,zmm16\t\n" "add r12, r13\t\n" "vbroadcastss zmm16,DWORD PTR [r9+28]\t\n" "vmulps zmm14,zmm17,zmm16\t\n" "vmulps zmm15,zmm18,zmm16\t\n" "mov r12, rcx\t\n" "test r14,r14\t\n" "jnz next_inner%=\t\n" "add r10,64\t\n" "jmp dump_C%=\t\n" "loop_inner%=:\t\n" "vmovaps zmm17,zmm31\t\n" "vcvtph2ps zmm18,YMMWORD PTR [r10 + 32]\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "vbroadcastss zmm16,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm17,zmm16\t\n" "vfmadd231ps zmm1,zmm18,zmm16\t\n" "vbroadcastss zmm16,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm17,zmm16\t\n" "vfmadd231ps zmm3,zmm18,zmm16\t\n" "vbroadcastss zmm16,DWORD PTR [r9+8]\t\n" "vfmadd231ps zmm4,zmm17,zmm16\t\n" "vfmadd231ps zmm5,zmm18,zmm16\t\n" "vbroadcastss zmm16,DWORD PTR [r9+12]\t\n" "vfmadd231ps zmm6,zmm17,zmm16\t\n" "vfmadd231ps zmm7,zmm18,zmm16\t\n" "vbroadcastss zmm16,DWORD PTR [r9+16]\t\n" "vfmadd231ps zmm8,zmm17,zmm16\t\n" "vfmadd231ps zmm9,zmm18,zmm16\t\n" "vbroadcastss zmm16,DWORD PTR [r9+20]\t\n" "vfmadd231ps zmm10,zmm17,zmm16\t\n" "vfmadd231ps zmm11,zmm18,zmm16\t\n" "vbroadcastss zmm16,DWORD PTR [r9+24]\t\n" "vfmadd231ps zmm12,zmm17,zmm16\t\n" "vfmadd231ps zmm13,zmm18,zmm16\t\n" "vbroadcastss zmm16,DWORD PTR [r9+28]\t\n" "vfmadd231ps zmm14,zmm17,zmm16\t\n" "vfmadd231ps zmm15,zmm18,zmm16\t\n" "next_inner%=:\t\n" "add r9,32\t\n" "add r10,64\t\n" "dec r14\t\n" "jnz loop_inner%=\t\n" "vmovaps zmm17,zmm31\t\n" "vcvtph2ps zmm18,YMMWORD PTR [r10 + 32]\t\n" "vbroadcastss zmm16,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm17,zmm16\t\n" "vfmadd231ps zmm1,zmm18,zmm16\t\n" "vbroadcastss zmm16,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm17,zmm16\t\n" "vfmadd231ps zmm3,zmm18,zmm16\t\n" "vbroadcastss zmm16,DWORD PTR [r9+8]\t\n" "vfmadd231ps zmm4,zmm17,zmm16\t\n" "vfmadd231ps zmm5,zmm18,zmm16\t\n" "vbroadcastss zmm16,DWORD PTR [r9+12]\t\n" "vfmadd231ps zmm6,zmm17,zmm16\t\n" "vfmadd231ps zmm7,zmm18,zmm16\t\n" "vbroadcastss zmm16,DWORD PTR [r9+16]\t\n" "vfmadd231ps zmm8,zmm17,zmm16\t\n" "vfmadd231ps zmm9,zmm18,zmm16\t\n" "vbroadcastss zmm16,DWORD PTR [r9+20]\t\n" "vfmadd231ps zmm10,zmm17,zmm16\t\n" "vfmadd231ps zmm11,zmm18,zmm16\t\n" "vbroadcastss zmm16,DWORD PTR [r9+24]\t\n" "vfmadd231ps zmm12,zmm17,zmm16\t\n" "vfmadd231ps zmm13,zmm18,zmm16\t\n" "vbroadcastss zmm16,DWORD PTR [r9+28]\t\n" "vfmadd231ps zmm14,zmm17,zmm16\t\n" "vfmadd231ps zmm15,zmm18,zmm16\t\n" "add r9,32\t\n" "add r10,64\t\n" // Dump C "dump_C%=:\t\n" "vmovups zmmword PTR [r12 + 0], zmm0\t\n" "vmovups zmmword PTR [r12 + 64], zmm1\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm2\t\n" "vmovups zmmword PTR [r12 + 64], zmm3\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm4\t\n" "vmovups zmmword PTR [r12 + 64], zmm5\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm6\t\n" "vmovups zmmword PTR [r12 + 64], zmm7\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm8\t\n" "vmovups zmmword PTR [r12 + 64], zmm9\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm10\t\n" "vmovups zmmword PTR [r12 + 64], zmm11\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm12\t\n" "vmovups zmmword PTR [r12 + 64], zmm13\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm14\t\n" "vmovups zmmword PTR [r12 + 64], zmm15\t\n" // next outer iteration "add rcx, 128\t\n" "mov r12, rcx\t\n" "mov r9, rax\t\n" "inc rbx\t\n" "cmp rbx, rdi\t\n" "jl loop_outter%=\t\n" : : [gp] "rm"(gp) : "r8", "r9", "r10", "r11", "r13", "r14", "rax", "rcx", "rsi", "rdi", "rbx", "r12", "r15", "memory"); } void NOINLINE gemmkernel_9x2_Avx512_fp16_fA0fB0fC0(GemmParamsFP16* gp) { asm volatile( #if FBGEMM_USE_CLANG_INTEL_SYNTAX_ASM_HACK "mov %[gp], %%r14\t\n" ".intel_syntax noprefix\t\n" #else "mov r14, %[gp]\t\n" #endif // Copy parameters // k "mov r8, [r14 + 0]\t\n" "dec r8\t\n" // A "mov r9, [r14 + 8]\t\n" // B "mov r10, [r14 + 16]\t\n" // beta "lea r15, [r14 + 24]\t\n" // C "mov r12, [r14 + 32]\t\n" // ldc "mov r13, [r14 + 40]\t\n" // b_block_cols "mov rdi, [r14 + 48]\t\n" // b_block_size "mov rsi, [r14 + 56]\t\n" // Make copies of A and C "mov rax, r9\t\n" "mov rcx, r12\t\n" "xor ebx, ebx\t\n" "loop_outter%=:\t\n" "mov r14, r8\t\n" "vbroadcastss zmm31,DWORD PTR [r15]\t\n" "vcvtph2ps zmm19,YMMWORD PTR [r10 + 0]\t\n" "vcvtph2ps zmm20,YMMWORD PTR [r10 + 32]\t\n" "vxorps xmm0, xmm0, xmm0\t\n" "vcomiss xmm31, xmm0\t\n" "jz zero_regs%=\t\n" // Setup values with beta multiplication "vmulps zmm0, zmm31, [r12 + 0]\t\n" "vmulps zmm1, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm2, zmm31, [r12 + 0]\t\n" "vmulps zmm3, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm4, zmm31, [r12 + 0]\t\n" "vmulps zmm5, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm6, zmm31, [r12 + 0]\t\n" "vmulps zmm7, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm8, zmm31, [r12 + 0]\t\n" "vmulps zmm9, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm10, zmm31, [r12 + 0]\t\n" "vmulps zmm11, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm12, zmm31, [r12 + 0]\t\n" "vmulps zmm13, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm14, zmm31, [r12 + 0]\t\n" "vmulps zmm15, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm16, zmm31, [r12 + 0]\t\n" "vmulps zmm17, zmm31, [r12 + 64]\t\n" "test r14,r14\t\n" "jz skip_preload%=\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "skip_preload%=:\t\n" "vbroadcastss zmm18,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm19,zmm18\t\n" "vfmadd231ps zmm1,zmm20,zmm18\t\n" "vbroadcastss zmm18,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm19,zmm18\t\n" "vfmadd231ps zmm3,zmm20,zmm18\t\n" "vbroadcastss zmm18,DWORD PTR [r9+8]\t\n" "vfmadd231ps zmm4,zmm19,zmm18\t\n" "vfmadd231ps zmm5,zmm20,zmm18\t\n" "vbroadcastss zmm18,DWORD PTR [r9+12]\t\n" "vfmadd231ps zmm6,zmm19,zmm18\t\n" "vfmadd231ps zmm7,zmm20,zmm18\t\n" "vbroadcastss zmm18,DWORD PTR [r9+16]\t\n" "vfmadd231ps zmm8,zmm19,zmm18\t\n" "vfmadd231ps zmm9,zmm20,zmm18\t\n" "vbroadcastss zmm18,DWORD PTR [r9+20]\t\n" "vfmadd231ps zmm10,zmm19,zmm18\t\n" "vfmadd231ps zmm11,zmm20,zmm18\t\n" "vbroadcastss zmm18,DWORD PTR [r9+24]\t\n" "vfmadd231ps zmm12,zmm19,zmm18\t\n" "vfmadd231ps zmm13,zmm20,zmm18\t\n" "vbroadcastss zmm18,DWORD PTR [r9+28]\t\n" "vfmadd231ps zmm14,zmm19,zmm18\t\n" "vfmadd231ps zmm15,zmm20,zmm18\t\n" "vbroadcastss zmm18,DWORD PTR [r9+32]\t\n" "vfmadd231ps zmm16,zmm19,zmm18\t\n" "vfmadd231ps zmm17,zmm20,zmm18\t\n" "mov r12, rcx\t\n" "test r14,r14\t\n" "jnz next_inner%=\t\n" "add r10,64\t\n" "jmp dump_C%=\t\n" "zero_regs%=:\t\n" "test r14,r14\t\n" "jz skip_preload_b_zero%=\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "skip_preload_b_zero%=:\t\n" "vbroadcastss zmm18,DWORD PTR [r9+0]\t\n" "vmulps zmm0,zmm19,zmm18\t\n" "vmulps zmm1,zmm20,zmm18\t\n" "add r12, r13\t\n" "vbroadcastss zmm18,DWORD PTR [r9+4]\t\n" "vmulps zmm2,zmm19,zmm18\t\n" "vmulps zmm3,zmm20,zmm18\t\n" "add r12, r13\t\n" "vbroadcastss zmm18,DWORD PTR [r9+8]\t\n" "vmulps zmm4,zmm19,zmm18\t\n" "vmulps zmm5,zmm20,zmm18\t\n" "add r12, r13\t\n" "vbroadcastss zmm18,DWORD PTR [r9+12]\t\n" "vmulps zmm6,zmm19,zmm18\t\n" "vmulps zmm7,zmm20,zmm18\t\n" "add r12, r13\t\n" "vbroadcastss zmm18,DWORD PTR [r9+16]\t\n" "vmulps zmm8,zmm19,zmm18\t\n" "vmulps zmm9,zmm20,zmm18\t\n" "add r12, r13\t\n" "vbroadcastss zmm18,DWORD PTR [r9+20]\t\n" "vmulps zmm10,zmm19,zmm18\t\n" "vmulps zmm11,zmm20,zmm18\t\n" "add r12, r13\t\n" "vbroadcastss zmm18,DWORD PTR [r9+24]\t\n" "vmulps zmm12,zmm19,zmm18\t\n" "vmulps zmm13,zmm20,zmm18\t\n" "add r12, r13\t\n" "vbroadcastss zmm18,DWORD PTR [r9+28]\t\n" "vmulps zmm14,zmm19,zmm18\t\n" "vmulps zmm15,zmm20,zmm18\t\n" "add r12, r13\t\n" "vbroadcastss zmm18,DWORD PTR [r9+32]\t\n" "vmulps zmm16,zmm19,zmm18\t\n" "vmulps zmm17,zmm20,zmm18\t\n" "mov r12, rcx\t\n" "test r14,r14\t\n" "jnz next_inner%=\t\n" "add r10,64\t\n" "jmp dump_C%=\t\n" "loop_inner%=:\t\n" "vmovaps zmm19,zmm31\t\n" "vcvtph2ps zmm20,YMMWORD PTR [r10 + 32]\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "vbroadcastss zmm18,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm19,zmm18\t\n" "vfmadd231ps zmm1,zmm20,zmm18\t\n" "vbroadcastss zmm18,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm19,zmm18\t\n" "vfmadd231ps zmm3,zmm20,zmm18\t\n" "vbroadcastss zmm18,DWORD PTR [r9+8]\t\n" "vfmadd231ps zmm4,zmm19,zmm18\t\n" "vfmadd231ps zmm5,zmm20,zmm18\t\n" "vbroadcastss zmm18,DWORD PTR [r9+12]\t\n" "vfmadd231ps zmm6,zmm19,zmm18\t\n" "vfmadd231ps zmm7,zmm20,zmm18\t\n" "vbroadcastss zmm18,DWORD PTR [r9+16]\t\n" "vfmadd231ps zmm8,zmm19,zmm18\t\n" "vfmadd231ps zmm9,zmm20,zmm18\t\n" "vbroadcastss zmm18,DWORD PTR [r9+20]\t\n" "vfmadd231ps zmm10,zmm19,zmm18\t\n" "vfmadd231ps zmm11,zmm20,zmm18\t\n" "vbroadcastss zmm18,DWORD PTR [r9+24]\t\n" "vfmadd231ps zmm12,zmm19,zmm18\t\n" "vfmadd231ps zmm13,zmm20,zmm18\t\n" "vbroadcastss zmm18,DWORD PTR [r9+28]\t\n" "vfmadd231ps zmm14,zmm19,zmm18\t\n" "vfmadd231ps zmm15,zmm20,zmm18\t\n" "vbroadcastss zmm18,DWORD PTR [r9+32]\t\n" "vfmadd231ps zmm16,zmm19,zmm18\t\n" "vfmadd231ps zmm17,zmm20,zmm18\t\n" "next_inner%=:\t\n" "add r9,36\t\n" "add r10,64\t\n" "dec r14\t\n" "jnz loop_inner%=\t\n" "vmovaps zmm19,zmm31\t\n" "vcvtph2ps zmm20,YMMWORD PTR [r10 + 32]\t\n" "vbroadcastss zmm18,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm19,zmm18\t\n" "vfmadd231ps zmm1,zmm20,zmm18\t\n" "vbroadcastss zmm18,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm19,zmm18\t\n" "vfmadd231ps zmm3,zmm20,zmm18\t\n" "vbroadcastss zmm18,DWORD PTR [r9+8]\t\n" "vfmadd231ps zmm4,zmm19,zmm18\t\n" "vfmadd231ps zmm5,zmm20,zmm18\t\n" "vbroadcastss zmm18,DWORD PTR [r9+12]\t\n" "vfmadd231ps zmm6,zmm19,zmm18\t\n" "vfmadd231ps zmm7,zmm20,zmm18\t\n" "vbroadcastss zmm18,DWORD PTR [r9+16]\t\n" "vfmadd231ps zmm8,zmm19,zmm18\t\n" "vfmadd231ps zmm9,zmm20,zmm18\t\n" "vbroadcastss zmm18,DWORD PTR [r9+20]\t\n" "vfmadd231ps zmm10,zmm19,zmm18\t\n" "vfmadd231ps zmm11,zmm20,zmm18\t\n" "vbroadcastss zmm18,DWORD PTR [r9+24]\t\n" "vfmadd231ps zmm12,zmm19,zmm18\t\n" "vfmadd231ps zmm13,zmm20,zmm18\t\n" "vbroadcastss zmm18,DWORD PTR [r9+28]\t\n" "vfmadd231ps zmm14,zmm19,zmm18\t\n" "vfmadd231ps zmm15,zmm20,zmm18\t\n" "vbroadcastss zmm18,DWORD PTR [r9+32]\t\n" "vfmadd231ps zmm16,zmm19,zmm18\t\n" "vfmadd231ps zmm17,zmm20,zmm18\t\n" "add r9,36\t\n" "add r10,64\t\n" // Dump C "dump_C%=:\t\n" "vmovups zmmword PTR [r12 + 0], zmm0\t\n" "vmovups zmmword PTR [r12 + 64], zmm1\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm2\t\n" "vmovups zmmword PTR [r12 + 64], zmm3\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm4\t\n" "vmovups zmmword PTR [r12 + 64], zmm5\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm6\t\n" "vmovups zmmword PTR [r12 + 64], zmm7\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm8\t\n" "vmovups zmmword PTR [r12 + 64], zmm9\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm10\t\n" "vmovups zmmword PTR [r12 + 64], zmm11\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm12\t\n" "vmovups zmmword PTR [r12 + 64], zmm13\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm14\t\n" "vmovups zmmword PTR [r12 + 64], zmm15\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm16\t\n" "vmovups zmmword PTR [r12 + 64], zmm17\t\n" // next outer iteration "add rcx, 128\t\n" "mov r12, rcx\t\n" "mov r9, rax\t\n" "inc rbx\t\n" "cmp rbx, rdi\t\n" "jl loop_outter%=\t\n" : : [gp] "rm"(gp) : "r8", "r9", "r10", "r11", "r13", "r14", "rax", "rcx", "rsi", "rdi", "rbx", "r12", "r15", "memory"); } void NOINLINE gemmkernel_10x2_Avx512_fp16_fA0fB0fC0(GemmParamsFP16* gp) { asm volatile( #if FBGEMM_USE_CLANG_INTEL_SYNTAX_ASM_HACK "mov %[gp], %%r14\t\n" ".intel_syntax noprefix\t\n" #else "mov r14, %[gp]\t\n" #endif // Copy parameters // k "mov r8, [r14 + 0]\t\n" "dec r8\t\n" // A "mov r9, [r14 + 8]\t\n" // B "mov r10, [r14 + 16]\t\n" // beta "lea r15, [r14 + 24]\t\n" // C "mov r12, [r14 + 32]\t\n" // ldc "mov r13, [r14 + 40]\t\n" // b_block_cols "mov rdi, [r14 + 48]\t\n" // b_block_size "mov rsi, [r14 + 56]\t\n" // Make copies of A and C "mov rax, r9\t\n" "mov rcx, r12\t\n" "xor ebx, ebx\t\n" "loop_outter%=:\t\n" "mov r14, r8\t\n" "vbroadcastss zmm31,DWORD PTR [r15]\t\n" "vcvtph2ps zmm21,YMMWORD PTR [r10 + 0]\t\n" "vcvtph2ps zmm22,YMMWORD PTR [r10 + 32]\t\n" "vxorps xmm0, xmm0, xmm0\t\n" "vcomiss xmm31, xmm0\t\n" "jz zero_regs%=\t\n" // Setup values with beta multiplication "vmulps zmm0, zmm31, [r12 + 0]\t\n" "vmulps zmm1, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm2, zmm31, [r12 + 0]\t\n" "vmulps zmm3, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm4, zmm31, [r12 + 0]\t\n" "vmulps zmm5, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm6, zmm31, [r12 + 0]\t\n" "vmulps zmm7, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm8, zmm31, [r12 + 0]\t\n" "vmulps zmm9, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm10, zmm31, [r12 + 0]\t\n" "vmulps zmm11, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm12, zmm31, [r12 + 0]\t\n" "vmulps zmm13, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm14, zmm31, [r12 + 0]\t\n" "vmulps zmm15, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm16, zmm31, [r12 + 0]\t\n" "vmulps zmm17, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm18, zmm31, [r12 + 0]\t\n" "vmulps zmm19, zmm31, [r12 + 64]\t\n" "test r14,r14\t\n" "jz skip_preload%=\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "skip_preload%=:\t\n" "vbroadcastss zmm20,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm21,zmm20\t\n" "vfmadd231ps zmm1,zmm22,zmm20\t\n" "vbroadcastss zmm20,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm21,zmm20\t\n" "vfmadd231ps zmm3,zmm22,zmm20\t\n" "vbroadcastss zmm20,DWORD PTR [r9+8]\t\n" "vfmadd231ps zmm4,zmm21,zmm20\t\n" "vfmadd231ps zmm5,zmm22,zmm20\t\n" "vbroadcastss zmm20,DWORD PTR [r9+12]\t\n" "vfmadd231ps zmm6,zmm21,zmm20\t\n" "vfmadd231ps zmm7,zmm22,zmm20\t\n" "vbroadcastss zmm20,DWORD PTR [r9+16]\t\n" "vfmadd231ps zmm8,zmm21,zmm20\t\n" "vfmadd231ps zmm9,zmm22,zmm20\t\n" "vbroadcastss zmm20,DWORD PTR [r9+20]\t\n" "vfmadd231ps zmm10,zmm21,zmm20\t\n" "vfmadd231ps zmm11,zmm22,zmm20\t\n" "vbroadcastss zmm20,DWORD PTR [r9+24]\t\n" "vfmadd231ps zmm12,zmm21,zmm20\t\n" "vfmadd231ps zmm13,zmm22,zmm20\t\n" "vbroadcastss zmm20,DWORD PTR [r9+28]\t\n" "vfmadd231ps zmm14,zmm21,zmm20\t\n" "vfmadd231ps zmm15,zmm22,zmm20\t\n" "vbroadcastss zmm20,DWORD PTR [r9+32]\t\n" "vfmadd231ps zmm16,zmm21,zmm20\t\n" "vfmadd231ps zmm17,zmm22,zmm20\t\n" "vbroadcastss zmm20,DWORD PTR [r9+36]\t\n" "vfmadd231ps zmm18,zmm21,zmm20\t\n" "vfmadd231ps zmm19,zmm22,zmm20\t\n" "mov r12, rcx\t\n" "test r14,r14\t\n" "jnz next_inner%=\t\n" "add r10,64\t\n" "jmp dump_C%=\t\n" "zero_regs%=:\t\n" "test r14,r14\t\n" "jz skip_preload_b_zero%=\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "skip_preload_b_zero%=:\t\n" "vbroadcastss zmm20,DWORD PTR [r9+0]\t\n" "vmulps zmm0,zmm21,zmm20\t\n" "vmulps zmm1,zmm22,zmm20\t\n" "add r12, r13\t\n" "vbroadcastss zmm20,DWORD PTR [r9+4]\t\n" "vmulps zmm2,zmm21,zmm20\t\n" "vmulps zmm3,zmm22,zmm20\t\n" "add r12, r13\t\n" "vbroadcastss zmm20,DWORD PTR [r9+8]\t\n" "vmulps zmm4,zmm21,zmm20\t\n" "vmulps zmm5,zmm22,zmm20\t\n" "add r12, r13\t\n" "vbroadcastss zmm20,DWORD PTR [r9+12]\t\n" "vmulps zmm6,zmm21,zmm20\t\n" "vmulps zmm7,zmm22,zmm20\t\n" "add r12, r13\t\n" "vbroadcastss zmm20,DWORD PTR [r9+16]\t\n" "vmulps zmm8,zmm21,zmm20\t\n" "vmulps zmm9,zmm22,zmm20\t\n" "add r12, r13\t\n" "vbroadcastss zmm20,DWORD PTR [r9+20]\t\n" "vmulps zmm10,zmm21,zmm20\t\n" "vmulps zmm11,zmm22,zmm20\t\n" "add r12, r13\t\n" "vbroadcastss zmm20,DWORD PTR [r9+24]\t\n" "vmulps zmm12,zmm21,zmm20\t\n" "vmulps zmm13,zmm22,zmm20\t\n" "add r12, r13\t\n" "vbroadcastss zmm20,DWORD PTR [r9+28]\t\n" "vmulps zmm14,zmm21,zmm20\t\n" "vmulps zmm15,zmm22,zmm20\t\n" "add r12, r13\t\n" "vbroadcastss zmm20,DWORD PTR [r9+32]\t\n" "vmulps zmm16,zmm21,zmm20\t\n" "vmulps zmm17,zmm22,zmm20\t\n" "add r12, r13\t\n" "vbroadcastss zmm20,DWORD PTR [r9+36]\t\n" "vmulps zmm18,zmm21,zmm20\t\n" "vmulps zmm19,zmm22,zmm20\t\n" "mov r12, rcx\t\n" "test r14,r14\t\n" "jnz next_inner%=\t\n" "add r10,64\t\n" "jmp dump_C%=\t\n" "loop_inner%=:\t\n" "vmovaps zmm21,zmm31\t\n" "vcvtph2ps zmm22,YMMWORD PTR [r10 + 32]\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "vbroadcastss zmm20,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm21,zmm20\t\n" "vfmadd231ps zmm1,zmm22,zmm20\t\n" "vbroadcastss zmm20,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm21,zmm20\t\n" "vfmadd231ps zmm3,zmm22,zmm20\t\n" "vbroadcastss zmm20,DWORD PTR [r9+8]\t\n" "vfmadd231ps zmm4,zmm21,zmm20\t\n" "vfmadd231ps zmm5,zmm22,zmm20\t\n" "vbroadcastss zmm20,DWORD PTR [r9+12]\t\n" "vfmadd231ps zmm6,zmm21,zmm20\t\n" "vfmadd231ps zmm7,zmm22,zmm20\t\n" "vbroadcastss zmm20,DWORD PTR [r9+16]\t\n" "vfmadd231ps zmm8,zmm21,zmm20\t\n" "vfmadd231ps zmm9,zmm22,zmm20\t\n" "vbroadcastss zmm20,DWORD PTR [r9+20]\t\n" "vfmadd231ps zmm10,zmm21,zmm20\t\n" "vfmadd231ps zmm11,zmm22,zmm20\t\n" "vbroadcastss zmm20,DWORD PTR [r9+24]\t\n" "vfmadd231ps zmm12,zmm21,zmm20\t\n" "vfmadd231ps zmm13,zmm22,zmm20\t\n" "vbroadcastss zmm20,DWORD PTR [r9+28]\t\n" "vfmadd231ps zmm14,zmm21,zmm20\t\n" "vfmadd231ps zmm15,zmm22,zmm20\t\n" "vbroadcastss zmm20,DWORD PTR [r9+32]\t\n" "vfmadd231ps zmm16,zmm21,zmm20\t\n" "vfmadd231ps zmm17,zmm22,zmm20\t\n" "vbroadcastss zmm20,DWORD PTR [r9+36]\t\n" "vfmadd231ps zmm18,zmm21,zmm20\t\n" "vfmadd231ps zmm19,zmm22,zmm20\t\n" "next_inner%=:\t\n" "add r9,40\t\n" "add r10,64\t\n" "dec r14\t\n" "jnz loop_inner%=\t\n" "vmovaps zmm21,zmm31\t\n" "vcvtph2ps zmm22,YMMWORD PTR [r10 + 32]\t\n" "vbroadcastss zmm20,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm21,zmm20\t\n" "vfmadd231ps zmm1,zmm22,zmm20\t\n" "vbroadcastss zmm20,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm21,zmm20\t\n" "vfmadd231ps zmm3,zmm22,zmm20\t\n" "vbroadcastss zmm20,DWORD PTR [r9+8]\t\n" "vfmadd231ps zmm4,zmm21,zmm20\t\n" "vfmadd231ps zmm5,zmm22,zmm20\t\n" "vbroadcastss zmm20,DWORD PTR [r9+12]\t\n" "vfmadd231ps zmm6,zmm21,zmm20\t\n" "vfmadd231ps zmm7,zmm22,zmm20\t\n" "vbroadcastss zmm20,DWORD PTR [r9+16]\t\n" "vfmadd231ps zmm8,zmm21,zmm20\t\n" "vfmadd231ps zmm9,zmm22,zmm20\t\n" "vbroadcastss zmm20,DWORD PTR [r9+20]\t\n" "vfmadd231ps zmm10,zmm21,zmm20\t\n" "vfmadd231ps zmm11,zmm22,zmm20\t\n" "vbroadcastss zmm20,DWORD PTR [r9+24]\t\n" "vfmadd231ps zmm12,zmm21,zmm20\t\n" "vfmadd231ps zmm13,zmm22,zmm20\t\n" "vbroadcastss zmm20,DWORD PTR [r9+28]\t\n" "vfmadd231ps zmm14,zmm21,zmm20\t\n" "vfmadd231ps zmm15,zmm22,zmm20\t\n" "vbroadcastss zmm20,DWORD PTR [r9+32]\t\n" "vfmadd231ps zmm16,zmm21,zmm20\t\n" "vfmadd231ps zmm17,zmm22,zmm20\t\n" "vbroadcastss zmm20,DWORD PTR [r9+36]\t\n" "vfmadd231ps zmm18,zmm21,zmm20\t\n" "vfmadd231ps zmm19,zmm22,zmm20\t\n" "add r9,40\t\n" "add r10,64\t\n" // Dump C "dump_C%=:\t\n" "vmovups zmmword PTR [r12 + 0], zmm0\t\n" "vmovups zmmword PTR [r12 + 64], zmm1\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm2\t\n" "vmovups zmmword PTR [r12 + 64], zmm3\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm4\t\n" "vmovups zmmword PTR [r12 + 64], zmm5\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm6\t\n" "vmovups zmmword PTR [r12 + 64], zmm7\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm8\t\n" "vmovups zmmword PTR [r12 + 64], zmm9\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm10\t\n" "vmovups zmmword PTR [r12 + 64], zmm11\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm12\t\n" "vmovups zmmword PTR [r12 + 64], zmm13\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm14\t\n" "vmovups zmmword PTR [r12 + 64], zmm15\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm16\t\n" "vmovups zmmword PTR [r12 + 64], zmm17\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm18\t\n" "vmovups zmmword PTR [r12 + 64], zmm19\t\n" // next outer iteration "add rcx, 128\t\n" "mov r12, rcx\t\n" "mov r9, rax\t\n" "inc rbx\t\n" "cmp rbx, rdi\t\n" "jl loop_outter%=\t\n" : : [gp] "rm"(gp) : "r8", "r9", "r10", "r11", "r13", "r14", "rax", "rcx", "rsi", "rdi", "rbx", "r12", "r15", "memory"); } void NOINLINE gemmkernel_11x2_Avx512_fp16_fA0fB0fC0(GemmParamsFP16* gp) { asm volatile( #if FBGEMM_USE_CLANG_INTEL_SYNTAX_ASM_HACK "mov %[gp], %%r14\t\n" ".intel_syntax noprefix\t\n" #else "mov r14, %[gp]\t\n" #endif // Copy parameters // k "mov r8, [r14 + 0]\t\n" "dec r8\t\n" // A "mov r9, [r14 + 8]\t\n" // B "mov r10, [r14 + 16]\t\n" // beta "lea r15, [r14 + 24]\t\n" // C "mov r12, [r14 + 32]\t\n" // ldc "mov r13, [r14 + 40]\t\n" // b_block_cols "mov rdi, [r14 + 48]\t\n" // b_block_size "mov rsi, [r14 + 56]\t\n" // Make copies of A and C "mov rax, r9\t\n" "mov rcx, r12\t\n" "xor ebx, ebx\t\n" "loop_outter%=:\t\n" "mov r14, r8\t\n" "vbroadcastss zmm31,DWORD PTR [r15]\t\n" "vcvtph2ps zmm23,YMMWORD PTR [r10 + 0]\t\n" "vcvtph2ps zmm24,YMMWORD PTR [r10 + 32]\t\n" "vxorps xmm0, xmm0, xmm0\t\n" "vcomiss xmm31, xmm0\t\n" "jz zero_regs%=\t\n" // Setup values with beta multiplication "vmulps zmm0, zmm31, [r12 + 0]\t\n" "vmulps zmm1, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm2, zmm31, [r12 + 0]\t\n" "vmulps zmm3, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm4, zmm31, [r12 + 0]\t\n" "vmulps zmm5, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm6, zmm31, [r12 + 0]\t\n" "vmulps zmm7, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm8, zmm31, [r12 + 0]\t\n" "vmulps zmm9, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm10, zmm31, [r12 + 0]\t\n" "vmulps zmm11, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm12, zmm31, [r12 + 0]\t\n" "vmulps zmm13, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm14, zmm31, [r12 + 0]\t\n" "vmulps zmm15, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm16, zmm31, [r12 + 0]\t\n" "vmulps zmm17, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm18, zmm31, [r12 + 0]\t\n" "vmulps zmm19, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm20, zmm31, [r12 + 0]\t\n" "vmulps zmm21, zmm31, [r12 + 64]\t\n" "test r14,r14\t\n" "jz skip_preload%=\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "skip_preload%=:\t\n" "vbroadcastss zmm22,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm23,zmm22\t\n" "vfmadd231ps zmm1,zmm24,zmm22\t\n" "vbroadcastss zmm22,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm23,zmm22\t\n" "vfmadd231ps zmm3,zmm24,zmm22\t\n" "vbroadcastss zmm22,DWORD PTR [r9+8]\t\n" "vfmadd231ps zmm4,zmm23,zmm22\t\n" "vfmadd231ps zmm5,zmm24,zmm22\t\n" "vbroadcastss zmm22,DWORD PTR [r9+12]\t\n" "vfmadd231ps zmm6,zmm23,zmm22\t\n" "vfmadd231ps zmm7,zmm24,zmm22\t\n" "vbroadcastss zmm22,DWORD PTR [r9+16]\t\n" "vfmadd231ps zmm8,zmm23,zmm22\t\n" "vfmadd231ps zmm9,zmm24,zmm22\t\n" "vbroadcastss zmm22,DWORD PTR [r9+20]\t\n" "vfmadd231ps zmm10,zmm23,zmm22\t\n" "vfmadd231ps zmm11,zmm24,zmm22\t\n" "vbroadcastss zmm22,DWORD PTR [r9+24]\t\n" "vfmadd231ps zmm12,zmm23,zmm22\t\n" "vfmadd231ps zmm13,zmm24,zmm22\t\n" "vbroadcastss zmm22,DWORD PTR [r9+28]\t\n" "vfmadd231ps zmm14,zmm23,zmm22\t\n" "vfmadd231ps zmm15,zmm24,zmm22\t\n" "vbroadcastss zmm22,DWORD PTR [r9+32]\t\n" "vfmadd231ps zmm16,zmm23,zmm22\t\n" "vfmadd231ps zmm17,zmm24,zmm22\t\n" "vbroadcastss zmm22,DWORD PTR [r9+36]\t\n" "vfmadd231ps zmm18,zmm23,zmm22\t\n" "vfmadd231ps zmm19,zmm24,zmm22\t\n" "vbroadcastss zmm22,DWORD PTR [r9+40]\t\n" "vfmadd231ps zmm20,zmm23,zmm22\t\n" "vfmadd231ps zmm21,zmm24,zmm22\t\n" "mov r12, rcx\t\n" "test r14,r14\t\n" "jnz next_inner%=\t\n" "add r10,64\t\n" "jmp dump_C%=\t\n" "zero_regs%=:\t\n" "test r14,r14\t\n" "jz skip_preload_b_zero%=\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "skip_preload_b_zero%=:\t\n" "vbroadcastss zmm22,DWORD PTR [r9+0]\t\n" "vmulps zmm0,zmm23,zmm22\t\n" "vmulps zmm1,zmm24,zmm22\t\n" "add r12, r13\t\n" "vbroadcastss zmm22,DWORD PTR [r9+4]\t\n" "vmulps zmm2,zmm23,zmm22\t\n" "vmulps zmm3,zmm24,zmm22\t\n" "add r12, r13\t\n" "vbroadcastss zmm22,DWORD PTR [r9+8]\t\n" "vmulps zmm4,zmm23,zmm22\t\n" "vmulps zmm5,zmm24,zmm22\t\n" "add r12, r13\t\n" "vbroadcastss zmm22,DWORD PTR [r9+12]\t\n" "vmulps zmm6,zmm23,zmm22\t\n" "vmulps zmm7,zmm24,zmm22\t\n" "add r12, r13\t\n" "vbroadcastss zmm22,DWORD PTR [r9+16]\t\n" "vmulps zmm8,zmm23,zmm22\t\n" "vmulps zmm9,zmm24,zmm22\t\n" "add r12, r13\t\n" "vbroadcastss zmm22,DWORD PTR [r9+20]\t\n" "vmulps zmm10,zmm23,zmm22\t\n" "vmulps zmm11,zmm24,zmm22\t\n" "add r12, r13\t\n" "vbroadcastss zmm22,DWORD PTR [r9+24]\t\n" "vmulps zmm12,zmm23,zmm22\t\n" "vmulps zmm13,zmm24,zmm22\t\n" "add r12, r13\t\n" "vbroadcastss zmm22,DWORD PTR [r9+28]\t\n" "vmulps zmm14,zmm23,zmm22\t\n" "vmulps zmm15,zmm24,zmm22\t\n" "add r12, r13\t\n" "vbroadcastss zmm22,DWORD PTR [r9+32]\t\n" "vmulps zmm16,zmm23,zmm22\t\n" "vmulps zmm17,zmm24,zmm22\t\n" "add r12, r13\t\n" "vbroadcastss zmm22,DWORD PTR [r9+36]\t\n" "vmulps zmm18,zmm23,zmm22\t\n" "vmulps zmm19,zmm24,zmm22\t\n" "add r12, r13\t\n" "vbroadcastss zmm22,DWORD PTR [r9+40]\t\n" "vmulps zmm20,zmm23,zmm22\t\n" "vmulps zmm21,zmm24,zmm22\t\n" "mov r12, rcx\t\n" "test r14,r14\t\n" "jnz next_inner%=\t\n" "add r10,64\t\n" "jmp dump_C%=\t\n" "loop_inner%=:\t\n" "vmovaps zmm23,zmm31\t\n" "vcvtph2ps zmm24,YMMWORD PTR [r10 + 32]\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "vbroadcastss zmm22,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm23,zmm22\t\n" "vfmadd231ps zmm1,zmm24,zmm22\t\n" "vbroadcastss zmm22,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm23,zmm22\t\n" "vfmadd231ps zmm3,zmm24,zmm22\t\n" "vbroadcastss zmm22,DWORD PTR [r9+8]\t\n" "vfmadd231ps zmm4,zmm23,zmm22\t\n" "vfmadd231ps zmm5,zmm24,zmm22\t\n" "vbroadcastss zmm22,DWORD PTR [r9+12]\t\n" "vfmadd231ps zmm6,zmm23,zmm22\t\n" "vfmadd231ps zmm7,zmm24,zmm22\t\n" "vbroadcastss zmm22,DWORD PTR [r9+16]\t\n" "vfmadd231ps zmm8,zmm23,zmm22\t\n" "vfmadd231ps zmm9,zmm24,zmm22\t\n" "vbroadcastss zmm22,DWORD PTR [r9+20]\t\n" "vfmadd231ps zmm10,zmm23,zmm22\t\n" "vfmadd231ps zmm11,zmm24,zmm22\t\n" "vbroadcastss zmm22,DWORD PTR [r9+24]\t\n" "vfmadd231ps zmm12,zmm23,zmm22\t\n" "vfmadd231ps zmm13,zmm24,zmm22\t\n" "vbroadcastss zmm22,DWORD PTR [r9+28]\t\n" "vfmadd231ps zmm14,zmm23,zmm22\t\n" "vfmadd231ps zmm15,zmm24,zmm22\t\n" "vbroadcastss zmm22,DWORD PTR [r9+32]\t\n" "vfmadd231ps zmm16,zmm23,zmm22\t\n" "vfmadd231ps zmm17,zmm24,zmm22\t\n" "vbroadcastss zmm22,DWORD PTR [r9+36]\t\n" "vfmadd231ps zmm18,zmm23,zmm22\t\n" "vfmadd231ps zmm19,zmm24,zmm22\t\n" "vbroadcastss zmm22,DWORD PTR [r9+40]\t\n" "vfmadd231ps zmm20,zmm23,zmm22\t\n" "vfmadd231ps zmm21,zmm24,zmm22\t\n" "next_inner%=:\t\n" "add r9,44\t\n" "add r10,64\t\n" "dec r14\t\n" "jnz loop_inner%=\t\n" "vmovaps zmm23,zmm31\t\n" "vcvtph2ps zmm24,YMMWORD PTR [r10 + 32]\t\n" "vbroadcastss zmm22,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm23,zmm22\t\n" "vfmadd231ps zmm1,zmm24,zmm22\t\n" "vbroadcastss zmm22,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm23,zmm22\t\n" "vfmadd231ps zmm3,zmm24,zmm22\t\n" "vbroadcastss zmm22,DWORD PTR [r9+8]\t\n" "vfmadd231ps zmm4,zmm23,zmm22\t\n" "vfmadd231ps zmm5,zmm24,zmm22\t\n" "vbroadcastss zmm22,DWORD PTR [r9+12]\t\n" "vfmadd231ps zmm6,zmm23,zmm22\t\n" "vfmadd231ps zmm7,zmm24,zmm22\t\n" "vbroadcastss zmm22,DWORD PTR [r9+16]\t\n" "vfmadd231ps zmm8,zmm23,zmm22\t\n" "vfmadd231ps zmm9,zmm24,zmm22\t\n" "vbroadcastss zmm22,DWORD PTR [r9+20]\t\n" "vfmadd231ps zmm10,zmm23,zmm22\t\n" "vfmadd231ps zmm11,zmm24,zmm22\t\n" "vbroadcastss zmm22,DWORD PTR [r9+24]\t\n" "vfmadd231ps zmm12,zmm23,zmm22\t\n" "vfmadd231ps zmm13,zmm24,zmm22\t\n" "vbroadcastss zmm22,DWORD PTR [r9+28]\t\n" "vfmadd231ps zmm14,zmm23,zmm22\t\n" "vfmadd231ps zmm15,zmm24,zmm22\t\n" "vbroadcastss zmm22,DWORD PTR [r9+32]\t\n" "vfmadd231ps zmm16,zmm23,zmm22\t\n" "vfmadd231ps zmm17,zmm24,zmm22\t\n" "vbroadcastss zmm22,DWORD PTR [r9+36]\t\n" "vfmadd231ps zmm18,zmm23,zmm22\t\n" "vfmadd231ps zmm19,zmm24,zmm22\t\n" "vbroadcastss zmm22,DWORD PTR [r9+40]\t\n" "vfmadd231ps zmm20,zmm23,zmm22\t\n" "vfmadd231ps zmm21,zmm24,zmm22\t\n" "add r9,44\t\n" "add r10,64\t\n" // Dump C "dump_C%=:\t\n" "vmovups zmmword PTR [r12 + 0], zmm0\t\n" "vmovups zmmword PTR [r12 + 64], zmm1\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm2\t\n" "vmovups zmmword PTR [r12 + 64], zmm3\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm4\t\n" "vmovups zmmword PTR [r12 + 64], zmm5\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm6\t\n" "vmovups zmmword PTR [r12 + 64], zmm7\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm8\t\n" "vmovups zmmword PTR [r12 + 64], zmm9\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm10\t\n" "vmovups zmmword PTR [r12 + 64], zmm11\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm12\t\n" "vmovups zmmword PTR [r12 + 64], zmm13\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm14\t\n" "vmovups zmmword PTR [r12 + 64], zmm15\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm16\t\n" "vmovups zmmword PTR [r12 + 64], zmm17\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm18\t\n" "vmovups zmmword PTR [r12 + 64], zmm19\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm20\t\n" "vmovups zmmword PTR [r12 + 64], zmm21\t\n" // next outer iteration "add rcx, 128\t\n" "mov r12, rcx\t\n" "mov r9, rax\t\n" "inc rbx\t\n" "cmp rbx, rdi\t\n" "jl loop_outter%=\t\n" : : [gp] "rm"(gp) : "r8", "r9", "r10", "r11", "r13", "r14", "rax", "rcx", "rsi", "rdi", "rbx", "r12", "r15", "memory"); } void NOINLINE gemmkernel_12x2_Avx512_fp16_fA0fB0fC0(GemmParamsFP16* gp) { asm volatile( #if FBGEMM_USE_CLANG_INTEL_SYNTAX_ASM_HACK "mov %[gp], %%r14\t\n" ".intel_syntax noprefix\t\n" #else "mov r14, %[gp]\t\n" #endif // Copy parameters // k "mov r8, [r14 + 0]\t\n" "dec r8\t\n" // A "mov r9, [r14 + 8]\t\n" // B "mov r10, [r14 + 16]\t\n" // beta "lea r15, [r14 + 24]\t\n" // C "mov r12, [r14 + 32]\t\n" // ldc "mov r13, [r14 + 40]\t\n" // b_block_cols "mov rdi, [r14 + 48]\t\n" // b_block_size "mov rsi, [r14 + 56]\t\n" // Make copies of A and C "mov rax, r9\t\n" "mov rcx, r12\t\n" "xor ebx, ebx\t\n" "loop_outter%=:\t\n" "mov r14, r8\t\n" "vbroadcastss zmm31,DWORD PTR [r15]\t\n" "vcvtph2ps zmm25,YMMWORD PTR [r10 + 0]\t\n" "vcvtph2ps zmm26,YMMWORD PTR [r10 + 32]\t\n" "vxorps xmm0, xmm0, xmm0\t\n" "vcomiss xmm31, xmm0\t\n" "jz zero_regs%=\t\n" // Setup values with beta multiplication "vmulps zmm0, zmm31, [r12 + 0]\t\n" "vmulps zmm1, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm2, zmm31, [r12 + 0]\t\n" "vmulps zmm3, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm4, zmm31, [r12 + 0]\t\n" "vmulps zmm5, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm6, zmm31, [r12 + 0]\t\n" "vmulps zmm7, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm8, zmm31, [r12 + 0]\t\n" "vmulps zmm9, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm10, zmm31, [r12 + 0]\t\n" "vmulps zmm11, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm12, zmm31, [r12 + 0]\t\n" "vmulps zmm13, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm14, zmm31, [r12 + 0]\t\n" "vmulps zmm15, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm16, zmm31, [r12 + 0]\t\n" "vmulps zmm17, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm18, zmm31, [r12 + 0]\t\n" "vmulps zmm19, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm20, zmm31, [r12 + 0]\t\n" "vmulps zmm21, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm22, zmm31, [r12 + 0]\t\n" "vmulps zmm23, zmm31, [r12 + 64]\t\n" "test r14,r14\t\n" "jz skip_preload%=\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "skip_preload%=:\t\n" "vbroadcastss zmm24,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm25,zmm24\t\n" "vfmadd231ps zmm1,zmm26,zmm24\t\n" "vbroadcastss zmm24,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm25,zmm24\t\n" "vfmadd231ps zmm3,zmm26,zmm24\t\n" "vbroadcastss zmm24,DWORD PTR [r9+8]\t\n" "vfmadd231ps zmm4,zmm25,zmm24\t\n" "vfmadd231ps zmm5,zmm26,zmm24\t\n" "vbroadcastss zmm24,DWORD PTR [r9+12]\t\n" "vfmadd231ps zmm6,zmm25,zmm24\t\n" "vfmadd231ps zmm7,zmm26,zmm24\t\n" "vbroadcastss zmm24,DWORD PTR [r9+16]\t\n" "vfmadd231ps zmm8,zmm25,zmm24\t\n" "vfmadd231ps zmm9,zmm26,zmm24\t\n" "vbroadcastss zmm24,DWORD PTR [r9+20]\t\n" "vfmadd231ps zmm10,zmm25,zmm24\t\n" "vfmadd231ps zmm11,zmm26,zmm24\t\n" "vbroadcastss zmm24,DWORD PTR [r9+24]\t\n" "vfmadd231ps zmm12,zmm25,zmm24\t\n" "vfmadd231ps zmm13,zmm26,zmm24\t\n" "vbroadcastss zmm24,DWORD PTR [r9+28]\t\n" "vfmadd231ps zmm14,zmm25,zmm24\t\n" "vfmadd231ps zmm15,zmm26,zmm24\t\n" "vbroadcastss zmm24,DWORD PTR [r9+32]\t\n" "vfmadd231ps zmm16,zmm25,zmm24\t\n" "vfmadd231ps zmm17,zmm26,zmm24\t\n" "vbroadcastss zmm24,DWORD PTR [r9+36]\t\n" "vfmadd231ps zmm18,zmm25,zmm24\t\n" "vfmadd231ps zmm19,zmm26,zmm24\t\n" "vbroadcastss zmm24,DWORD PTR [r9+40]\t\n" "vfmadd231ps zmm20,zmm25,zmm24\t\n" "vfmadd231ps zmm21,zmm26,zmm24\t\n" "vbroadcastss zmm24,DWORD PTR [r9+44]\t\n" "vfmadd231ps zmm22,zmm25,zmm24\t\n" "vfmadd231ps zmm23,zmm26,zmm24\t\n" "mov r12, rcx\t\n" "test r14,r14\t\n" "jnz next_inner%=\t\n" "add r10,64\t\n" "jmp dump_C%=\t\n" "zero_regs%=:\t\n" "test r14,r14\t\n" "jz skip_preload_b_zero%=\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "skip_preload_b_zero%=:\t\n" "vbroadcastss zmm24,DWORD PTR [r9+0]\t\n" "vmulps zmm0,zmm25,zmm24\t\n" "vmulps zmm1,zmm26,zmm24\t\n" "add r12, r13\t\n" "vbroadcastss zmm24,DWORD PTR [r9+4]\t\n" "vmulps zmm2,zmm25,zmm24\t\n" "vmulps zmm3,zmm26,zmm24\t\n" "add r12, r13\t\n" "vbroadcastss zmm24,DWORD PTR [r9+8]\t\n" "vmulps zmm4,zmm25,zmm24\t\n" "vmulps zmm5,zmm26,zmm24\t\n" "add r12, r13\t\n" "vbroadcastss zmm24,DWORD PTR [r9+12]\t\n" "vmulps zmm6,zmm25,zmm24\t\n" "vmulps zmm7,zmm26,zmm24\t\n" "add r12, r13\t\n" "vbroadcastss zmm24,DWORD PTR [r9+16]\t\n" "vmulps zmm8,zmm25,zmm24\t\n" "vmulps zmm9,zmm26,zmm24\t\n" "add r12, r13\t\n" "vbroadcastss zmm24,DWORD PTR [r9+20]\t\n" "vmulps zmm10,zmm25,zmm24\t\n" "vmulps zmm11,zmm26,zmm24\t\n" "add r12, r13\t\n" "vbroadcastss zmm24,DWORD PTR [r9+24]\t\n" "vmulps zmm12,zmm25,zmm24\t\n" "vmulps zmm13,zmm26,zmm24\t\n" "add r12, r13\t\n" "vbroadcastss zmm24,DWORD PTR [r9+28]\t\n" "vmulps zmm14,zmm25,zmm24\t\n" "vmulps zmm15,zmm26,zmm24\t\n" "add r12, r13\t\n" "vbroadcastss zmm24,DWORD PTR [r9+32]\t\n" "vmulps zmm16,zmm25,zmm24\t\n" "vmulps zmm17,zmm26,zmm24\t\n" "add r12, r13\t\n" "vbroadcastss zmm24,DWORD PTR [r9+36]\t\n" "vmulps zmm18,zmm25,zmm24\t\n" "vmulps zmm19,zmm26,zmm24\t\n" "add r12, r13\t\n" "vbroadcastss zmm24,DWORD PTR [r9+40]\t\n" "vmulps zmm20,zmm25,zmm24\t\n" "vmulps zmm21,zmm26,zmm24\t\n" "add r12, r13\t\n" "vbroadcastss zmm24,DWORD PTR [r9+44]\t\n" "vmulps zmm22,zmm25,zmm24\t\n" "vmulps zmm23,zmm26,zmm24\t\n" "mov r12, rcx\t\n" "test r14,r14\t\n" "jnz next_inner%=\t\n" "add r10,64\t\n" "jmp dump_C%=\t\n" "loop_inner%=:\t\n" "vmovaps zmm25,zmm31\t\n" "vcvtph2ps zmm26,YMMWORD PTR [r10 + 32]\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "vbroadcastss zmm24,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm25,zmm24\t\n" "vfmadd231ps zmm1,zmm26,zmm24\t\n" "vbroadcastss zmm24,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm25,zmm24\t\n" "vfmadd231ps zmm3,zmm26,zmm24\t\n" "vbroadcastss zmm24,DWORD PTR [r9+8]\t\n" "vfmadd231ps zmm4,zmm25,zmm24\t\n" "vfmadd231ps zmm5,zmm26,zmm24\t\n" "vbroadcastss zmm24,DWORD PTR [r9+12]\t\n" "vfmadd231ps zmm6,zmm25,zmm24\t\n" "vfmadd231ps zmm7,zmm26,zmm24\t\n" "vbroadcastss zmm24,DWORD PTR [r9+16]\t\n" "vfmadd231ps zmm8,zmm25,zmm24\t\n" "vfmadd231ps zmm9,zmm26,zmm24\t\n" "vbroadcastss zmm24,DWORD PTR [r9+20]\t\n" "vfmadd231ps zmm10,zmm25,zmm24\t\n" "vfmadd231ps zmm11,zmm26,zmm24\t\n" "vbroadcastss zmm24,DWORD PTR [r9+24]\t\n" "vfmadd231ps zmm12,zmm25,zmm24\t\n" "vfmadd231ps zmm13,zmm26,zmm24\t\n" "vbroadcastss zmm24,DWORD PTR [r9+28]\t\n" "vfmadd231ps zmm14,zmm25,zmm24\t\n" "vfmadd231ps zmm15,zmm26,zmm24\t\n" "vbroadcastss zmm24,DWORD PTR [r9+32]\t\n" "vfmadd231ps zmm16,zmm25,zmm24\t\n" "vfmadd231ps zmm17,zmm26,zmm24\t\n" "vbroadcastss zmm24,DWORD PTR [r9+36]\t\n" "vfmadd231ps zmm18,zmm25,zmm24\t\n" "vfmadd231ps zmm19,zmm26,zmm24\t\n" "vbroadcastss zmm24,DWORD PTR [r9+40]\t\n" "vfmadd231ps zmm20,zmm25,zmm24\t\n" "vfmadd231ps zmm21,zmm26,zmm24\t\n" "vbroadcastss zmm24,DWORD PTR [r9+44]\t\n" "vfmadd231ps zmm22,zmm25,zmm24\t\n" "vfmadd231ps zmm23,zmm26,zmm24\t\n" "next_inner%=:\t\n" "add r9,48\t\n" "add r10,64\t\n" "dec r14\t\n" "jnz loop_inner%=\t\n" "vmovaps zmm25,zmm31\t\n" "vcvtph2ps zmm26,YMMWORD PTR [r10 + 32]\t\n" "vbroadcastss zmm24,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm25,zmm24\t\n" "vfmadd231ps zmm1,zmm26,zmm24\t\n" "vbroadcastss zmm24,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm25,zmm24\t\n" "vfmadd231ps zmm3,zmm26,zmm24\t\n" "vbroadcastss zmm24,DWORD PTR [r9+8]\t\n" "vfmadd231ps zmm4,zmm25,zmm24\t\n" "vfmadd231ps zmm5,zmm26,zmm24\t\n" "vbroadcastss zmm24,DWORD PTR [r9+12]\t\n" "vfmadd231ps zmm6,zmm25,zmm24\t\n" "vfmadd231ps zmm7,zmm26,zmm24\t\n" "vbroadcastss zmm24,DWORD PTR [r9+16]\t\n" "vfmadd231ps zmm8,zmm25,zmm24\t\n" "vfmadd231ps zmm9,zmm26,zmm24\t\n" "vbroadcastss zmm24,DWORD PTR [r9+20]\t\n" "vfmadd231ps zmm10,zmm25,zmm24\t\n" "vfmadd231ps zmm11,zmm26,zmm24\t\n" "vbroadcastss zmm24,DWORD PTR [r9+24]\t\n" "vfmadd231ps zmm12,zmm25,zmm24\t\n" "vfmadd231ps zmm13,zmm26,zmm24\t\n" "vbroadcastss zmm24,DWORD PTR [r9+28]\t\n" "vfmadd231ps zmm14,zmm25,zmm24\t\n" "vfmadd231ps zmm15,zmm26,zmm24\t\n" "vbroadcastss zmm24,DWORD PTR [r9+32]\t\n" "vfmadd231ps zmm16,zmm25,zmm24\t\n" "vfmadd231ps zmm17,zmm26,zmm24\t\n" "vbroadcastss zmm24,DWORD PTR [r9+36]\t\n" "vfmadd231ps zmm18,zmm25,zmm24\t\n" "vfmadd231ps zmm19,zmm26,zmm24\t\n" "vbroadcastss zmm24,DWORD PTR [r9+40]\t\n" "vfmadd231ps zmm20,zmm25,zmm24\t\n" "vfmadd231ps zmm21,zmm26,zmm24\t\n" "vbroadcastss zmm24,DWORD PTR [r9+44]\t\n" "vfmadd231ps zmm22,zmm25,zmm24\t\n" "vfmadd231ps zmm23,zmm26,zmm24\t\n" "add r9,48\t\n" "add r10,64\t\n" // Dump C "dump_C%=:\t\n" "vmovups zmmword PTR [r12 + 0], zmm0\t\n" "vmovups zmmword PTR [r12 + 64], zmm1\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm2\t\n" "vmovups zmmword PTR [r12 + 64], zmm3\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm4\t\n" "vmovups zmmword PTR [r12 + 64], zmm5\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm6\t\n" "vmovups zmmword PTR [r12 + 64], zmm7\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm8\t\n" "vmovups zmmword PTR [r12 + 64], zmm9\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm10\t\n" "vmovups zmmword PTR [r12 + 64], zmm11\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm12\t\n" "vmovups zmmword PTR [r12 + 64], zmm13\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm14\t\n" "vmovups zmmword PTR [r12 + 64], zmm15\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm16\t\n" "vmovups zmmword PTR [r12 + 64], zmm17\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm18\t\n" "vmovups zmmword PTR [r12 + 64], zmm19\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm20\t\n" "vmovups zmmword PTR [r12 + 64], zmm21\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm22\t\n" "vmovups zmmword PTR [r12 + 64], zmm23\t\n" // next outer iteration "add rcx, 128\t\n" "mov r12, rcx\t\n" "mov r9, rax\t\n" "inc rbx\t\n" "cmp rbx, rdi\t\n" "jl loop_outter%=\t\n" : : [gp] "rm"(gp) : "r8", "r9", "r10", "r11", "r13", "r14", "rax", "rcx", "rsi", "rdi", "rbx", "r12", "r15", "memory"); } void NOINLINE gemmkernel_13x2_Avx512_fp16_fA0fB0fC0(GemmParamsFP16* gp) { asm volatile( #if FBGEMM_USE_CLANG_INTEL_SYNTAX_ASM_HACK "mov %[gp], %%r14\t\n" ".intel_syntax noprefix\t\n" #else "mov r14, %[gp]\t\n" #endif // Copy parameters // k "mov r8, [r14 + 0]\t\n" "dec r8\t\n" // A "mov r9, [r14 + 8]\t\n" // B "mov r10, [r14 + 16]\t\n" // beta "lea r15, [r14 + 24]\t\n" // C "mov r12, [r14 + 32]\t\n" // ldc "mov r13, [r14 + 40]\t\n" // b_block_cols "mov rdi, [r14 + 48]\t\n" // b_block_size "mov rsi, [r14 + 56]\t\n" // Make copies of A and C "mov rax, r9\t\n" "mov rcx, r12\t\n" "xor ebx, ebx\t\n" "loop_outter%=:\t\n" "mov r14, r8\t\n" "vbroadcastss zmm31,DWORD PTR [r15]\t\n" "vcvtph2ps zmm27,YMMWORD PTR [r10 + 0]\t\n" "vcvtph2ps zmm28,YMMWORD PTR [r10 + 32]\t\n" "vxorps xmm0, xmm0, xmm0\t\n" "vcomiss xmm31, xmm0\t\n" "jz zero_regs%=\t\n" // Setup values with beta multiplication "vmulps zmm0, zmm31, [r12 + 0]\t\n" "vmulps zmm1, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm2, zmm31, [r12 + 0]\t\n" "vmulps zmm3, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm4, zmm31, [r12 + 0]\t\n" "vmulps zmm5, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm6, zmm31, [r12 + 0]\t\n" "vmulps zmm7, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm8, zmm31, [r12 + 0]\t\n" "vmulps zmm9, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm10, zmm31, [r12 + 0]\t\n" "vmulps zmm11, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm12, zmm31, [r12 + 0]\t\n" "vmulps zmm13, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm14, zmm31, [r12 + 0]\t\n" "vmulps zmm15, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm16, zmm31, [r12 + 0]\t\n" "vmulps zmm17, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm18, zmm31, [r12 + 0]\t\n" "vmulps zmm19, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm20, zmm31, [r12 + 0]\t\n" "vmulps zmm21, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm22, zmm31, [r12 + 0]\t\n" "vmulps zmm23, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm24, zmm31, [r12 + 0]\t\n" "vmulps zmm25, zmm31, [r12 + 64]\t\n" "test r14,r14\t\n" "jz skip_preload%=\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "skip_preload%=:\t\n" "vbroadcastss zmm26,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm27,zmm26\t\n" "vfmadd231ps zmm1,zmm28,zmm26\t\n" "vbroadcastss zmm26,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm27,zmm26\t\n" "vfmadd231ps zmm3,zmm28,zmm26\t\n" "vbroadcastss zmm26,DWORD PTR [r9+8]\t\n" "vfmadd231ps zmm4,zmm27,zmm26\t\n" "vfmadd231ps zmm5,zmm28,zmm26\t\n" "vbroadcastss zmm26,DWORD PTR [r9+12]\t\n" "vfmadd231ps zmm6,zmm27,zmm26\t\n" "vfmadd231ps zmm7,zmm28,zmm26\t\n" "vbroadcastss zmm26,DWORD PTR [r9+16]\t\n" "vfmadd231ps zmm8,zmm27,zmm26\t\n" "vfmadd231ps zmm9,zmm28,zmm26\t\n" "vbroadcastss zmm26,DWORD PTR [r9+20]\t\n" "vfmadd231ps zmm10,zmm27,zmm26\t\n" "vfmadd231ps zmm11,zmm28,zmm26\t\n" "vbroadcastss zmm26,DWORD PTR [r9+24]\t\n" "vfmadd231ps zmm12,zmm27,zmm26\t\n" "vfmadd231ps zmm13,zmm28,zmm26\t\n" "vbroadcastss zmm26,DWORD PTR [r9+28]\t\n" "vfmadd231ps zmm14,zmm27,zmm26\t\n" "vfmadd231ps zmm15,zmm28,zmm26\t\n" "vbroadcastss zmm26,DWORD PTR [r9+32]\t\n" "vfmadd231ps zmm16,zmm27,zmm26\t\n" "vfmadd231ps zmm17,zmm28,zmm26\t\n" "vbroadcastss zmm26,DWORD PTR [r9+36]\t\n" "vfmadd231ps zmm18,zmm27,zmm26\t\n" "vfmadd231ps zmm19,zmm28,zmm26\t\n" "vbroadcastss zmm26,DWORD PTR [r9+40]\t\n" "vfmadd231ps zmm20,zmm27,zmm26\t\n" "vfmadd231ps zmm21,zmm28,zmm26\t\n" "vbroadcastss zmm26,DWORD PTR [r9+44]\t\n" "vfmadd231ps zmm22,zmm27,zmm26\t\n" "vfmadd231ps zmm23,zmm28,zmm26\t\n" "vbroadcastss zmm26,DWORD PTR [r9+48]\t\n" "vfmadd231ps zmm24,zmm27,zmm26\t\n" "vfmadd231ps zmm25,zmm28,zmm26\t\n" "mov r12, rcx\t\n" "test r14,r14\t\n" "jnz next_inner%=\t\n" "add r10,64\t\n" "jmp dump_C%=\t\n" "zero_regs%=:\t\n" "test r14,r14\t\n" "jz skip_preload_b_zero%=\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "skip_preload_b_zero%=:\t\n" "vbroadcastss zmm26,DWORD PTR [r9+0]\t\n" "vmulps zmm0,zmm27,zmm26\t\n" "vmulps zmm1,zmm28,zmm26\t\n" "add r12, r13\t\n" "vbroadcastss zmm26,DWORD PTR [r9+4]\t\n" "vmulps zmm2,zmm27,zmm26\t\n" "vmulps zmm3,zmm28,zmm26\t\n" "add r12, r13\t\n" "vbroadcastss zmm26,DWORD PTR [r9+8]\t\n" "vmulps zmm4,zmm27,zmm26\t\n" "vmulps zmm5,zmm28,zmm26\t\n" "add r12, r13\t\n" "vbroadcastss zmm26,DWORD PTR [r9+12]\t\n" "vmulps zmm6,zmm27,zmm26\t\n" "vmulps zmm7,zmm28,zmm26\t\n" "add r12, r13\t\n" "vbroadcastss zmm26,DWORD PTR [r9+16]\t\n" "vmulps zmm8,zmm27,zmm26\t\n" "vmulps zmm9,zmm28,zmm26\t\n" "add r12, r13\t\n" "vbroadcastss zmm26,DWORD PTR [r9+20]\t\n" "vmulps zmm10,zmm27,zmm26\t\n" "vmulps zmm11,zmm28,zmm26\t\n" "add r12, r13\t\n" "vbroadcastss zmm26,DWORD PTR [r9+24]\t\n" "vmulps zmm12,zmm27,zmm26\t\n" "vmulps zmm13,zmm28,zmm26\t\n" "add r12, r13\t\n" "vbroadcastss zmm26,DWORD PTR [r9+28]\t\n" "vmulps zmm14,zmm27,zmm26\t\n" "vmulps zmm15,zmm28,zmm26\t\n" "add r12, r13\t\n" "vbroadcastss zmm26,DWORD PTR [r9+32]\t\n" "vmulps zmm16,zmm27,zmm26\t\n" "vmulps zmm17,zmm28,zmm26\t\n" "add r12, r13\t\n" "vbroadcastss zmm26,DWORD PTR [r9+36]\t\n" "vmulps zmm18,zmm27,zmm26\t\n" "vmulps zmm19,zmm28,zmm26\t\n" "add r12, r13\t\n" "vbroadcastss zmm26,DWORD PTR [r9+40]\t\n" "vmulps zmm20,zmm27,zmm26\t\n" "vmulps zmm21,zmm28,zmm26\t\n" "add r12, r13\t\n" "vbroadcastss zmm26,DWORD PTR [r9+44]\t\n" "vmulps zmm22,zmm27,zmm26\t\n" "vmulps zmm23,zmm28,zmm26\t\n" "add r12, r13\t\n" "vbroadcastss zmm26,DWORD PTR [r9+48]\t\n" "vmulps zmm24,zmm27,zmm26\t\n" "vmulps zmm25,zmm28,zmm26\t\n" "mov r12, rcx\t\n" "test r14,r14\t\n" "jnz next_inner%=\t\n" "add r10,64\t\n" "jmp dump_C%=\t\n" "loop_inner%=:\t\n" "vmovaps zmm27,zmm31\t\n" "vcvtph2ps zmm28,YMMWORD PTR [r10 + 32]\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "vbroadcastss zmm26,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm27,zmm26\t\n" "vfmadd231ps zmm1,zmm28,zmm26\t\n" "vbroadcastss zmm26,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm27,zmm26\t\n" "vfmadd231ps zmm3,zmm28,zmm26\t\n" "vbroadcastss zmm26,DWORD PTR [r9+8]\t\n" "vfmadd231ps zmm4,zmm27,zmm26\t\n" "vfmadd231ps zmm5,zmm28,zmm26\t\n" "vbroadcastss zmm26,DWORD PTR [r9+12]\t\n" "vfmadd231ps zmm6,zmm27,zmm26\t\n" "vfmadd231ps zmm7,zmm28,zmm26\t\n" "vbroadcastss zmm26,DWORD PTR [r9+16]\t\n" "vfmadd231ps zmm8,zmm27,zmm26\t\n" "vfmadd231ps zmm9,zmm28,zmm26\t\n" "vbroadcastss zmm26,DWORD PTR [r9+20]\t\n" "vfmadd231ps zmm10,zmm27,zmm26\t\n" "vfmadd231ps zmm11,zmm28,zmm26\t\n" "vbroadcastss zmm26,DWORD PTR [r9+24]\t\n" "vfmadd231ps zmm12,zmm27,zmm26\t\n" "vfmadd231ps zmm13,zmm28,zmm26\t\n" "vbroadcastss zmm26,DWORD PTR [r9+28]\t\n" "vfmadd231ps zmm14,zmm27,zmm26\t\n" "vfmadd231ps zmm15,zmm28,zmm26\t\n" "vbroadcastss zmm26,DWORD PTR [r9+32]\t\n" "vfmadd231ps zmm16,zmm27,zmm26\t\n" "vfmadd231ps zmm17,zmm28,zmm26\t\n" "vbroadcastss zmm26,DWORD PTR [r9+36]\t\n" "vfmadd231ps zmm18,zmm27,zmm26\t\n" "vfmadd231ps zmm19,zmm28,zmm26\t\n" "vbroadcastss zmm26,DWORD PTR [r9+40]\t\n" "vfmadd231ps zmm20,zmm27,zmm26\t\n" "vfmadd231ps zmm21,zmm28,zmm26\t\n" "vbroadcastss zmm26,DWORD PTR [r9+44]\t\n" "vfmadd231ps zmm22,zmm27,zmm26\t\n" "vfmadd231ps zmm23,zmm28,zmm26\t\n" "vbroadcastss zmm26,DWORD PTR [r9+48]\t\n" "vfmadd231ps zmm24,zmm27,zmm26\t\n" "vfmadd231ps zmm25,zmm28,zmm26\t\n" "next_inner%=:\t\n" "add r9,52\t\n" "add r10,64\t\n" "dec r14\t\n" "jnz loop_inner%=\t\n" "vmovaps zmm27,zmm31\t\n" "vcvtph2ps zmm28,YMMWORD PTR [r10 + 32]\t\n" "vbroadcastss zmm26,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm27,zmm26\t\n" "vfmadd231ps zmm1,zmm28,zmm26\t\n" "vbroadcastss zmm26,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm27,zmm26\t\n" "vfmadd231ps zmm3,zmm28,zmm26\t\n" "vbroadcastss zmm26,DWORD PTR [r9+8]\t\n" "vfmadd231ps zmm4,zmm27,zmm26\t\n" "vfmadd231ps zmm5,zmm28,zmm26\t\n" "vbroadcastss zmm26,DWORD PTR [r9+12]\t\n" "vfmadd231ps zmm6,zmm27,zmm26\t\n" "vfmadd231ps zmm7,zmm28,zmm26\t\n" "vbroadcastss zmm26,DWORD PTR [r9+16]\t\n" "vfmadd231ps zmm8,zmm27,zmm26\t\n" "vfmadd231ps zmm9,zmm28,zmm26\t\n" "vbroadcastss zmm26,DWORD PTR [r9+20]\t\n" "vfmadd231ps zmm10,zmm27,zmm26\t\n" "vfmadd231ps zmm11,zmm28,zmm26\t\n" "vbroadcastss zmm26,DWORD PTR [r9+24]\t\n" "vfmadd231ps zmm12,zmm27,zmm26\t\n" "vfmadd231ps zmm13,zmm28,zmm26\t\n" "vbroadcastss zmm26,DWORD PTR [r9+28]\t\n" "vfmadd231ps zmm14,zmm27,zmm26\t\n" "vfmadd231ps zmm15,zmm28,zmm26\t\n" "vbroadcastss zmm26,DWORD PTR [r9+32]\t\n" "vfmadd231ps zmm16,zmm27,zmm26\t\n" "vfmadd231ps zmm17,zmm28,zmm26\t\n" "vbroadcastss zmm26,DWORD PTR [r9+36]\t\n" "vfmadd231ps zmm18,zmm27,zmm26\t\n" "vfmadd231ps zmm19,zmm28,zmm26\t\n" "vbroadcastss zmm26,DWORD PTR [r9+40]\t\n" "vfmadd231ps zmm20,zmm27,zmm26\t\n" "vfmadd231ps zmm21,zmm28,zmm26\t\n" "vbroadcastss zmm26,DWORD PTR [r9+44]\t\n" "vfmadd231ps zmm22,zmm27,zmm26\t\n" "vfmadd231ps zmm23,zmm28,zmm26\t\n" "vbroadcastss zmm26,DWORD PTR [r9+48]\t\n" "vfmadd231ps zmm24,zmm27,zmm26\t\n" "vfmadd231ps zmm25,zmm28,zmm26\t\n" "add r9,52\t\n" "add r10,64\t\n" // Dump C "dump_C%=:\t\n" "vmovups zmmword PTR [r12 + 0], zmm0\t\n" "vmovups zmmword PTR [r12 + 64], zmm1\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm2\t\n" "vmovups zmmword PTR [r12 + 64], zmm3\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm4\t\n" "vmovups zmmword PTR [r12 + 64], zmm5\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm6\t\n" "vmovups zmmword PTR [r12 + 64], zmm7\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm8\t\n" "vmovups zmmword PTR [r12 + 64], zmm9\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm10\t\n" "vmovups zmmword PTR [r12 + 64], zmm11\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm12\t\n" "vmovups zmmword PTR [r12 + 64], zmm13\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm14\t\n" "vmovups zmmword PTR [r12 + 64], zmm15\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm16\t\n" "vmovups zmmword PTR [r12 + 64], zmm17\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm18\t\n" "vmovups zmmword PTR [r12 + 64], zmm19\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm20\t\n" "vmovups zmmword PTR [r12 + 64], zmm21\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm22\t\n" "vmovups zmmword PTR [r12 + 64], zmm23\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm24\t\n" "vmovups zmmword PTR [r12 + 64], zmm25\t\n" // next outer iteration "add rcx, 128\t\n" "mov r12, rcx\t\n" "mov r9, rax\t\n" "inc rbx\t\n" "cmp rbx, rdi\t\n" "jl loop_outter%=\t\n" : : [gp] "rm"(gp) : "r8", "r9", "r10", "r11", "r13", "r14", "rax", "rcx", "rsi", "rdi", "rbx", "r12", "r15", "memory"); } void NOINLINE gemmkernel_14x2_Avx512_fp16_fA0fB0fC0(GemmParamsFP16* gp) { asm volatile( #if FBGEMM_USE_CLANG_INTEL_SYNTAX_ASM_HACK "mov %[gp], %%r14\t\n" ".intel_syntax noprefix\t\n" #else "mov r14, %[gp]\t\n" #endif // Copy parameters // k "mov r8, [r14 + 0]\t\n" "dec r8\t\n" // A "mov r9, [r14 + 8]\t\n" // B "mov r10, [r14 + 16]\t\n" // beta "lea r15, [r14 + 24]\t\n" // C "mov r12, [r14 + 32]\t\n" // ldc "mov r13, [r14 + 40]\t\n" // b_block_cols "mov rdi, [r14 + 48]\t\n" // b_block_size "mov rsi, [r14 + 56]\t\n" // Make copies of A and C "mov rax, r9\t\n" "mov rcx, r12\t\n" "xor ebx, ebx\t\n" "loop_outter%=:\t\n" "mov r14, r8\t\n" "vbroadcastss zmm31,DWORD PTR [r15]\t\n" "vcvtph2ps zmm29,YMMWORD PTR [r10 + 0]\t\n" "vcvtph2ps zmm30,YMMWORD PTR [r10 + 32]\t\n" "vxorps xmm0, xmm0, xmm0\t\n" "vcomiss xmm31, xmm0\t\n" "jz zero_regs%=\t\n" // Setup values with beta multiplication "vmulps zmm0, zmm31, [r12 + 0]\t\n" "vmulps zmm1, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm2, zmm31, [r12 + 0]\t\n" "vmulps zmm3, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm4, zmm31, [r12 + 0]\t\n" "vmulps zmm5, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm6, zmm31, [r12 + 0]\t\n" "vmulps zmm7, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm8, zmm31, [r12 + 0]\t\n" "vmulps zmm9, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm10, zmm31, [r12 + 0]\t\n" "vmulps zmm11, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm12, zmm31, [r12 + 0]\t\n" "vmulps zmm13, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm14, zmm31, [r12 + 0]\t\n" "vmulps zmm15, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm16, zmm31, [r12 + 0]\t\n" "vmulps zmm17, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm18, zmm31, [r12 + 0]\t\n" "vmulps zmm19, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm20, zmm31, [r12 + 0]\t\n" "vmulps zmm21, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm22, zmm31, [r12 + 0]\t\n" "vmulps zmm23, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm24, zmm31, [r12 + 0]\t\n" "vmulps zmm25, zmm31, [r12 + 64]\t\n" "add r12, r13\t\n" "vmulps zmm26, zmm31, [r12 + 0]\t\n" "vmulps zmm27, zmm31, [r12 + 64]\t\n" "test r14,r14\t\n" "jz skip_preload%=\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "skip_preload%=:\t\n" "vbroadcastss zmm28,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm29,zmm28\t\n" "vfmadd231ps zmm1,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm29,zmm28\t\n" "vfmadd231ps zmm3,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+8]\t\n" "vfmadd231ps zmm4,zmm29,zmm28\t\n" "vfmadd231ps zmm5,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+12]\t\n" "vfmadd231ps zmm6,zmm29,zmm28\t\n" "vfmadd231ps zmm7,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+16]\t\n" "vfmadd231ps zmm8,zmm29,zmm28\t\n" "vfmadd231ps zmm9,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+20]\t\n" "vfmadd231ps zmm10,zmm29,zmm28\t\n" "vfmadd231ps zmm11,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+24]\t\n" "vfmadd231ps zmm12,zmm29,zmm28\t\n" "vfmadd231ps zmm13,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+28]\t\n" "vfmadd231ps zmm14,zmm29,zmm28\t\n" "vfmadd231ps zmm15,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+32]\t\n" "vfmadd231ps zmm16,zmm29,zmm28\t\n" "vfmadd231ps zmm17,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+36]\t\n" "vfmadd231ps zmm18,zmm29,zmm28\t\n" "vfmadd231ps zmm19,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+40]\t\n" "vfmadd231ps zmm20,zmm29,zmm28\t\n" "vfmadd231ps zmm21,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+44]\t\n" "vfmadd231ps zmm22,zmm29,zmm28\t\n" "vfmadd231ps zmm23,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+48]\t\n" "vfmadd231ps zmm24,zmm29,zmm28\t\n" "vfmadd231ps zmm25,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+52]\t\n" "vfmadd231ps zmm26,zmm29,zmm28\t\n" "vfmadd231ps zmm27,zmm30,zmm28\t\n" "mov r12, rcx\t\n" "test r14,r14\t\n" "jnz next_inner%=\t\n" "add r10,64\t\n" "jmp dump_C%=\t\n" "zero_regs%=:\t\n" "test r14,r14\t\n" "jz skip_preload_b_zero%=\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "skip_preload_b_zero%=:\t\n" "vbroadcastss zmm28,DWORD PTR [r9+0]\t\n" "vmulps zmm0,zmm29,zmm28\t\n" "vmulps zmm1,zmm30,zmm28\t\n" "add r12, r13\t\n" "vbroadcastss zmm28,DWORD PTR [r9+4]\t\n" "vmulps zmm2,zmm29,zmm28\t\n" "vmulps zmm3,zmm30,zmm28\t\n" "add r12, r13\t\n" "vbroadcastss zmm28,DWORD PTR [r9+8]\t\n" "vmulps zmm4,zmm29,zmm28\t\n" "vmulps zmm5,zmm30,zmm28\t\n" "add r12, r13\t\n" "vbroadcastss zmm28,DWORD PTR [r9+12]\t\n" "vmulps zmm6,zmm29,zmm28\t\n" "vmulps zmm7,zmm30,zmm28\t\n" "add r12, r13\t\n" "vbroadcastss zmm28,DWORD PTR [r9+16]\t\n" "vmulps zmm8,zmm29,zmm28\t\n" "vmulps zmm9,zmm30,zmm28\t\n" "add r12, r13\t\n" "vbroadcastss zmm28,DWORD PTR [r9+20]\t\n" "vmulps zmm10,zmm29,zmm28\t\n" "vmulps zmm11,zmm30,zmm28\t\n" "add r12, r13\t\n" "vbroadcastss zmm28,DWORD PTR [r9+24]\t\n" "vmulps zmm12,zmm29,zmm28\t\n" "vmulps zmm13,zmm30,zmm28\t\n" "add r12, r13\t\n" "vbroadcastss zmm28,DWORD PTR [r9+28]\t\n" "vmulps zmm14,zmm29,zmm28\t\n" "vmulps zmm15,zmm30,zmm28\t\n" "add r12, r13\t\n" "vbroadcastss zmm28,DWORD PTR [r9+32]\t\n" "vmulps zmm16,zmm29,zmm28\t\n" "vmulps zmm17,zmm30,zmm28\t\n" "add r12, r13\t\n" "vbroadcastss zmm28,DWORD PTR [r9+36]\t\n" "vmulps zmm18,zmm29,zmm28\t\n" "vmulps zmm19,zmm30,zmm28\t\n" "add r12, r13\t\n" "vbroadcastss zmm28,DWORD PTR [r9+40]\t\n" "vmulps zmm20,zmm29,zmm28\t\n" "vmulps zmm21,zmm30,zmm28\t\n" "add r12, r13\t\n" "vbroadcastss zmm28,DWORD PTR [r9+44]\t\n" "vmulps zmm22,zmm29,zmm28\t\n" "vmulps zmm23,zmm30,zmm28\t\n" "add r12, r13\t\n" "vbroadcastss zmm28,DWORD PTR [r9+48]\t\n" "vmulps zmm24,zmm29,zmm28\t\n" "vmulps zmm25,zmm30,zmm28\t\n" "add r12, r13\t\n" "vbroadcastss zmm28,DWORD PTR [r9+52]\t\n" "vmulps zmm26,zmm29,zmm28\t\n" "vmulps zmm27,zmm30,zmm28\t\n" "mov r12, rcx\t\n" "test r14,r14\t\n" "jnz next_inner%=\t\n" "add r10,64\t\n" "jmp dump_C%=\t\n" "loop_inner%=:\t\n" "vmovaps zmm29,zmm31\t\n" "vcvtph2ps zmm30,YMMWORD PTR [r10 + 32]\t\n" "vcvtph2ps zmm31,YMMWORD PTR [r10 + 64]\t\n" "vbroadcastss zmm28,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm29,zmm28\t\n" "vfmadd231ps zmm1,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm29,zmm28\t\n" "vfmadd231ps zmm3,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+8]\t\n" "vfmadd231ps zmm4,zmm29,zmm28\t\n" "vfmadd231ps zmm5,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+12]\t\n" "vfmadd231ps zmm6,zmm29,zmm28\t\n" "vfmadd231ps zmm7,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+16]\t\n" "vfmadd231ps zmm8,zmm29,zmm28\t\n" "vfmadd231ps zmm9,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+20]\t\n" "vfmadd231ps zmm10,zmm29,zmm28\t\n" "vfmadd231ps zmm11,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+24]\t\n" "vfmadd231ps zmm12,zmm29,zmm28\t\n" "vfmadd231ps zmm13,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+28]\t\n" "vfmadd231ps zmm14,zmm29,zmm28\t\n" "vfmadd231ps zmm15,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+32]\t\n" "vfmadd231ps zmm16,zmm29,zmm28\t\n" "vfmadd231ps zmm17,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+36]\t\n" "vfmadd231ps zmm18,zmm29,zmm28\t\n" "vfmadd231ps zmm19,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+40]\t\n" "vfmadd231ps zmm20,zmm29,zmm28\t\n" "vfmadd231ps zmm21,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+44]\t\n" "vfmadd231ps zmm22,zmm29,zmm28\t\n" "vfmadd231ps zmm23,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+48]\t\n" "vfmadd231ps zmm24,zmm29,zmm28\t\n" "vfmadd231ps zmm25,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+52]\t\n" "vfmadd231ps zmm26,zmm29,zmm28\t\n" "vfmadd231ps zmm27,zmm30,zmm28\t\n" "next_inner%=:\t\n" "add r9,56\t\n" "add r10,64\t\n" "dec r14\t\n" "jnz loop_inner%=\t\n" "vmovaps zmm29,zmm31\t\n" "vcvtph2ps zmm30,YMMWORD PTR [r10 + 32]\t\n" "vbroadcastss zmm28,DWORD PTR [r9+0]\t\n" "vfmadd231ps zmm0,zmm29,zmm28\t\n" "vfmadd231ps zmm1,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+4]\t\n" "vfmadd231ps zmm2,zmm29,zmm28\t\n" "vfmadd231ps zmm3,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+8]\t\n" "vfmadd231ps zmm4,zmm29,zmm28\t\n" "vfmadd231ps zmm5,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+12]\t\n" "vfmadd231ps zmm6,zmm29,zmm28\t\n" "vfmadd231ps zmm7,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+16]\t\n" "vfmadd231ps zmm8,zmm29,zmm28\t\n" "vfmadd231ps zmm9,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+20]\t\n" "vfmadd231ps zmm10,zmm29,zmm28\t\n" "vfmadd231ps zmm11,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+24]\t\n" "vfmadd231ps zmm12,zmm29,zmm28\t\n" "vfmadd231ps zmm13,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+28]\t\n" "vfmadd231ps zmm14,zmm29,zmm28\t\n" "vfmadd231ps zmm15,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+32]\t\n" "vfmadd231ps zmm16,zmm29,zmm28\t\n" "vfmadd231ps zmm17,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+36]\t\n" "vfmadd231ps zmm18,zmm29,zmm28\t\n" "vfmadd231ps zmm19,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+40]\t\n" "vfmadd231ps zmm20,zmm29,zmm28\t\n" "vfmadd231ps zmm21,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+44]\t\n" "vfmadd231ps zmm22,zmm29,zmm28\t\n" "vfmadd231ps zmm23,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+48]\t\n" "vfmadd231ps zmm24,zmm29,zmm28\t\n" "vfmadd231ps zmm25,zmm30,zmm28\t\n" "vbroadcastss zmm28,DWORD PTR [r9+52]\t\n" "vfmadd231ps zmm26,zmm29,zmm28\t\n" "vfmadd231ps zmm27,zmm30,zmm28\t\n" "add r9,56\t\n" "add r10,64\t\n" // Dump C "dump_C%=:\t\n" "vmovups zmmword PTR [r12 + 0], zmm0\t\n" "vmovups zmmword PTR [r12 + 64], zmm1\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm2\t\n" "vmovups zmmword PTR [r12 + 64], zmm3\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm4\t\n" "vmovups zmmword PTR [r12 + 64], zmm5\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm6\t\n" "vmovups zmmword PTR [r12 + 64], zmm7\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm8\t\n" "vmovups zmmword PTR [r12 + 64], zmm9\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm10\t\n" "vmovups zmmword PTR [r12 + 64], zmm11\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm12\t\n" "vmovups zmmword PTR [r12 + 64], zmm13\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm14\t\n" "vmovups zmmword PTR [r12 + 64], zmm15\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm16\t\n" "vmovups zmmword PTR [r12 + 64], zmm17\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm18\t\n" "vmovups zmmword PTR [r12 + 64], zmm19\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm20\t\n" "vmovups zmmword PTR [r12 + 64], zmm21\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm22\t\n" "vmovups zmmword PTR [r12 + 64], zmm23\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm24\t\n" "vmovups zmmword PTR [r12 + 64], zmm25\t\n" "add r12, r13\t\n" "vmovups zmmword PTR [r12 + 0], zmm26\t\n" "vmovups zmmword PTR [r12 + 64], zmm27\t\n" // next outer iteration "add rcx, 128\t\n" "mov r12, rcx\t\n" "mov r9, rax\t\n" "inc rbx\t\n" "cmp rbx, rdi\t\n" "jl loop_outter%=\t\n" : : [gp] "rm"(gp) : "r8", "r9", "r10", "r11", "r13", "r14", "rax", "rcx", "rsi", "rdi", "rbx", "r12", "r15", "memory"); } } // namespace fbgemm