| /* | |
| * Copyright (c) 2016, 2024, Intel Corporation. All rights reserved. | |
| * | |
| * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. | |
| * | |
| * This code is free software; you can redistribute it and/or modify it | |
| * under the terms of the GNU General Public License version 2 only, as | |
| * published by the Free Software Foundation. | |
| * | |
| * This code is distributed in the hope that it will be useful, but WITHOUT | |
| * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
| * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
| * version 2 for more details (a copy is included in the LICENSE file that | |
| * accompanied this code). | |
| * | |
| * You should have received a copy of the GNU General Public License version | |
| * 2 along with this work; if not, write to the Free Software Foundation, | |
| * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. | |
| * | |
| * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA | |
| * or visit www.oracle.com if you need additional information or have any | |
| * questions. | |
| * | |
| */ | |
| #include "asm/assembler.hpp" | |
| #include "asm/assembler.inline.hpp" | |
| #include "runtime/stubRoutines.hpp" | |
| #include "macroAssembler_x86.hpp" | |
| // ofs and limit are used for multi-block byte array. | |
| // int com.sun.security.provider.DigestBase.implCompressMultiBlock(byte[] b, int ofs, int limit) | |
| void MacroAssembler::fast_sha1(XMMRegister abcd, XMMRegister e0, XMMRegister e1, XMMRegister msg0, | |
| XMMRegister msg1, XMMRegister msg2, XMMRegister msg3, XMMRegister shuf_mask, | |
| Register buf, Register state, Register ofs, Register limit, Register rsp, bool multi_block) { | |
| Label done_hash, loop0; | |
| address upper_word_mask = StubRoutines::x86::upper_word_mask_addr(); | |
| address shuffle_byte_flip_mask = StubRoutines::x86::shuffle_byte_flip_mask_addr(); | |
| movdqu(abcd, Address(state, 0)); | |
| pinsrd(e0, Address(state, 16), 3); | |
| movdqu(shuf_mask, ExternalAddress(upper_word_mask)); // 0xFFFFFFFF000000000000000000000000 | |
| pand(e0, shuf_mask); | |
| pshufd(abcd, abcd, 0x1B); | |
| movdqu(shuf_mask, ExternalAddress(shuffle_byte_flip_mask)); //0x000102030405060708090a0b0c0d0e0f | |
| bind(loop0); | |
| // Save hash values for addition after rounds | |
| movdqu(Address(rsp, 0), e0); | |
| movdqu(Address(rsp, 16), abcd); | |
| // Rounds 0 - 3 | |
| movdqu(msg0, Address(buf, 0)); | |
| pshufb(msg0, shuf_mask); | |
| paddd(e0, msg0); | |
| movdqa(e1, abcd); | |
| sha1rnds4(abcd, e0, 0); | |
| // Rounds 4 - 7 | |
| movdqu(msg1, Address(buf, 16)); | |
| pshufb(msg1, shuf_mask); | |
| sha1nexte(e1, msg1); | |
| movdqa(e0, abcd); | |
| sha1rnds4(abcd, e1, 0); | |
| sha1msg1(msg0, msg1); | |
| // Rounds 8 - 11 | |
| movdqu(msg2, Address(buf, 32)); | |
| pshufb(msg2, shuf_mask); | |
| sha1nexte(e0, msg2); | |
| movdqa(e1, abcd); | |
| sha1rnds4(abcd, e0, 0); | |
| sha1msg1(msg1, msg2); | |
| pxor(msg0, msg2); | |
| // Rounds 12 - 15 | |
| movdqu(msg3, Address(buf, 48)); | |
| pshufb(msg3, shuf_mask); | |
| sha1nexte(e1, msg3); | |
| movdqa(e0, abcd); | |
| sha1msg2(msg0, msg3); | |
| sha1rnds4(abcd, e1, 0); | |
| sha1msg1(msg2, msg3); | |
| pxor(msg1, msg3); | |
| // Rounds 16 - 19 | |
| sha1nexte(e0, msg0); | |
| movdqa(e1, abcd); | |
| sha1msg2(msg1, msg0); | |
| sha1rnds4(abcd, e0, 0); | |
| sha1msg1(msg3, msg0); | |
| pxor(msg2, msg0); | |
| // Rounds 20 - 23 | |
| sha1nexte(e1, msg1); | |
| movdqa(e0, abcd); | |
| sha1msg2(msg2, msg1); | |
| sha1rnds4(abcd, e1, 1); | |
| sha1msg1(msg0, msg1); | |
| pxor(msg3, msg1); | |
| // Rounds 24 - 27 | |
| sha1nexte(e0, msg2); | |
| movdqa(e1, abcd); | |
| sha1msg2(msg3, msg2); | |
| sha1rnds4(abcd, e0, 1); | |
| sha1msg1(msg1, msg2); | |
| pxor(msg0, msg2); | |
| // Rounds 28 - 31 | |
| sha1nexte(e1, msg3); | |
| movdqa(e0, abcd); | |
| sha1msg2(msg0, msg3); | |
| sha1rnds4(abcd, e1, 1); | |
| sha1msg1(msg2, msg3); | |
| pxor(msg1, msg3); | |
| // Rounds 32 - 35 | |
| sha1nexte(e0, msg0); | |
| movdqa(e1, abcd); | |
| sha1msg2(msg1, msg0); | |
| sha1rnds4(abcd, e0, 1); | |
| sha1msg1(msg3, msg0); | |
| pxor(msg2, msg0); | |
| // Rounds 36 - 39 | |
| sha1nexte(e1, msg1); | |
| movdqa(e0, abcd); | |
| sha1msg2(msg2, msg1); | |
| sha1rnds4(abcd, e1, 1); | |
| sha1msg1(msg0, msg1); | |
| pxor(msg3, msg1); | |
| // Rounds 40 - 43 | |
| sha1nexte(e0, msg2); | |
| movdqa(e1, abcd); | |
| sha1msg2(msg3, msg2); | |
| sha1rnds4(abcd, e0, 2); | |
| sha1msg1(msg1, msg2); | |
| pxor(msg0, msg2); | |
| // Rounds 44 - 47 | |
| sha1nexte(e1, msg3); | |
| movdqa(e0, abcd); | |
| sha1msg2(msg0, msg3); | |
| sha1rnds4(abcd, e1, 2); | |
| sha1msg1(msg2, msg3); | |
| pxor(msg1, msg3); | |
| // Rounds 48 - 51 | |
| sha1nexte(e0, msg0); | |
| movdqa(e1, abcd); | |
| sha1msg2(msg1, msg0); | |
| sha1rnds4(abcd, e0, 2); | |
| sha1msg1(msg3, msg0); | |
| pxor(msg2, msg0); | |
| // Rounds 52 - 55 | |
| sha1nexte(e1, msg1); | |
| movdqa(e0, abcd); | |
| sha1msg2(msg2, msg1); | |
| sha1rnds4(abcd, e1, 2); | |
| sha1msg1(msg0, msg1); | |
| pxor(msg3, msg1); | |
| // Rounds 56 - 59 | |
| sha1nexte(e0, msg2); | |
| movdqa(e1, abcd); | |
| sha1msg2(msg3, msg2); | |
| sha1rnds4(abcd, e0, 2); | |
| sha1msg1(msg1, msg2); | |
| pxor(msg0, msg2); | |
| // Rounds 60 - 63 | |
| sha1nexte(e1, msg3); | |
| movdqa(e0, abcd); | |
| sha1msg2(msg0, msg3); | |
| sha1rnds4(abcd, e1, 3); | |
| sha1msg1(msg2, msg3); | |
| pxor(msg1, msg3); | |
| // Rounds 64 - 67 | |
| sha1nexte(e0, msg0); | |
| movdqa(e1, abcd); | |
| sha1msg2(msg1, msg0); | |
| sha1rnds4(abcd, e0, 3); | |
| sha1msg1(msg3, msg0); | |
| pxor(msg2, msg0); | |
| // Rounds 68 - 71 | |
| sha1nexte(e1, msg1); | |
| movdqa(e0, abcd); | |
| sha1msg2(msg2, msg1); | |
| sha1rnds4(abcd, e1, 3); | |
| pxor(msg3, msg1); | |
| // Rounds 72 - 75 | |
| sha1nexte(e0, msg2); | |
| movdqa(e1, abcd); | |
| sha1msg2(msg3, msg2); | |
| sha1rnds4(abcd, e0, 3); | |
| // Rounds 76 - 79 | |
| sha1nexte(e1, msg3); | |
| movdqa(e0, abcd); | |
| sha1rnds4(abcd, e1, 3); | |
| // add current hash values with previously saved | |
| movdqu(msg0, Address(rsp, 0)); | |
| sha1nexte(e0, msg0); | |
| movdqu(msg0, Address(rsp, 16)); | |
| paddd(abcd, msg0); | |
| if (multi_block) { | |
| // increment data pointer and loop if more to process | |
| addptr(buf, 64); | |
| addptr(ofs, 64); | |
| cmpptr(ofs, limit); | |
| jcc(Assembler::belowEqual, loop0); | |
| movptr(rax, ofs); //return ofs | |
| } | |
| // write hash values back in the correct order | |
| pshufd(abcd, abcd, 0x1b); | |
| movdqu(Address(state, 0), abcd); | |
| pextrd(Address(state, 16), e0, 3); | |
| bind(done_hash); | |
| } | |
| // xmm0 (msg) is used as an implicit argument to sh256rnds2 | |
| // and state0 and state1 can never use xmm0 register. | |
| // ofs and limit are used for multi-block byte array. | |
| // int com.sun.security.provider.DigestBase.implCompressMultiBlock(byte[] b, int ofs, int limit) | |
| void MacroAssembler::fast_sha256(XMMRegister msg, XMMRegister state0, XMMRegister state1, XMMRegister msgtmp0, | |
| XMMRegister msgtmp1, XMMRegister msgtmp2, XMMRegister msgtmp3, XMMRegister msgtmp4, | |
| Register buf, Register state, Register ofs, Register limit, Register rsp, | |
| bool multi_block, XMMRegister shuf_mask) { | |
| Label done_hash, loop0; | |
| address K256 = StubRoutines::x86::k256_addr(); | |
| address pshuffle_byte_flip_mask = StubRoutines::x86::pshuffle_byte_flip_mask_addr(); | |
| movdqu(state0, Address(state, 0)); | |
| movdqu(state1, Address(state, 16)); | |
| pshufd(state0, state0, 0xB1); | |
| pshufd(state1, state1, 0x1B); | |
| movdqa(msgtmp4, state0); | |
| palignr(state0, state1, 8); | |
| pblendw(state1, msgtmp4, 0xF0); | |
| movdqu(shuf_mask, ExternalAddress(pshuffle_byte_flip_mask)); | |
| lea(rax, ExternalAddress(K256)); | |
| bind(loop0); | |
| movdqu(Address(rsp, 0), state0); | |
| movdqu(Address(rsp, 16), state1); | |
| // Rounds 0-3 | |
| movdqu(msg, Address(buf, 0)); | |
| pshufb(msg, shuf_mask); | |
| movdqa(msgtmp0, msg); | |
| paddd(msg, Address(rax, 0)); | |
| sha256rnds2(state1, state0); | |
| pshufd(msg, msg, 0x0E); | |
| sha256rnds2(state0, state1); | |
| // Rounds 4-7 | |
| movdqu(msg, Address(buf, 16)); | |
| pshufb(msg, shuf_mask); | |
| movdqa(msgtmp1, msg); | |
| paddd(msg, Address(rax, 16)); | |
| sha256rnds2(state1, state0); | |
| pshufd(msg, msg, 0x0E); | |
| sha256rnds2(state0, state1); | |
| sha256msg1(msgtmp0, msgtmp1); | |
| // Rounds 8-11 | |
| movdqu(msg, Address(buf, 32)); | |
| pshufb(msg, shuf_mask); | |
| movdqa(msgtmp2, msg); | |
| paddd(msg, Address(rax, 32)); | |
| sha256rnds2(state1, state0); | |
| pshufd(msg, msg, 0x0E); | |
| sha256rnds2(state0, state1); | |
| sha256msg1(msgtmp1, msgtmp2); | |
| // Rounds 12-15 | |
| movdqu(msg, Address(buf, 48)); | |
| pshufb(msg, shuf_mask); | |
| movdqa(msgtmp3, msg); | |
| paddd(msg, Address(rax, 48)); | |
| sha256rnds2(state1, state0); | |
| movdqa(msgtmp4, msgtmp3); | |
| palignr(msgtmp4, msgtmp2, 4); | |
| paddd(msgtmp0, msgtmp4); | |
| sha256msg2(msgtmp0, msgtmp3); | |
| pshufd(msg, msg, 0x0E); | |
| sha256rnds2(state0, state1); | |
| sha256msg1(msgtmp2, msgtmp3); | |
| // Rounds 16-19 | |
| movdqa(msg, msgtmp0); | |
| paddd(msg, Address(rax, 64)); | |
| sha256rnds2(state1, state0); | |
| movdqa(msgtmp4, msgtmp0); | |
| palignr(msgtmp4, msgtmp3, 4); | |
| paddd(msgtmp1, msgtmp4); | |
| sha256msg2(msgtmp1, msgtmp0); | |
| pshufd(msg, msg, 0x0E); | |
| sha256rnds2(state0, state1); | |
| sha256msg1(msgtmp3, msgtmp0); | |
| // Rounds 20-23 | |
| movdqa(msg, msgtmp1); | |
| paddd(msg, Address(rax, 80)); | |
| sha256rnds2(state1, state0); | |
| movdqa(msgtmp4, msgtmp1); | |
| palignr(msgtmp4, msgtmp0, 4); | |
| paddd(msgtmp2, msgtmp4); | |
| sha256msg2(msgtmp2, msgtmp1); | |
| pshufd(msg, msg, 0x0E); | |
| sha256rnds2(state0, state1); | |
| sha256msg1(msgtmp0, msgtmp1); | |
| // Rounds 24-27 | |
| movdqa(msg, msgtmp2); | |
| paddd(msg, Address(rax, 96)); | |
| sha256rnds2(state1, state0); | |
| movdqa(msgtmp4, msgtmp2); | |
| palignr(msgtmp4, msgtmp1, 4); | |
| paddd(msgtmp3, msgtmp4); | |
| sha256msg2(msgtmp3, msgtmp2); | |
| pshufd(msg, msg, 0x0E); | |
| sha256rnds2(state0, state1); | |
| sha256msg1(msgtmp1, msgtmp2); | |
| // Rounds 28-31 | |
| movdqa(msg, msgtmp3); | |
| paddd(msg, Address(rax, 112)); | |
| sha256rnds2(state1, state0); | |
| movdqa(msgtmp4, msgtmp3); | |
| palignr(msgtmp4, msgtmp2, 4); | |
| paddd(msgtmp0, msgtmp4); | |
| sha256msg2(msgtmp0, msgtmp3); | |
| pshufd(msg, msg, 0x0E); | |
| sha256rnds2(state0, state1); | |
| sha256msg1(msgtmp2, msgtmp3); | |
| // Rounds 32-35 | |
| movdqa(msg, msgtmp0); | |
| paddd(msg, Address(rax, 128)); | |
| sha256rnds2(state1, state0); | |
| movdqa(msgtmp4, msgtmp0); | |
| palignr(msgtmp4, msgtmp3, 4); | |
| paddd(msgtmp1, msgtmp4); | |
| sha256msg2(msgtmp1, msgtmp0); | |
| pshufd(msg, msg, 0x0E); | |
| sha256rnds2(state0, state1); | |
| sha256msg1(msgtmp3, msgtmp0); | |
| // Rounds 36-39 | |
| movdqa(msg, msgtmp1); | |
| paddd(msg, Address(rax, 144)); | |
| sha256rnds2(state1, state0); | |
| movdqa(msgtmp4, msgtmp1); | |
| palignr(msgtmp4, msgtmp0, 4); | |
| paddd(msgtmp2, msgtmp4); | |
| sha256msg2(msgtmp2, msgtmp1); | |
| pshufd(msg, msg, 0x0E); | |
| sha256rnds2(state0, state1); | |
| sha256msg1(msgtmp0, msgtmp1); | |
| // Rounds 40-43 | |
| movdqa(msg, msgtmp2); | |
| paddd(msg, Address(rax, 160)); | |
| sha256rnds2(state1, state0); | |
| movdqa(msgtmp4, msgtmp2); | |
| palignr(msgtmp4, msgtmp1, 4); | |
| paddd(msgtmp3, msgtmp4); | |
| sha256msg2(msgtmp3, msgtmp2); | |
| pshufd(msg, msg, 0x0E); | |
| sha256rnds2(state0, state1); | |
| sha256msg1(msgtmp1, msgtmp2); | |
| // Rounds 44-47 | |
| movdqa(msg, msgtmp3); | |
| paddd(msg, Address(rax, 176)); | |
| sha256rnds2(state1, state0); | |
| movdqa(msgtmp4, msgtmp3); | |
| palignr(msgtmp4, msgtmp2, 4); | |
| paddd(msgtmp0, msgtmp4); | |
| sha256msg2(msgtmp0, msgtmp3); | |
| pshufd(msg, msg, 0x0E); | |
| sha256rnds2(state0, state1); | |
| sha256msg1(msgtmp2, msgtmp3); | |
| // Rounds 48-51 | |
| movdqa(msg, msgtmp0); | |
| paddd(msg, Address(rax, 192)); | |
| sha256rnds2(state1, state0); | |
| movdqa(msgtmp4, msgtmp0); | |
| palignr(msgtmp4, msgtmp3, 4); | |
| paddd(msgtmp1, msgtmp4); | |
| sha256msg2(msgtmp1, msgtmp0); | |
| pshufd(msg, msg, 0x0E); | |
| sha256rnds2(state0, state1); | |
| sha256msg1(msgtmp3, msgtmp0); | |
| // Rounds 52-55 | |
| movdqa(msg, msgtmp1); | |
| paddd(msg, Address(rax, 208)); | |
| sha256rnds2(state1, state0); | |
| movdqa(msgtmp4, msgtmp1); | |
| palignr(msgtmp4, msgtmp0, 4); | |
| paddd(msgtmp2, msgtmp4); | |
| sha256msg2(msgtmp2, msgtmp1); | |
| pshufd(msg, msg, 0x0E); | |
| sha256rnds2(state0, state1); | |
| // Rounds 56-59 | |
| movdqa(msg, msgtmp2); | |
| paddd(msg, Address(rax, 224)); | |
| sha256rnds2(state1, state0); | |
| movdqa(msgtmp4, msgtmp2); | |
| palignr(msgtmp4, msgtmp1, 4); | |
| paddd(msgtmp3, msgtmp4); | |
| sha256msg2(msgtmp3, msgtmp2); | |
| pshufd(msg, msg, 0x0E); | |
| sha256rnds2(state0, state1); | |
| // Rounds 60-63 | |
| movdqa(msg, msgtmp3); | |
| paddd(msg, Address(rax, 240)); | |
| sha256rnds2(state1, state0); | |
| pshufd(msg, msg, 0x0E); | |
| sha256rnds2(state0, state1); | |
| movdqu(msg, Address(rsp, 0)); | |
| paddd(state0, msg); | |
| movdqu(msg, Address(rsp, 16)); | |
| paddd(state1, msg); | |
| if (multi_block) { | |
| // increment data pointer and loop if more to process | |
| addptr(buf, 64); | |
| addptr(ofs, 64); | |
| cmpptr(ofs, limit); | |
| jcc(Assembler::belowEqual, loop0); | |
| movptr(rax, ofs); //return ofs | |
| } | |
| pshufd(state0, state0, 0x1B); | |
| pshufd(state1, state1, 0xB1); | |
| movdqa(msgtmp4, state0); | |
| pblendw(state0, state1, 0xF0); | |
| palignr(state1, msgtmp4, 8); | |
| movdqu(Address(state, 0), state0); | |
| movdqu(Address(state, 16), state1); | |
| bind(done_hash); | |
| } | |
| /* | |
| The algorithm below is based on Intel publication: | |
| "Fast SHA-256 Implementations on Intel(R) Architecture Processors" by Jim Guilford, Kirk Yap and Vinodh Gopal. | |
| The assembly code was originally provided by Sean Gulley and in many places preserves | |
| the original assembly NAMES and comments to simplify matching Java assembly with its original. | |
| The Java version was substantially redesigned to replace 1200 assembly instruction with | |
| much shorter run-time generator of the same code in memory. | |
| */ | |
| void MacroAssembler::sha256_AVX2_one_round_compute( | |
| Register reg_old_h, | |
| Register reg_a, | |
| Register reg_b, | |
| Register reg_c, | |
| Register reg_d, | |
| Register reg_e, | |
| Register reg_f, | |
| Register reg_g, | |
| Register reg_h, | |
| int iter) { | |
| const Register& reg_y0 = r13; | |
| const Register& reg_y1 = r14; | |
| const Register& reg_y2 = r15; | |
| const Register& reg_y3 = rcx; | |
| const Register& reg_T1 = r12; | |
| //;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; RND iter ;;;;;;;;;;;;;;;;;;;;;;;;;;; | |
| if (iter%4 > 0) { | |
| addl(reg_old_h, reg_y2); // reg_h = k + w + reg_h + S0 + S1 + CH = t1 + S0; -- | |
| } | |
| movl(reg_y2, reg_f); // reg_y2 = reg_f ; CH | |
| rorxl(reg_y0, reg_e, 25); // reg_y0 = reg_e >> 25 ; S1A | |
| rorxl(reg_y1, reg_e, 11); // reg_y1 = reg_e >> 11 ; S1B | |
| xorl(reg_y2, reg_g); // reg_y2 = reg_f^reg_g ; CH | |
| xorl(reg_y0, reg_y1); // reg_y0 = (reg_e>>25) ^ (reg_h>>11) ; S1 | |
| rorxl(reg_y1, reg_e, 6); // reg_y1 = (reg_e >> 6) ; S1 | |
| andl(reg_y2, reg_e); // reg_y2 = (reg_f^reg_g)®_e ; CH | |
| if (iter%4 > 0) { | |
| addl(reg_old_h, reg_y3); // reg_h = t1 + S0 + MAJ ; -- | |
| } | |
| xorl(reg_y0, reg_y1); // reg_y0 = (reg_e>>25) ^ (reg_e>>11) ^ (reg_e>>6) ; S1 | |
| rorxl(reg_T1, reg_a, 13); // reg_T1 = reg_a >> 13 ; S0B | |
| xorl(reg_y2, reg_g); // reg_y2 = CH = ((reg_f^reg_g)®_e)^reg_g ; CH | |
| rorxl(reg_y1, reg_a, 22); // reg_y1 = reg_a >> 22 ; S0A | |
| movl(reg_y3, reg_a); // reg_y3 = reg_a ; MAJA | |
| xorl(reg_y1, reg_T1); // reg_y1 = (reg_a>>22) ^ (reg_a>>13) ; S0 | |
| rorxl(reg_T1, reg_a, 2); // reg_T1 = (reg_a >> 2) ; S0 | |
| addl(reg_h, Address(rsp, rdx, Address::times_1, 4*iter)); // reg_h = k + w + reg_h ; -- | |
| orl(reg_y3, reg_c); // reg_y3 = reg_a|reg_c ; MAJA | |
| xorl(reg_y1, reg_T1); // reg_y1 = (reg_a>>22) ^ (reg_a>>13) ^ (reg_a>>2) ; S0 | |
| movl(reg_T1, reg_a); // reg_T1 = reg_a ; MAJB | |
| andl(reg_y3, reg_b); // reg_y3 = (reg_a|reg_c)®_b ; MAJA | |
| andl(reg_T1, reg_c); // reg_T1 = reg_a®_c ; MAJB | |
| addl(reg_y2, reg_y0); // reg_y2 = S1 + CH ; -- | |
| addl(reg_d, reg_h); // reg_d = k + w + reg_h + reg_d ; -- | |
| orl(reg_y3, reg_T1); // reg_y3 = MAJ = (reg_a|reg_c)®_b)|(reg_a®_c) ; MAJ | |
| addl(reg_h, reg_y1); // reg_h = k + w + reg_h + S0 ; -- | |
| addl(reg_d, reg_y2); // reg_d = k + w + reg_h + reg_d + S1 + CH = reg_d + t1 ; -- | |
| if (iter%4 == 3) { | |
| addl(reg_h, reg_y2); // reg_h = k + w + reg_h + S0 + S1 + CH = t1 + S0; -- | |
| addl(reg_h, reg_y3); // reg_h = t1 + S0 + MAJ ; -- | |
| } | |
| } | |
| void MacroAssembler::sha256_AVX2_four_rounds_compute_first(int start) { | |
| sha256_AVX2_one_round_compute(rax, rax, rbx, rdi, rsi, r8, r9, r10, r11, start + 0); | |
| sha256_AVX2_one_round_compute(r11, r11, rax, rbx, rdi, rsi, r8, r9, r10, start + 1); | |
| sha256_AVX2_one_round_compute(r10, r10, r11, rax, rbx, rdi, rsi, r8, r9, start + 2); | |
| sha256_AVX2_one_round_compute(r9, r9, r10, r11, rax, rbx, rdi, rsi, r8, start + 3); | |
| } | |
| void MacroAssembler::sha256_AVX2_four_rounds_compute_last(int start) { | |
| sha256_AVX2_one_round_compute(r8, r8, r9, r10, r11, rax, rbx, rdi, rsi, start + 0); | |
| sha256_AVX2_one_round_compute(rsi, rsi, r8, r9, r10, r11, rax, rbx, rdi, start + 1); | |
| sha256_AVX2_one_round_compute(rdi, rdi, rsi, r8, r9, r10, r11, rax, rbx, start + 2); | |
| sha256_AVX2_one_round_compute(rbx, rbx, rdi, rsi, r8, r9, r10, r11, rax, start + 3); | |
| } | |
| void MacroAssembler::sha256_AVX2_one_round_and_sched( | |
| XMMRegister xmm_0, /* == ymm4 on 0, 1, 2, 3 iterations, then rotate 4 registers left on 4, 8, 12 iterations */ | |
| XMMRegister xmm_1, /* ymm5 */ /* full cycle is 16 iterations */ | |
| XMMRegister xmm_2, /* ymm6 */ | |
| XMMRegister xmm_3, /* ymm7 */ | |
| Register reg_a, /* == rax on 0 iteration, then rotate 8 register right on each next iteration */ | |
| Register reg_b, /* rbx */ /* full cycle is 8 iterations */ | |
| Register reg_c, /* rdi */ | |
| Register reg_d, /* rsi */ | |
| Register reg_e, /* r8 */ | |
| Register reg_f, /* r9d */ | |
| Register reg_g, /* r10d */ | |
| Register reg_h, /* r11d */ | |
| int iter) | |
| { | |
| movl(rcx, reg_a); // rcx = reg_a ; MAJA | |
| rorxl(r13, reg_e, 25); // r13 = reg_e >> 25 ; S1A | |
| rorxl(r14, reg_e, 11); // r14 = reg_e >> 11 ; S1B | |
| addl(reg_h, Address(rsp, rdx, Address::times_1, 4*iter)); | |
| orl(rcx, reg_c); // rcx = reg_a|reg_c ; MAJA | |
| movl(r15, reg_f); // r15 = reg_f ; CH | |
| rorxl(r12, reg_a, 13); // r12 = reg_a >> 13 ; S0B | |
| xorl(r13, r14); // r13 = (reg_e>>25) ^ (reg_e>>11) ; S1 | |
| xorl(r15, reg_g); // r15 = reg_f^reg_g ; CH | |
| rorxl(r14, reg_e, 6); // r14 = (reg_e >> 6) ; S1 | |
| andl(r15, reg_e); // r15 = (reg_f^reg_g)®_e ; CH | |
| xorl(r13, r14); // r13 = (reg_e>>25) ^ (reg_e>>11) ^ (reg_e>>6) ; S1 | |
| rorxl(r14, reg_a, 22); // r14 = reg_a >> 22 ; S0A | |
| addl(reg_d, reg_h); // reg_d = k + w + reg_h + reg_d ; -- | |
| andl(rcx, reg_b); // rcx = (reg_a|reg_c)®_b ; MAJA | |
| xorl(r14, r12); // r14 = (reg_a>>22) ^ (reg_a>>13) ; S0 | |
| rorxl(r12, reg_a, 2); // r12 = (reg_a >> 2) ; S0 | |
| xorl(r15, reg_g); // r15 = CH = ((reg_f^reg_g)®_e)^reg_g ; CH | |
| xorl(r14, r12); // r14 = (reg_a>>22) ^ (reg_a>>13) ^ (reg_a>>2) ; S0 | |
| movl(r12, reg_a); // r12 = reg_a ; MAJB | |
| andl(r12, reg_c); // r12 = reg_a®_c ; MAJB | |
| addl(r15, r13); // r15 = S1 + CH ; -- | |
| orl(rcx, r12); // rcx = MAJ = (reg_a|reg_c)®_b)|(reg_a®_c) ; MAJ | |
| addl(reg_h, r14); // reg_h = k + w + reg_h + S0 ; -- | |
| addl(reg_d, r15); // reg_d = k + w + reg_h + reg_d + S1 + CH = reg_d + t1 ; -- | |
| addl(reg_h, r15); // reg_h = k + w + reg_h + S0 + S1 + CH = t1 + S0; -- | |
| addl(reg_h, rcx); // reg_h = t1 + S0 + MAJ ; -- | |
| if (iter%4 == 0) { | |
| vpalignr(xmm0, xmm_3, xmm_2, 4, AVX_256bit); // ymm0 = W[-7] | |
| vpaddd(xmm0, xmm0, xmm_0, AVX_256bit); // ymm0 = W[-7] + W[-16]; y1 = (e >> 6) ; S1 | |
| vpalignr(xmm1, xmm_1, xmm_0, 4, AVX_256bit); // ymm1 = W[-15] | |
| vpsrld(xmm2, xmm1, 7, AVX_256bit); | |
| vpslld(xmm3, xmm1, 32-7, AVX_256bit); | |
| vpor(xmm3, xmm3, xmm2, AVX_256bit); // ymm3 = W[-15] ror 7 | |
| vpsrld(xmm2, xmm1,18, AVX_256bit); | |
| } else if (iter%4 == 1 ) { | |
| vpsrld(xmm8, xmm1, 3, AVX_256bit); // ymm8 = W[-15] >> 3 | |
| vpslld(xmm1, xmm1, 32-18, AVX_256bit); | |
| vpxor(xmm3, xmm3, xmm1, AVX_256bit); | |
| vpxor(xmm3, xmm3, xmm2, AVX_256bit); // ymm3 = W[-15] ror 7 ^ W[-15] ror 18 | |
| vpxor(xmm1, xmm3, xmm8, AVX_256bit); // ymm1 = s0 | |
| vpshufd(xmm2, xmm_3, 0xFA, AVX_256bit); // 11111010b ; ymm2 = W[-2] {BBAA} | |
| vpaddd(xmm0, xmm0, xmm1, AVX_256bit); // ymm0 = W[-16] + W[-7] + s0 | |
| vpsrld(xmm8, xmm2, 10, AVX_256bit); // ymm8 = W[-2] >> 10 {BBAA} | |
| } else if (iter%4 == 2) { | |
| vpsrlq(xmm3, xmm2, 19, AVX_256bit); // ymm3 = W[-2] ror 19 {xBxA} | |
| vpsrlq(xmm2, xmm2, 17, AVX_256bit); // ymm2 = W[-2] ror 17 {xBxA} | |
| vpxor(xmm2, xmm2, xmm3, AVX_256bit); | |
| vpxor(xmm8, xmm8, xmm2, AVX_256bit); // ymm8 = s1 {xBxA} | |
| vpshufb(xmm8, xmm8, xmm10, AVX_256bit); // ymm8 = s1 {00BA} | |
| vpaddd(xmm0, xmm0, xmm8, AVX_256bit); // ymm0 = {..., ..., W[1], W[0]} | |
| vpshufd(xmm2, xmm0, 0x50, AVX_256bit); // 01010000b ; ymm2 = W[-2] {DDCC} | |
| } else if (iter%4 == 3) { | |
| vpsrld(xmm11, xmm2, 10, AVX_256bit); // ymm11 = W[-2] >> 10 {DDCC} | |
| vpsrlq(xmm3, xmm2, 19, AVX_256bit); // ymm3 = W[-2] ror 19 {xDxC} | |
| vpsrlq(xmm2, xmm2, 17, AVX_256bit); // ymm2 = W[-2] ror 17 {xDxC} | |
| vpxor(xmm2, xmm2, xmm3, AVX_256bit); | |
| vpxor(xmm11, xmm11, xmm2, AVX_256bit); // ymm11 = s1 {xDxC} | |
| vpshufb(xmm11, xmm11, xmm12, AVX_256bit); // ymm11 = s1 {DC00} | |
| vpaddd(xmm_0, xmm11, xmm0, AVX_256bit); // xmm_0 = {W[3], W[2], W[1], W[0]} | |
| } | |
| } | |
| void MacroAssembler::addm(int disp, Register r1, Register r2) { | |
| addl(r2, Address(r1, disp)); | |
| movl(Address(r1, disp), r2); | |
| } | |
| void MacroAssembler::addmq(int disp, Register r1, Register r2) { | |
| addq(r2, Address(r1, disp)); | |
| movq(Address(r1, disp), r2); | |
| } | |
| void MacroAssembler::sha256_AVX2(XMMRegister msg, XMMRegister state0, XMMRegister state1, XMMRegister msgtmp0, | |
| XMMRegister msgtmp1, XMMRegister msgtmp2, XMMRegister msgtmp3, XMMRegister msgtmp4, | |
| Register buf, Register state, Register ofs, Register limit, Register rsp, | |
| bool multi_block, XMMRegister shuf_mask) { | |
| Label loop0, loop1, loop2, loop3, | |
| last_block_enter, do_last_block, only_one_block, done_hash, | |
| compute_size, compute_size_end, | |
| compute_size1, compute_size_end1; | |
| address K256_W = StubRoutines::x86::k256_W_addr(); | |
| address pshuffle_byte_flip_mask = StubRoutines::x86::pshuffle_byte_flip_mask_addr(); | |
| address pshuffle_byte_flip_mask_addr = nullptr; | |
| const XMMRegister& SHUF_00BA = xmm10; // ymm10: shuffle xBxA -> 00BA | |
| const XMMRegister& SHUF_DC00 = xmm12; // ymm12: shuffle xDxC -> DC00 | |
| const XMMRegister& BYTE_FLIP_MASK = xmm13; // ymm13 | |
| const XMMRegister& X_BYTE_FLIP_MASK = xmm13; //XMM version of BYTE_FLIP_MASK | |
| const Register& NUM_BLKS = r8; // 3rd arg | |
| const Register& CTX = rdx; // 2nd arg | |
| const Register& INP = rcx; // 1st arg | |
| const Register& c = rdi; | |
| const Register& d = rsi; | |
| const Register& e = r8; // clobbers NUM_BLKS | |
| const Register& y3 = rcx; // clobbers INP | |
| const Register& TBL = rbp; | |
| const Register& SRND = CTX; // SRND is same register as CTX | |
| const Register& a = rax; | |
| const Register& b = rbx; | |
| const Register& f = r9; | |
| const Register& g = r10; | |
| const Register& h = r11; | |
| const Register& T1 = r12; | |
| const Register& y0 = r13; | |
| const Register& y1 = r14; | |
| const Register& y2 = r15; | |
| enum { | |
| _XFER_SIZE = 2*64*4, // 2 blocks, 64 rounds, 4 bytes/round | |
| _INP_END_SIZE = 8, | |
| _INP_SIZE = 8, | |
| _CTX_SIZE = 8, | |
| _RSP_SIZE = 8, | |
| _XFER = 0, | |
| _INP_END = _XFER + _XFER_SIZE, | |
| _INP = _INP_END + _INP_END_SIZE, | |
| _CTX = _INP + _INP_SIZE, | |
| _RSP = _CTX + _CTX_SIZE, | |
| STACK_SIZE = _RSP + _RSP_SIZE | |
| }; | |
| #ifndef _WIN64 | |
| push(rcx); // linux: this is limit, need at the end | |
| push(rdx); // linux: this is ofs | |
| #else | |
| push(r8); // win64: this is ofs | |
| push(r9); // win64: this is limit, we need them again at the very and | |
| #endif | |
| push(rbx); | |
| #ifdef _WIN64 | |
| push(rsi); | |
| push(rdi); | |
| #endif | |
| push(rbp); | |
| push(r12); | |
| push(r13); | |
| push(r14); | |
| push(r15); | |
| movq(rax, rsp); | |
| subq(rsp, STACK_SIZE); | |
| andq(rsp, -32); | |
| movq(Address(rsp, _RSP), rax); | |
| #ifndef _WIN64 | |
| // copy linux params to win64 params, therefore the rest of code will be the same for both | |
| movq(r9, rcx); | |
| movq(r8, rdx); | |
| movq(rdx, rsi); | |
| movq(rcx, rdi); | |
| #endif | |
| // setting original assembly ABI | |
| /** message to encrypt in INP */ | |
| lea(INP, Address(rcx, 0)); // rcx == message (buf) ;; linux: INP = buf = rdi | |
| /** digest in CTX */ | |
| movq(CTX, rdx); // rdx = digest (state) ;; linux: CTX = state = rsi | |
| /** NUM_BLK is the length of message, need to set it from ofs and limit */ | |
| if (multi_block) { | |
| // Win64: cannot directly update NUM_BLKS, since NUM_BLKS = ofs = r8 | |
| // on entry r8 = ofs | |
| // on exit r8 = NUM_BLKS | |
| xorq(rax, rax); | |
| bind(compute_size); | |
| cmpptr(r8, r9); // assume the original ofs <= limit ;; linux: cmp rcx, rdx | |
| jccb(Assembler::aboveEqual, compute_size_end); | |
| addq(r8, 64); //;; linux: ofs = rdx | |
| addq(rax, 64); | |
| jmpb(compute_size); | |
| bind(compute_size_end); | |
| movq(NUM_BLKS, rax); // NUM_BLK (r8) ;; linux: NUM_BLK = rdx | |
| cmpq(NUM_BLKS, 0); | |
| jcc(Assembler::equal, done_hash); | |
| } else { | |
| xorq(NUM_BLKS, NUM_BLKS); | |
| addq(NUM_BLKS, 64); | |
| }//if (!multi_block) | |
| lea(NUM_BLKS, Address(INP, NUM_BLKS, Address::times_1, -64)); // pointer to the last block | |
| movq(Address(rsp, _INP_END), NUM_BLKS); // | |
| cmpptr(INP, NUM_BLKS); //cmp INP, NUM_BLKS | |
| jcc(Assembler::equal, only_one_block); //je only_one_block | |
| // load initial digest | |
| movl(a, Address(CTX, 4*0)); | |
| movl(b, Address(CTX, 4*1)); | |
| movl(c, Address(CTX, 4*2)); | |
| movl(d, Address(CTX, 4*3)); | |
| movl(e, Address(CTX, 4*4)); | |
| movl(f, Address(CTX, 4*5)); | |
| // load g - r10 after it is used as scratch | |
| movl(h, Address(CTX, 4*7)); | |
| pshuffle_byte_flip_mask_addr = pshuffle_byte_flip_mask; | |
| vmovdqu(BYTE_FLIP_MASK, ExternalAddress(pshuffle_byte_flip_mask_addr + 0)); // [PSHUFFLE_BYTE_FLIP_MASK wrt rip] | |
| vmovdqu(SHUF_00BA, ExternalAddress(pshuffle_byte_flip_mask_addr + 32)); // [_SHUF_00BA wrt rip] | |
| vmovdqu(SHUF_DC00, ExternalAddress(pshuffle_byte_flip_mask_addr + 64)); // [_SHUF_DC00 wrt rip] | |
| movl(g, Address(CTX, 4*6)); | |
| movq(Address(rsp, _CTX), CTX); // store | |
| bind(loop0); | |
| lea(TBL, ExternalAddress(K256_W)); | |
| // assume buffers not aligned | |
| // Load first 16 dwords from two blocks | |
| vmovdqu(xmm0, Address(INP, 0*32)); | |
| vmovdqu(xmm1, Address(INP, 1*32)); | |
| vmovdqu(xmm2, Address(INP, 2*32)); | |
| vmovdqu(xmm3, Address(INP, 3*32)); | |
| // byte swap data | |
| vpshufb(xmm0, xmm0, BYTE_FLIP_MASK, AVX_256bit); | |
| vpshufb(xmm1, xmm1, BYTE_FLIP_MASK, AVX_256bit); | |
| vpshufb(xmm2, xmm2, BYTE_FLIP_MASK, AVX_256bit); | |
| vpshufb(xmm3, xmm3, BYTE_FLIP_MASK, AVX_256bit); | |
| // transpose data into high/low halves | |
| vperm2i128(xmm4, xmm0, xmm2, 0x20); | |
| vperm2i128(xmm5, xmm0, xmm2, 0x31); | |
| vperm2i128(xmm6, xmm1, xmm3, 0x20); | |
| vperm2i128(xmm7, xmm1, xmm3, 0x31); | |
| bind(last_block_enter); | |
| addq(INP, 64); | |
| movq(Address(rsp, _INP), INP); | |
| //;; schedule 48 input dwords, by doing 3 rounds of 12 each | |
| xorq(SRND, SRND); | |
| align(16); | |
| bind(loop1); | |
| vpaddd(xmm9, xmm4, Address(TBL, SRND, Address::times_1, 0*32), AVX_256bit); | |
| vmovdqu(Address(rsp, SRND, Address::times_1, _XFER + 0*32), xmm9); | |
| sha256_AVX2_one_round_and_sched(xmm4, xmm5, xmm6, xmm7, rax, rbx, rdi, rsi, r8, r9, r10, r11, 0); | |
| sha256_AVX2_one_round_and_sched(xmm4, xmm5, xmm6, xmm7, r11, rax, rbx, rdi, rsi, r8, r9, r10, 1); | |
| sha256_AVX2_one_round_and_sched(xmm4, xmm5, xmm6, xmm7, r10, r11, rax, rbx, rdi, rsi, r8, r9, 2); | |
| sha256_AVX2_one_round_and_sched(xmm4, xmm5, xmm6, xmm7, r9, r10, r11, rax, rbx, rdi, rsi, r8, 3); | |
| vpaddd(xmm9, xmm5, Address(TBL, SRND, Address::times_1, 1*32), AVX_256bit); | |
| vmovdqu(Address(rsp, SRND, Address::times_1, _XFER + 1*32), xmm9); | |
| sha256_AVX2_one_round_and_sched(xmm5, xmm6, xmm7, xmm4, r8, r9, r10, r11, rax, rbx, rdi, rsi, 8+0); | |
| sha256_AVX2_one_round_and_sched(xmm5, xmm6, xmm7, xmm4, rsi, r8, r9, r10, r11, rax, rbx, rdi, 8+1); | |
| sha256_AVX2_one_round_and_sched(xmm5, xmm6, xmm7, xmm4, rdi, rsi, r8, r9, r10, r11, rax, rbx, 8+2); | |
| sha256_AVX2_one_round_and_sched(xmm5, xmm6, xmm7, xmm4, rbx, rdi, rsi, r8, r9, r10, r11, rax, 8+3); | |
| vpaddd(xmm9, xmm6, Address(TBL, SRND, Address::times_1, 2*32), AVX_256bit); | |
| vmovdqu(Address(rsp, SRND, Address::times_1, _XFER + 2*32), xmm9); | |
| sha256_AVX2_one_round_and_sched(xmm6, xmm7, xmm4, xmm5, rax, rbx, rdi, rsi, r8, r9, r10, r11, 16+0); | |
| sha256_AVX2_one_round_and_sched(xmm6, xmm7, xmm4, xmm5, r11, rax, rbx, rdi, rsi, r8, r9, r10, 16+1); | |
| sha256_AVX2_one_round_and_sched(xmm6, xmm7, xmm4, xmm5, r10, r11, rax, rbx, rdi, rsi, r8, r9, 16+2); | |
| sha256_AVX2_one_round_and_sched(xmm6, xmm7, xmm4, xmm5, r9, r10, r11, rax, rbx, rdi, rsi, r8, 16+3); | |
| vpaddd(xmm9, xmm7, Address(TBL, SRND, Address::times_1, 3*32), AVX_256bit); | |
| vmovdqu(Address(rsp, SRND, Address::times_1, _XFER + 3*32), xmm9); | |
| sha256_AVX2_one_round_and_sched(xmm7, xmm4, xmm5, xmm6, r8, r9, r10, r11, rax, rbx, rdi, rsi, 24+0); | |
| sha256_AVX2_one_round_and_sched(xmm7, xmm4, xmm5, xmm6, rsi, r8, r9, r10, r11, rax, rbx, rdi, 24+1); | |
| sha256_AVX2_one_round_and_sched(xmm7, xmm4, xmm5, xmm6, rdi, rsi, r8, r9, r10, r11, rax, rbx, 24+2); | |
| sha256_AVX2_one_round_and_sched(xmm7, xmm4, xmm5, xmm6, rbx, rdi, rsi, r8, r9, r10, r11, rax, 24+3); | |
| addq(SRND, 4*32); | |
| cmpq(SRND, 3 * 4*32); | |
| jcc(Assembler::below, loop1); | |
| bind(loop2); | |
| // Do last 16 rounds with no scheduling | |
| vpaddd(xmm9, xmm4, Address(TBL, SRND, Address::times_1, 0*32), AVX_256bit); | |
| vmovdqu(Address(rsp, SRND, Address::times_1, _XFER + 0*32), xmm9); | |
| sha256_AVX2_four_rounds_compute_first(0); | |
| vpaddd(xmm9, xmm5, Address(TBL, SRND, Address::times_1, 1*32), AVX_256bit); | |
| vmovdqu(Address(rsp, SRND, Address::times_1, _XFER + 1*32), xmm9); | |
| sha256_AVX2_four_rounds_compute_last(0 + 8); | |
| addq(SRND, 2*32); | |
| vmovdqu(xmm4, xmm6); | |
| vmovdqu(xmm5, xmm7); | |
| cmpq(SRND, 4 * 4*32); | |
| jcc(Assembler::below, loop2); | |
| movq(CTX, Address(rsp, _CTX)); | |
| movq(INP, Address(rsp, _INP)); | |
| addm(4*0, CTX, a); | |
| addm(4*1, CTX, b); | |
| addm(4*2, CTX, c); | |
| addm(4*3, CTX, d); | |
| addm(4*4, CTX, e); | |
| addm(4*5, CTX, f); | |
| addm(4*6, CTX, g); | |
| addm(4*7, CTX, h); | |
| cmpq(INP, Address(rsp, _INP_END)); | |
| jcc(Assembler::above, done_hash); | |
| //Do second block using previously scheduled results | |
| xorq(SRND, SRND); | |
| align(16); | |
| bind(loop3); | |
| sha256_AVX2_four_rounds_compute_first(4); | |
| sha256_AVX2_four_rounds_compute_last(4+8); | |
| addq(SRND, 2*32); | |
| cmpq(SRND, 4 * 4*32); | |
| jcc(Assembler::below, loop3); | |
| movq(CTX, Address(rsp, _CTX)); | |
| movq(INP, Address(rsp, _INP)); | |
| addq(INP, 64); | |
| addm(4*0, CTX, a); | |
| addm(4*1, CTX, b); | |
| addm(4*2, CTX, c); | |
| addm(4*3, CTX, d); | |
| addm(4*4, CTX, e); | |
| addm(4*5, CTX, f); | |
| addm(4*6, CTX, g); | |
| addm(4*7, CTX, h); | |
| cmpq(INP, Address(rsp, _INP_END)); | |
| jcc(Assembler::below, loop0); | |
| jccb(Assembler::above, done_hash); | |
| bind(do_last_block); | |
| lea(TBL, ExternalAddress(K256_W)); | |
| movdqu(xmm4, Address(INP, 0*16)); | |
| movdqu(xmm5, Address(INP, 1*16)); | |
| movdqu(xmm6, Address(INP, 2*16)); | |
| movdqu(xmm7, Address(INP, 3*16)); | |
| vpshufb(xmm4, xmm4, xmm13, AVX_128bit); | |
| vpshufb(xmm5, xmm5, xmm13, AVX_128bit); | |
| vpshufb(xmm6, xmm6, xmm13, AVX_128bit); | |
| vpshufb(xmm7, xmm7, xmm13, AVX_128bit); | |
| jmp(last_block_enter); | |
| bind(only_one_block); | |
| // load initial digest ;; table should be preloaded with following values | |
| movl(a, Address(CTX, 4*0)); // 0x6a09e667 | |
| movl(b, Address(CTX, 4*1)); // 0xbb67ae85 | |
| movl(c, Address(CTX, 4*2)); // 0x3c6ef372 | |
| movl(d, Address(CTX, 4*3)); // 0xa54ff53a | |
| movl(e, Address(CTX, 4*4)); // 0x510e527f | |
| movl(f, Address(CTX, 4*5)); // 0x9b05688c | |
| // load g - r10 after use as scratch | |
| movl(h, Address(CTX, 4*7)); // 0x5be0cd19 | |
| pshuffle_byte_flip_mask_addr = pshuffle_byte_flip_mask; | |
| vmovdqu(BYTE_FLIP_MASK, ExternalAddress(pshuffle_byte_flip_mask_addr + 0)); // [PSHUFFLE_BYTE_FLIP_MASK wrt rip] | |
| vmovdqu(SHUF_00BA, ExternalAddress(pshuffle_byte_flip_mask_addr + 32)); // [_SHUF_00BA wrt rip] | |
| vmovdqu(SHUF_DC00, ExternalAddress(pshuffle_byte_flip_mask_addr + 64)); // [_SHUF_DC00 wrt rip] | |
| movl(g, Address(CTX, 4*6)); // 0x1f83d9ab | |
| movq(Address(rsp, _CTX), CTX); | |
| jmpb(do_last_block); | |
| bind(done_hash); | |
| movq(rsp, Address(rsp, _RSP)); | |
| pop(r15); | |
| pop(r14); | |
| pop(r13); | |
| pop(r12); | |
| pop(rbp); | |
| #ifdef _WIN64 | |
| pop(rdi); | |
| pop(rsi); | |
| #endif | |
| pop(rbx); | |
| #ifdef _WIN64 | |
| pop(r9); | |
| pop(r8); | |
| #else | |
| pop(rdx); | |
| pop(rcx); | |
| #endif | |
| if (multi_block) { | |
| #ifdef _WIN64 | |
| const Register& limit_end = r9; | |
| const Register& ofs_end = r8; | |
| #else | |
| const Register& limit_end = rcx; | |
| const Register& ofs_end = rdx; | |
| #endif | |
| movq(rax, ofs_end); | |
| bind(compute_size1); | |
| cmpptr(rax, limit_end); // assume the original ofs <= limit | |
| jccb(Assembler::aboveEqual, compute_size_end1); | |
| addq(rax, 64); | |
| jmpb(compute_size1); | |
| bind(compute_size_end1); | |
| } | |
| } | |
| void MacroAssembler::sha512_AVX2_one_round_compute(Register old_h, Register a, Register b, Register c, | |
| Register d, Register e, Register f, Register g, Register h, | |
| int iteration) | |
| { | |
| const Register& y0 = r13; | |
| const Register& y1 = r14; | |
| const Register& y2 = r15; | |
| #ifdef _WIN64 | |
| const Register& y3 = rcx; | |
| #else | |
| const Register& y3 = rdi; | |
| #endif | |
| const Register& T1 = r12; | |
| if (iteration % 4 > 0) { | |
| addq(old_h, y2); //h = k + w + h + S0 + S1 + CH = t1 + S0; | |
| } | |
| movq(y2, f); //y2 = f; CH | |
| rorxq(y0, e, 41); //y0 = e >> 41; S1A | |
| rorxq(y1, e, 18); //y1 = e >> 18; S1B | |
| xorq(y2, g); //y2 = f^g; CH | |
| xorq(y0, y1); //y0 = (e >> 41) ^ (e >> 18); S1 | |
| rorxq(y1, e, 14); //y1 = (e >> 14); S1 | |
| andq(y2, e); //y2 = (f^g)&e; CH | |
| if (iteration % 4 > 0 ) { | |
| addq(old_h, y3); //h = t1 + S0 + MAJ | |
| } | |
| xorq(y0, y1); //y0 = (e >> 41) ^ (e >> 18) ^ (e >> 14); S1 | |
| rorxq(T1, a, 34); //T1 = a >> 34; S0B | |
| xorq(y2, g); //y2 = CH = ((f^g)&e) ^g; CH | |
| rorxq(y1, a, 39); //y1 = a >> 39; S0A | |
| movq(y3, a); //y3 = a; MAJA | |
| xorq(y1, T1); //y1 = (a >> 39) ^ (a >> 34); S0 | |
| rorxq(T1, a, 28); //T1 = (a >> 28); S0 | |
| addq(h, Address(rsp, (8 * iteration))); //h = k + w + h; -- | |
| orq(y3, c); //y3 = a | c; MAJA | |
| xorq(y1, T1); //y1 = (a >> 39) ^ (a >> 34) ^ (a >> 28); S0 | |
| movq(T1, a); //T1 = a; MAJB | |
| andq(y3, b); //y3 = (a | c)&b; MAJA | |
| andq(T1, c); //T1 = a&c; MAJB | |
| addq(y2, y0); //y2 = S1 + CH; -- | |
| addq(d, h); //d = k + w + h + d; -- | |
| orq(y3, T1); //y3 = MAJ = (a | c)&b) | (a&c); MAJ | |
| addq(h, y1); //h = k + w + h + S0; -- | |
| addq(d, y2); //d = k + w + h + d + S1 + CH = d + t1; -- | |
| if (iteration % 4 == 3) { | |
| addq(h, y2); //h = k + w + h + S0 + S1 + CH = t1 + S0; -- | |
| addq(h, y3); //h = t1 + S0 + MAJ; -- | |
| } | |
| } | |
| void MacroAssembler::sha512_AVX2_one_round_and_schedule( | |
| XMMRegister xmm4, // ymm4 | |
| XMMRegister xmm5, // ymm5 | |
| XMMRegister xmm6, // ymm6 | |
| XMMRegister xmm7, // ymm7 | |
| Register a, //rax | |
| Register b, //rbx | |
| Register c, //rdi | |
| Register d, //rsi | |
| Register e, //r8 | |
| Register f, //r9 | |
| Register g, //r10 | |
| Register h, //r11 | |
| int iteration) | |
| { | |
| const Register& y0 = r13; | |
| const Register& y1 = r14; | |
| const Register& y2 = r15; | |
| #ifdef _WIN64 | |
| const Register& y3 = rcx; | |
| #else | |
| const Register& y3 = rdi; | |
| #endif | |
| const Register& T1 = r12; | |
| if (iteration % 4 == 0) { | |
| // Extract w[t - 7] | |
| // xmm0 = W[-7] | |
| vperm2f128(xmm0, xmm7, xmm6, 3); | |
| vpalignr(xmm0, xmm0, xmm6, 8, AVX_256bit); | |
| // Calculate w[t - 16] + w[t - 7] | |
| vpaddq(xmm0, xmm0, xmm4, AVX_256bit); //xmm0 = W[-7] + W[-16] | |
| // Extract w[t - 15] | |
| //xmm1 = W[-15] | |
| vperm2f128(xmm1, xmm5, xmm4, 3); | |
| vpalignr(xmm1, xmm1, xmm4, 8, AVX_256bit); | |
| // Calculate sigma0 | |
| // Calculate w[t - 15] ror 1 | |
| vpsrlq(xmm2, xmm1, 1, AVX_256bit); | |
| vpsllq(xmm3, xmm1, (64 - 1), AVX_256bit); | |
| vpor(xmm3, xmm3, xmm2, AVX_256bit); //xmm3 = W[-15] ror 1 | |
| // Calculate w[t - 15] shr 7 | |
| vpsrlq(xmm8, xmm1, 7, AVX_256bit); //xmm8 = W[-15] >> 7 | |
| } else if (iteration % 4 == 1) { | |
| //Calculate w[t - 15] ror 8 | |
| vpsrlq(xmm2, xmm1, 8, AVX_256bit); | |
| vpsllq(xmm1, xmm1, (64 - 8), AVX_256bit); | |
| vpor(xmm1, xmm1, xmm2, AVX_256bit); //xmm1 = W[-15] ror 8 | |
| //XOR the three components | |
| vpxor(xmm3, xmm3, xmm8, AVX_256bit); //xmm3 = W[-15] ror 1 ^ W[-15] >> 7 | |
| vpxor(xmm1, xmm3, xmm1, AVX_256bit); //xmm1 = s0 | |
| //Add three components, w[t - 16], w[t - 7] and sigma0 | |
| vpaddq(xmm0, xmm0, xmm1, AVX_256bit); //xmm0 = W[-16] + W[-7] + s0 | |
| // Move to appropriate lanes for calculating w[16] and w[17] | |
| vperm2f128(xmm4, xmm0, xmm0, 0); //xmm4 = W[-16] + W[-7] + s0{ BABA } | |
| //Move to appropriate lanes for calculating w[18] and w[19] | |
| vpand(xmm0, xmm0, xmm10, AVX_256bit); //xmm0 = W[-16] + W[-7] + s0{ DC00 } | |
| //Calculate w[16] and w[17] in both 128 bit lanes | |
| //Calculate sigma1 for w[16] and w[17] on both 128 bit lanes | |
| vperm2f128(xmm2, xmm7, xmm7, 17); //xmm2 = W[-2] {BABA} | |
| vpsrlq(xmm8, xmm2, 6, AVX_256bit); //xmm8 = W[-2] >> 6 {BABA} | |
| } else if (iteration % 4 == 2) { | |
| vpsrlq(xmm3, xmm2, 19, AVX_256bit); //xmm3 = W[-2] >> 19 {BABA} | |
| vpsllq(xmm1, xmm2, (64 - 19), AVX_256bit); //xmm1 = W[-2] << 19 {BABA} | |
| vpor(xmm3, xmm3, xmm1, AVX_256bit); //xmm3 = W[-2] ror 19 {BABA} | |
| vpxor(xmm8, xmm8, xmm3, AVX_256bit);// xmm8 = W[-2] ror 19 ^ W[-2] >> 6 {BABA} | |
| vpsrlq(xmm3, xmm2, 61, AVX_256bit); //xmm3 = W[-2] >> 61 {BABA} | |
| vpsllq(xmm1, xmm2, (64 - 61), AVX_256bit); //xmm1 = W[-2] << 61 {BABA} | |
| vpor(xmm3, xmm3, xmm1, AVX_256bit); //xmm3 = W[-2] ror 61 {BABA} | |
| vpxor(xmm8, xmm8, xmm3, AVX_256bit); //xmm8 = s1 = (W[-2] ror 19) ^ (W[-2] ror 61) ^ (W[-2] >> 6) { BABA } | |
| //Add sigma1 to the other components to get w[16] and w[17] | |
| vpaddq(xmm4, xmm4, xmm8, AVX_256bit); //xmm4 = { W[1], W[0], W[1], W[0] } | |
| //Calculate sigma1 for w[18] and w[19] for upper 128 bit lane | |
| vpsrlq(xmm8, xmm4, 6, AVX_256bit); //xmm8 = W[-2] >> 6 {DC--} | |
| } else if (iteration % 4 == 3){ | |
| vpsrlq(xmm3, xmm4, 19, AVX_256bit); //xmm3 = W[-2] >> 19 {DC--} | |
| vpsllq(xmm1, xmm4, (64 - 19), AVX_256bit); //xmm1 = W[-2] << 19 {DC--} | |
| vpor(xmm3, xmm3, xmm1, AVX_256bit); //xmm3 = W[-2] ror 19 {DC--} | |
| vpxor(xmm8, xmm8, xmm3, AVX_256bit); //xmm8 = W[-2] ror 19 ^ W[-2] >> 6 {DC--} | |
| vpsrlq(xmm3, xmm4, 61, AVX_256bit); //xmm3 = W[-2] >> 61 {DC--} | |
| vpsllq(xmm1, xmm4, (64 - 61), AVX_256bit); //xmm1 = W[-2] << 61 {DC--} | |
| vpor(xmm3, xmm3, xmm1, AVX_256bit); //xmm3 = W[-2] ror 61 {DC--} | |
| vpxor(xmm8, xmm8, xmm3, AVX_256bit); //xmm8 = s1 = (W[-2] ror 19) ^ (W[-2] ror 61) ^ (W[-2] >> 6) { DC-- } | |
| //Add the sigma0 + w[t - 7] + w[t - 16] for w[18] and w[19] to newly calculated sigma1 to get w[18] and w[19] | |
| vpaddq(xmm2, xmm0, xmm8, AVX_256bit); //xmm2 = { W[3], W[2], --, -- } | |
| //Form w[19, w[18], w17], w[16] | |
| vpblendd(xmm4, xmm4, xmm2, 0xF0, AVX_256bit); //xmm4 = { W[3], W[2], W[1], W[0] } | |
| } | |
| movq(y3, a); //y3 = a; MAJA | |
| rorxq(y0, e, 41); // y0 = e >> 41; S1A | |
| rorxq(y1, e, 18); //y1 = e >> 18; S1B | |
| addq(h, Address(rsp, (iteration * 8))); //h = k + w + h; -- | |
| orq(y3, c); //y3 = a | c; MAJA | |
| movq(y2, f); //y2 = f; CH | |
| xorq(y2, g); //y2 = f^g; CH | |
| rorxq(T1, a, 34); //T1 = a >> 34; S0B | |
| xorq(y0, y1); //y0 = (e >> 41) ^ (e >> 18); S1 | |
| rorxq(y1, e, 14); //y1 = (e >> 14); S1 | |
| andq(y2, e); //y2 = (f^g) & e; CH | |
| addq(d, h); //d = k + w + h + d; -- | |
| andq(y3, b); //y3 = (a | c)&b; MAJA | |
| xorq(y0, y1); //y0 = (e >> 41) ^ (e >> 18) ^ (e >> 14); S1 | |
| rorxq(y1, a, 39); //y1 = a >> 39; S0A | |
| xorq(y1, T1); //y1 = (a >> 39) ^ (a >> 34); S0 | |
| rorxq(T1, a, 28); //T1 = (a >> 28); S0 | |
| xorq(y2, g); //y2 = CH = ((f^g)&e) ^ g; CH | |
| xorq(y1, T1); //y1 = (a >> 39) ^ (a >> 34) ^ (a >> 28); S0 | |
| movq(T1, a); //T1 = a; MAJB | |
| andq(T1, c); //T1 = a&c; MAJB | |
| addq(y2, y0); //y2 = S1 + CH; -- | |
| orq(y3, T1); //y3 = MAJ = (a | c)&b) | (a&c); MAJ | |
| addq(h, y1); //h = k + w + h + S0; -- | |
| addq(d, y2); //d = k + w + h + d + S1 + CH = d + t1; -- | |
| addq(h, y2); //h = k + w + h + S0 + S1 + CH = t1 + S0; -- | |
| addq(h, y3); //h = t1 + S0 + MAJ; -- | |
| } | |
| void MacroAssembler::sha512_AVX2(XMMRegister msg, XMMRegister state0, XMMRegister state1, XMMRegister msgtmp0, | |
| XMMRegister msgtmp1, XMMRegister msgtmp2, XMMRegister msgtmp3, XMMRegister msgtmp4, | |
| Register buf, Register state, Register ofs, Register limit, Register rsp, | |
| bool multi_block, XMMRegister shuf_mask) | |
| { | |
| Label loop0, loop1, loop2, done_hash, | |
| compute_block_size, compute_size, | |
| compute_block_size_end, compute_size_end; | |
| address K512_W = StubRoutines::x86::k512_W_addr(); | |
| address pshuffle_byte_flip_mask_sha512 = StubRoutines::x86::pshuffle_byte_flip_mask_addr_sha512(); | |
| address pshuffle_byte_flip_mask_addr = nullptr; | |
| const XMMRegister& XFER = xmm0; // YTMP0 | |
| const XMMRegister& BYTE_FLIP_MASK = xmm9; // ymm9 | |
| const XMMRegister& YMM_MASK_LO = xmm10; // ymm10 | |
| #ifdef _WIN64 | |
| const Register& INP = rcx; //1st arg | |
| const Register& CTX = rdx; //2nd arg | |
| const Register& NUM_BLKS = r8; //3rd arg | |
| const Register& c = rdi; | |
| const Register& d = rsi; | |
| const Register& e = r8; | |
| const Register& y3 = rcx; | |
| const Register& offset = r8; | |
| const Register& input_limit = r9; | |
| #else | |
| const Register& INP = rdi; //1st arg | |
| const Register& CTX = rsi; //2nd arg | |
| const Register& NUM_BLKS = rdx; //3rd arg | |
| const Register& c = rcx; | |
| const Register& d = r8; | |
| const Register& e = rdx; | |
| const Register& y3 = rdi; | |
| const Register& offset = rdx; | |
| const Register& input_limit = rcx; | |
| #endif | |
| const Register& TBL = rbp; | |
| const Register& a = rax; | |
| const Register& b = rbx; | |
| const Register& f = r9; | |
| const Register& g = r10; | |
| const Register& h = r11; | |
| //Local variables as defined in assembly file. | |
| enum | |
| { | |
| _XFER_SIZE = 4 * 8, // resq 4 => reserve 4 quadwords. Hence 4 * 8 | |
| _SRND_SIZE = 8, // resq 1 | |
| _INP_SIZE = 8, | |
| _INP_END_SIZE = 8, | |
| _RSP_SAVE_SIZE = 8, // defined as resq 1 | |
| #ifdef _WIN64 | |
| _GPR_SAVE_SIZE = 8 * 8, // defined as resq 8 | |
| #else | |
| _GPR_SAVE_SIZE = 6 * 8 // resq 6 | |
| #endif | |
| }; | |
| enum | |
| { | |
| _XFER = 0, | |
| _SRND = _XFER + _XFER_SIZE, // 32 | |
| _INP = _SRND + _SRND_SIZE, // 40 | |
| _INP_END = _INP + _INP_SIZE, // 48 | |
| _RSP = _INP_END + _INP_END_SIZE, // 56 | |
| _GPR = _RSP + _RSP_SAVE_SIZE, // 64 | |
| _STACK_SIZE = _GPR + _GPR_SAVE_SIZE // 128 for windows and 112 for linux. | |
| }; | |
| //Saving offset and limit as it will help with blocksize calculation for multiblock SHA512. | |
| #ifdef _WIN64 | |
| push(r8); // win64: this is ofs | |
| push(r9); // win64: this is limit, we need them again at the very end. | |
| #else | |
| push(rdx); // linux : this is ofs, need at the end for multiblock calculation | |
| push(rcx); // linux: This is the limit. | |
| #endif | |
| //Allocate Stack Space | |
| movq(rax, rsp); | |
| subq(rsp, _STACK_SIZE); | |
| andq(rsp, -32); | |
| movq(Address(rsp, _RSP), rax); | |
| //Save GPRs | |
| movq(Address(rsp, _GPR), rbp); | |
| movq(Address(rsp, (_GPR + 8)), rbx); | |
| movq(Address(rsp, (_GPR + 16)), r12); | |
| movq(Address(rsp, (_GPR + 24)), r13); | |
| movq(Address(rsp, (_GPR + 32)), r14); | |
| movq(Address(rsp, (_GPR + 40)), r15); | |
| #ifdef _WIN64 | |
| movq(Address(rsp, (_GPR + 48)), rsi); | |
| movq(Address(rsp, (_GPR + 56)), rdi); | |
| #endif | |
| vpblendd(xmm0, xmm0, xmm1, 0xF0, AVX_128bit); | |
| vpblendd(xmm0, xmm0, xmm1, 0xF0, AVX_256bit); | |
| if (multi_block) { | |
| xorq(rax, rax); | |
| bind(compute_block_size); | |
| cmpptr(offset, input_limit); // Assuming that offset is less than limit. | |
| jccb(Assembler::aboveEqual, compute_block_size_end); | |
| addq(offset, 128); | |
| addq(rax, 128); | |
| jmpb(compute_block_size); | |
| bind(compute_block_size_end); | |
| movq(NUM_BLKS, rax); | |
| cmpq(NUM_BLKS, 0); | |
| jcc(Assembler::equal, done_hash); | |
| } else { | |
| xorq(NUM_BLKS, NUM_BLKS); //If single block. | |
| addq(NUM_BLKS, 128); | |
| } | |
| addq(NUM_BLKS, INP); //pointer to end of data | |
| movq(Address(rsp, _INP_END), NUM_BLKS); | |
| //load initial digest | |
| movq(a, Address(CTX, 8 * 0)); | |
| movq(b, Address(CTX, 8 * 1)); | |
| movq(c, Address(CTX, 8 * 2)); | |
| movq(d, Address(CTX, 8 * 3)); | |
| movq(e, Address(CTX, 8 * 4)); | |
| movq(f, Address(CTX, 8 * 5)); | |
| // load g - r10 after it is used as scratch | |
| movq(h, Address(CTX, 8 * 7)); | |
| pshuffle_byte_flip_mask_addr = pshuffle_byte_flip_mask_sha512; | |
| vmovdqu(BYTE_FLIP_MASK, ExternalAddress(pshuffle_byte_flip_mask_addr + 0)); // PSHUFFLE_BYTE_FLIP_MASK wrt rip | |
| vmovdqu(YMM_MASK_LO, ExternalAddress(pshuffle_byte_flip_mask_addr + 32)); | |
| movq(g, Address(CTX, 8 * 6)); | |
| bind(loop0); | |
| lea(TBL, ExternalAddress(K512_W)); | |
| //byte swap first 16 dwords | |
| vmovdqu(xmm4, Address(INP, 32 * 0)); | |
| vpshufb(xmm4, xmm4, BYTE_FLIP_MASK, AVX_256bit); | |
| vmovdqu(xmm5, Address(INP, 32 * 1)); | |
| vpshufb(xmm5, xmm5, BYTE_FLIP_MASK, AVX_256bit); | |
| vmovdqu(xmm6, Address(INP, 32 * 2)); | |
| vpshufb(xmm6, xmm6, BYTE_FLIP_MASK, AVX_256bit); | |
| vmovdqu(xmm7, Address(INP, 32 * 3)); | |
| vpshufb(xmm7, xmm7, BYTE_FLIP_MASK, AVX_256bit); | |
| movq(Address(rsp, _INP), INP); | |
| movslq(Address(rsp, _SRND), 4); | |
| align(16); | |
| //Schedule 64 input dwords, by calling sha512_AVX2_one_round_and_schedule | |
| bind(loop1); | |
| vpaddq(xmm0, xmm4, Address(TBL, 0 * 32), AVX_256bit); | |
| vmovdqu(Address(rsp, _XFER), xmm0); | |
| //four rounds and schedule | |
| sha512_AVX2_one_round_and_schedule(xmm4, xmm5, xmm6, xmm7, a, b, c, d, e, f, g, h, 0); | |
| sha512_AVX2_one_round_and_schedule(xmm4, xmm5, xmm6, xmm7, h, a, b, c, d, e, f, g, 1); | |
| sha512_AVX2_one_round_and_schedule(xmm4, xmm5, xmm6, xmm7, g, h, a, b, c, d, e, f, 2); | |
| sha512_AVX2_one_round_and_schedule(xmm4, xmm5, xmm6, xmm7, f, g, h, a, b, c, d, e, 3); | |
| vpaddq(xmm0, xmm5, Address(TBL, 1 * 32), AVX_256bit); | |
| vmovdqu(Address(rsp, _XFER), xmm0); | |
| //four rounds and schedule | |
| sha512_AVX2_one_round_and_schedule(xmm5, xmm6, xmm7, xmm4, e, f, g, h, a, b, c, d, 0); | |
| sha512_AVX2_one_round_and_schedule(xmm5, xmm6, xmm7, xmm4, d, e, f, g, h, a, b, c, 1); | |
| sha512_AVX2_one_round_and_schedule(xmm5, xmm6, xmm7, xmm4, c, d, e, f, g, h, a, b, 2); | |
| sha512_AVX2_one_round_and_schedule(xmm5, xmm6, xmm7, xmm4, b, c, d, e, f, g, h, a, 3); | |
| vpaddq(xmm0, xmm6, Address(TBL, 2 * 32), AVX_256bit); | |
| vmovdqu(Address(rsp, _XFER), xmm0); | |
| //four rounds and schedule | |
| sha512_AVX2_one_round_and_schedule(xmm6, xmm7, xmm4, xmm5, a, b, c, d, e, f, g, h, 0); | |
| sha512_AVX2_one_round_and_schedule(xmm6, xmm7, xmm4, xmm5, h, a, b, c, d, e, f, g, 1); | |
| sha512_AVX2_one_round_and_schedule(xmm6, xmm7, xmm4, xmm5, g, h, a, b, c, d, e, f, 2); | |
| sha512_AVX2_one_round_and_schedule(xmm6, xmm7, xmm4, xmm5, f, g, h, a, b, c, d, e, 3); | |
| vpaddq(xmm0, xmm7, Address(TBL, 3 * 32), AVX_256bit); | |
| vmovdqu(Address(rsp, _XFER), xmm0); | |
| addq(TBL, 4 * 32); | |
| //four rounds and schedule | |
| sha512_AVX2_one_round_and_schedule(xmm7, xmm4, xmm5, xmm6, e, f, g, h, a, b, c, d, 0); | |
| sha512_AVX2_one_round_and_schedule(xmm7, xmm4, xmm5, xmm6, d, e, f, g, h, a, b, c, 1); | |
| sha512_AVX2_one_round_and_schedule(xmm7, xmm4, xmm5, xmm6, c, d, e, f, g, h, a, b, 2); | |
| sha512_AVX2_one_round_and_schedule(xmm7, xmm4, xmm5, xmm6, b, c, d, e, f, g, h, a, 3); | |
| subq(Address(rsp, _SRND), 1); | |
| jcc(Assembler::notEqual, loop1); | |
| movslq(Address(rsp, _SRND), 2); | |
| bind(loop2); | |
| vpaddq(xmm0, xmm4, Address(TBL, 0 * 32), AVX_256bit); | |
| vmovdqu(Address(rsp, _XFER), xmm0); | |
| //four rounds and compute. | |
| sha512_AVX2_one_round_compute(a, a, b, c, d, e, f, g, h, 0); | |
| sha512_AVX2_one_round_compute(h, h, a, b, c, d, e, f, g, 1); | |
| sha512_AVX2_one_round_compute(g, g, h, a, b, c, d, e, f, 2); | |
| sha512_AVX2_one_round_compute(f, f, g, h, a, b, c, d, e, 3); | |
| vpaddq(xmm0, xmm5, Address(TBL, 1 * 32), AVX_256bit); | |
| vmovdqu(Address(rsp, _XFER), xmm0); | |
| addq(TBL, 2 * 32); | |
| // four rounds and compute. | |
| sha512_AVX2_one_round_compute(e, e, f, g, h, a, b, c, d, 0); | |
| sha512_AVX2_one_round_compute(d, d, e, f, g, h, a, b, c, 1); | |
| sha512_AVX2_one_round_compute(c, c, d, e, f, g, h, a, b, 2); | |
| sha512_AVX2_one_round_compute(b, b, c, d, e, f, g, h, a, 3); | |
| vmovdqu(xmm4, xmm6); | |
| vmovdqu(xmm5, xmm7); | |
| subq(Address(rsp, _SRND), 1); | |
| jcc(Assembler::notEqual, loop2); | |
| addmq(8 * 0, CTX, a); | |
| addmq(8 * 1, CTX, b); | |
| addmq(8 * 2, CTX, c); | |
| addmq(8 * 3, CTX, d); | |
| addmq(8 * 4, CTX, e); | |
| addmq(8 * 5, CTX, f); | |
| addmq(8 * 6, CTX, g); | |
| addmq(8 * 7, CTX, h); | |
| movq(INP, Address(rsp, _INP)); | |
| addq(INP, 128); | |
| cmpq(INP, Address(rsp, _INP_END)); | |
| jcc(Assembler::notEqual, loop0); | |
| bind(done_hash); | |
| //Restore GPRs | |
| movq(rbp, Address(rsp, (_GPR + 0))); | |
| movq(rbx, Address(rsp, (_GPR + 8))); | |
| movq(r12, Address(rsp, (_GPR + 16))); | |
| movq(r13, Address(rsp, (_GPR + 24))); | |
| movq(r14, Address(rsp, (_GPR + 32))); | |
| movq(r15, Address(rsp, (_GPR + 40))); | |
| #ifdef _WIN64 | |
| movq(rsi, Address(rsp, (_GPR + 48))); | |
| movq(rdi, Address(rsp, (_GPR + 56))); | |
| #endif | |
| //Restore Stack Pointer | |
| movq(rsp, Address(rsp, _RSP)); | |
| #ifdef _WIN64 | |
| pop(r9); | |
| pop(r8); | |
| #else | |
| pop(rcx); | |
| pop(rdx); | |
| #endif | |
| if (multi_block) { | |
| #ifdef _WIN64 | |
| const Register& limit_end = r9; | |
| const Register& ofs_end = r8; | |
| #else | |
| const Register& limit_end = rcx; | |
| const Register& ofs_end = rdx; | |
| #endif | |
| movq(rax, ofs_end); | |
| bind(compute_size); | |
| cmpptr(rax, limit_end); | |
| jccb(Assembler::aboveEqual, compute_size_end); | |
| addq(rax, 128); | |
| jmpb(compute_size); | |
| bind(compute_size_end); | |
| } | |
| } | |
| //Implemented using Intel IpSec implementation (intel-ipsec-mb on github) | |
| void MacroAssembler::sha512_update_ni_x1(Register arg_hash, Register arg_msg, Register ofs, Register limit, bool multi_block) { | |
| Label done_hash, block_loop; | |
| address K512_W = StubRoutines::x86::k512_W_addr(); | |
| vbroadcasti128(xmm15, ExternalAddress(StubRoutines::x86::pshuffle_byte_flip_mask_addr_sha512()), Assembler::AVX_256bit, r10); | |
| //load current hash value and transform | |
| vmovdqu(xmm0, Address(arg_hash)); | |
| vmovdqu(xmm1, Address(arg_hash, 32)); | |
| //ymm0 = D C B A, ymm1 = H G F E | |
| vperm2i128(xmm2, xmm0, xmm1, 0x20); | |
| vperm2i128(xmm3, xmm0, xmm1, 0x31); | |
| //ymm2 = F E B A, ymm3 = H G D C | |
| vpermq(xmm13, xmm2, 0x1b, Assembler::AVX_256bit); | |
| vpermq(xmm14, xmm3, 0x1b, Assembler::AVX_256bit); | |
| //ymm13 = A B E F, ymm14 = C D G H | |
| lea(rax, ExternalAddress(K512_W)); | |
| align(CodeEntryAlignment); | |
| bind(block_loop); | |
| vmovdqu(xmm11, xmm13);//ABEF | |
| vmovdqu(xmm12, xmm14);//CDGH | |
| //R0 - R3 | |
| vmovdqu(xmm0, Address(arg_msg, 0 * 32)); | |
| vpshufb(xmm3, xmm0, xmm15, Assembler::AVX_256bit);//ymm0 / ymm3 = W[0..3] | |
| vpaddq(xmm0, xmm3, Address(rax, 0 * 32), Assembler::AVX_256bit); | |
| sha512rnds2(xmm12, xmm11, xmm0); | |
| vperm2i128(xmm0, xmm0, xmm0, 0x01); | |
| sha512rnds2(xmm11, xmm12, xmm0); | |
| //R4 - R7 | |
| vmovdqu(xmm0, Address(arg_msg, 1 * 32)); | |
| vpshufb(xmm4, xmm0, xmm15, Assembler::AVX_256bit);//ymm0 / ymm4 = W[4..7] | |
| vpaddq(xmm0, xmm4, Address(rax, 1 * 32), Assembler::AVX_256bit); | |
| sha512rnds2(xmm12, xmm11, xmm0); | |
| vperm2i128(xmm0, xmm0, xmm0, 0x01); | |
| sha512rnds2(xmm11, xmm12, xmm0); | |
| sha512msg1(xmm3, xmm4); //ymm3 = W[0..3] + S0(W[1..4]) | |
| //R8 - R11 | |
| vmovdqu(xmm0, Address(arg_msg, 2 * 32)); | |
| vpshufb(xmm5, xmm0, xmm15, Assembler::AVX_256bit);//ymm0 / ymm5 = W[8..11] | |
| vpaddq(xmm0, xmm5, Address(rax, 2 * 32), Assembler::AVX_256bit); | |
| sha512rnds2(xmm12, xmm11, xmm0); | |
| vperm2i128(xmm0, xmm0, xmm0, 0x01); | |
| sha512rnds2(xmm11, xmm12, xmm0); | |
| sha512msg1(xmm4, xmm5);//ymm4 = W[4..7] + S0(W[5..8]) | |
| //R12 - R15 | |
| vmovdqu(xmm0, Address(arg_msg, 3 * 32)); | |
| vpshufb(xmm6, xmm0, xmm15, Assembler::AVX_256bit); //ymm0 / ymm6 = W[12..15] | |
| vpaddq(xmm0, xmm6, Address(rax, 3 * 32), Assembler::AVX_256bit); | |
| vpermq(xmm8, xmm6, 0x1b, Assembler::AVX_256bit); //ymm8 = W[12] W[13] W[14] W[15] | |
| vpermq(xmm9, xmm5, 0x39, Assembler::AVX_256bit); //ymm9 = W[8] W[11] W[10] W[9] | |
| vpblendd(xmm8, xmm8, xmm9, 0x3f, Assembler::AVX_256bit); //ymm8 = W[12] W[11] W[10] W[9] | |
| vpaddq(xmm3, xmm3, xmm8, Assembler::AVX_256bit); | |
| sha512msg2(xmm3, xmm6);//W[16..19] = xmm3 + W[9..12] + S1(W[14..17]) | |
| sha512rnds2(xmm12, xmm11, xmm0); | |
| vperm2i128(xmm0, xmm0, xmm0, 0x01); | |
| sha512rnds2(xmm11, xmm12, xmm0); | |
| sha512msg1(xmm5, xmm6); //ymm5 = W[8..11] + S0(W[9..12]) | |
| //R16 - R19, R32 - R35, R48 - R51 | |
| for (int i = 4, j = 3; j > 0; j--) { | |
| vpaddq(xmm0, xmm3, Address(rax, i * 32), Assembler::AVX_256bit); | |
| vpermq(xmm8, xmm3, 0x1b, Assembler::AVX_256bit);//ymm8 = W[16] W[17] W[18] W[19] | |
| vpermq(xmm9, xmm6, 0x39, Assembler::AVX_256bit);//ymm9 = W[12] W[15] W[14] W[13] | |
| vpblendd(xmm7, xmm8, xmm9, 0x3f, Assembler::AVX_256bit);//xmm7 = W[16] W[15] W[14] W[13] | |
| vpaddq(xmm4, xmm4, xmm7, Assembler::AVX_256bit);//ymm4 = W[4..7] + S0(W[5..8]) + W[13..16] | |
| sha512msg2(xmm4, xmm3);//ymm4 += S1(W[14..17]) | |
| sha512rnds2(xmm12, xmm11, xmm0); | |
| vperm2i128(xmm0, xmm0, xmm0, 0x01); | |
| sha512rnds2(xmm11, xmm12, xmm0); | |
| sha512msg1(xmm6, xmm3); //ymm6 = W[12..15] + S0(W[13..16]) | |
| i += 1; | |
| //R20 - R23, R36 - R39, R52 - R55 | |
| vpaddq(xmm0, xmm4, Address(rax, i * 32), Assembler::AVX_256bit); | |
| vpermq(xmm8, xmm4, 0x1b, Assembler::AVX_256bit);//ymm8 = W[20] W[21] W[22] W[23] | |
| vpermq(xmm9, xmm3, 0x39, Assembler::AVX_256bit);//ymm9 = W[16] W[19] W[18] W[17] | |
| vpblendd(xmm7, xmm8, xmm9, 0x3f, Assembler::AVX_256bit);//ymm7 = W[20] W[19] W[18] W[17] | |
| vpaddq(xmm5, xmm5, xmm7, Assembler::AVX_256bit);//ymm5 = W[8..11] + S0(W[9..12]) + W[17..20] | |
| sha512msg2(xmm5, xmm4);//ymm5 += S1(W[18..21]) | |
| sha512rnds2(xmm12, xmm11, xmm0); | |
| vperm2i128(xmm0, xmm0, xmm0, 0x01); | |
| sha512rnds2(xmm11, xmm12, xmm0); | |
| sha512msg1(xmm3, xmm4); //ymm3 = W[16..19] + S0(W[17..20]) | |
| i += 1; | |
| //R24 - R27, R40 - R43, R56 - R59 | |
| vpaddq(xmm0, xmm5, Address(rax, i * 32), Assembler::AVX_256bit); | |
| vpermq(xmm8, xmm5, 0x1b, Assembler::AVX_256bit);//ymm8 = W[24] W[25] W[26] W[27] | |
| vpermq(xmm9, xmm4, 0x39, Assembler::AVX_256bit);//ymm9 = W[20] W[23] W[22] W[21] | |
| vpblendd(xmm7, xmm8, xmm9, 0x3f, Assembler::AVX_256bit);//ymm7 = W[24] W[23] W[22] W[21] | |
| vpaddq(xmm6, xmm6, xmm7, Assembler::AVX_256bit);//ymm6 = W[12..15] + S0(W[13..16]) + W[21..24] | |
| sha512msg2(xmm6, xmm5);//ymm6 += S1(W[22..25]) | |
| sha512rnds2(xmm12, xmm11, xmm0); | |
| vperm2i128(xmm0, xmm0, xmm0, 0x01); | |
| sha512rnds2(xmm11, xmm12, xmm0); | |
| sha512msg1(xmm4, xmm5);//ymm4 = W[20..23] + S0(W[21..24]) | |
| i += 1; | |
| //R28 - R31, R44 - R47, R60 - R63 | |
| vpaddq(xmm0, xmm6, Address(rax, i * 32), Assembler::AVX_256bit); | |
| vpermq(xmm8, xmm6, 0x1b, Assembler::AVX_256bit);//ymm8 = W[28] W[29] W[30] W[31] | |
| vpermq(xmm9, xmm5, 0x39, Assembler::AVX_256bit);//ymm9 = W[24] W[27] W[26] W[25] | |
| vpblendd(xmm7, xmm8, xmm9, 0x3f, Assembler::AVX_256bit);//ymm7 = W[28] W[27] W[26] W[25] | |
| vpaddq(xmm3, xmm3, xmm7, Assembler::AVX_256bit);//ymm3 = W[16..19] + S0(W[17..20]) + W[25..28] | |
| sha512msg2(xmm3, xmm6); //ymm3 += S1(W[26..29]) | |
| sha512rnds2(xmm12, xmm11, xmm0); | |
| vperm2i128(xmm0, xmm0, xmm0, 0x01); | |
| sha512rnds2(xmm11, xmm12, xmm0); | |
| sha512msg1(xmm5, xmm6);//ymm5 = W[24..27] + S0(W[25..28]) | |
| i += 1; | |
| } | |
| //R64 - R67 | |
| vpaddq(xmm0, xmm3, Address(rax, 16 * 32), Assembler::AVX_256bit); | |
| vpermq(xmm8, xmm3, 0x1b, Assembler::AVX_256bit);//ymm8 = W[64] W[65] W[66] W[67] | |
| vpermq(xmm9, xmm6, 0x39, Assembler::AVX_256bit);//ymm9 = W[60] W[63] W[62] W[61] | |
| vpblendd(xmm7, xmm8, xmm9, 0x3f, Assembler::AVX_256bit);//ymm7 = W[64] W[63] W[62] W[61] | |
| vpaddq(xmm4, xmm4, xmm7, Assembler::AVX_256bit);//ymm4 = W[52..55] + S0(W[53..56]) + W[61..64] | |
| sha512msg2(xmm4, xmm3);//ymm4 += S1(W[62..65]) | |
| sha512rnds2(xmm12, xmm11, xmm0); | |
| vperm2i128(xmm0, xmm0, xmm0, 0x01); | |
| sha512rnds2(xmm11, xmm12, xmm0); | |
| sha512msg1(xmm6, xmm3);//ymm6 = W[60..63] + S0(W[61..64]) | |
| //R68 - R71 | |
| vpaddq(xmm0, xmm4, Address(rax, 17 * 32), Assembler::AVX_256bit); | |
| vpermq(xmm8, xmm4, 0x1b, Assembler::AVX_256bit);//ymm8 = W[68] W[69] W[70] W[71] | |
| vpermq(xmm9, xmm3, 0x39, Assembler::AVX_256bit);//ymm9 = W[64] W[67] W[66] W[65] | |
| vpblendd(xmm7, xmm8, xmm9, 0x3f, Assembler::AVX_256bit);//ymm7 = W[68] W[67] W[66] W[65] | |
| vpaddq(xmm5, xmm5, xmm7, Assembler::AVX_256bit);//ymm5 = W[56..59] + S0(W[57..60]) + W[65..68] | |
| sha512msg2(xmm5, xmm4);//ymm5 += S1(W[66..69]) | |
| sha512rnds2(xmm12, xmm11, xmm0); | |
| vperm2i128(xmm0, xmm0, xmm0, 0x01); | |
| sha512rnds2(xmm11, xmm12, xmm0); | |
| //R72 - R75 | |
| vpaddq(xmm0, xmm5, Address(rax, 18 * 32), Assembler::AVX_256bit); | |
| vpermq(xmm8, xmm5, 0x1b, Assembler::AVX_256bit);//ymm8 = W[72] W[73] W[74] W[75] | |
| vpermq(xmm9, xmm4, 0x39, Assembler::AVX_256bit);//ymm9 = W[68] W[71] W[70] W[69] | |
| vpblendd(xmm7, xmm8, xmm9, 0x3f, Assembler::AVX_256bit);//ymm7 = W[72] W[71] W[70] W[69] | |
| vpaddq(xmm6, xmm6, xmm7, Assembler::AVX_256bit);//ymm6 = W[60..63] + S0(W[61..64]) + W[69..72] | |
| sha512msg2(xmm6, xmm5);//ymm6 += S1(W[70..73]) | |
| sha512rnds2(xmm12, xmm11, xmm0); | |
| vperm2i128(xmm0, xmm0, xmm0, 0x01); | |
| sha512rnds2(xmm11, xmm12, xmm0); | |
| //R76 - R79 | |
| vpaddq(xmm0, xmm6, Address(rax, 19 * 32), Assembler::AVX_256bit); | |
| sha512rnds2(xmm12, xmm11, xmm0); | |
| vperm2i128(xmm0, xmm0, xmm0, 0x01); | |
| sha512rnds2(xmm11, xmm12, xmm0); | |
| //update hash value | |
| vpaddq(xmm14, xmm14, xmm12, Assembler::AVX_256bit); | |
| vpaddq(xmm13, xmm13, xmm11, Assembler::AVX_256bit); | |
| if (multi_block) { | |
| addptr(arg_msg, 4 * 32); | |
| addptr(ofs, 128); | |
| cmpptr(ofs, limit); | |
| jcc(Assembler::belowEqual, block_loop); | |
| movptr(rax, ofs); //return ofs | |
| } | |
| //store the hash value back in memory | |
| //xmm13 = ABEF | |
| //xmm14 = CDGH | |
| vperm2i128(xmm1, xmm13, xmm14, 0x31); | |
| vperm2i128(xmm2, xmm13, xmm14, 0x20); | |
| vpermq(xmm1, xmm1, 0xb1, Assembler::AVX_256bit);//ymm1 = D C B A | |
| vpermq(xmm2, xmm2, 0xb1, Assembler::AVX_256bit);//ymm2 = H G F E | |
| vmovdqu(Address(arg_hash, 0 * 32), xmm1); | |
| vmovdqu(Address(arg_hash, 1 * 32), xmm2); | |
| bind(done_hash); | |
| } |