jochen
be92762969
The live tests reached one shared bus and assert, read and remove the mesh's own objects by their fixed names, so packages run in parallel deleted what each other read and the suite passed only one package at a time; a red suite read as noise. internal/testbus starts a server per test, linked in at the nats-server release go.mod pins, and a test holds that pin to the catalogue's bus image and to the facts snapshot's bus when there is one, so the tests never run a bus the mesh does not. The waiter test read a timing (the most connections held at one look) and now reads the state it means (the fewest held across the wait). make check runs the packages in parallel under the race detector, with a timeout.
20884 lines
523 KiB
ArmAsm
20884 lines
523 KiB
ArmAsm
// Code generated by command: go run gen.go -out ../encodeblock_amd64.s -stubs ../encodeblock_asm.go -pkg=s2. DO NOT EDIT.
|
|
|
|
//go:build (amd64 || arm64) && !appengine && !noasm && gc && !noasm
|
|
|
|
#include "textflag.h"
|
|
|
|
// func _dummy_()
|
|
TEXT ·_dummy_(SB), $0
|
|
#ifdef GOAMD64_v4
|
|
#ifndef GOAMD64_v3
|
|
#define GOAMD64_v3
|
|
#endif
|
|
#endif
|
|
RET
|
|
|
|
// func encodeBlockAsm(dst []byte, src []byte, tmp *[65536]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeBlockAsm(SB), $24-64
|
|
MOVQ tmp+48(FP), AX
|
|
MOVQ dst_base+0(FP), CX
|
|
MOVQ $0x00000200, DX
|
|
MOVQ AX, BX
|
|
PXOR X0, X0
|
|
|
|
zero_loop_encodeBlockAsm:
|
|
MOVOU X0, (BX)
|
|
MOVOU X0, 16(BX)
|
|
MOVOU X0, 32(BX)
|
|
MOVOU X0, 48(BX)
|
|
MOVOU X0, 64(BX)
|
|
MOVOU X0, 80(BX)
|
|
MOVOU X0, 96(BX)
|
|
MOVOU X0, 112(BX)
|
|
ADDQ $0x80, BX
|
|
DECQ DX
|
|
JNZ zero_loop_encodeBlockAsm
|
|
MOVL $0x00000000, 12(SP)
|
|
MOVQ src_len+32(FP), DX
|
|
LEAQ -9(DX), BX
|
|
LEAQ -8(DX), SI
|
|
MOVL SI, 8(SP)
|
|
SHRQ $0x05, DX
|
|
SUBL DX, BX
|
|
LEAQ (CX)(BX*1), BX
|
|
MOVQ BX, (SP)
|
|
MOVL $0x00000001, DX
|
|
MOVL DX, 16(SP)
|
|
MOVQ src_base+24(FP), BX
|
|
|
|
search_loop_encodeBlockAsm:
|
|
MOVL DX, SI
|
|
SUBL 12(SP), SI
|
|
SHRL $0x06, SI
|
|
LEAL 4(DX)(SI*1), SI
|
|
CMPL SI, 8(SP)
|
|
JAE emit_remainder_encodeBlockAsm
|
|
MOVQ (BX)(DX*1), DI
|
|
MOVL SI, 20(SP)
|
|
MOVQ $0x0000cf1bbcdcbf9b, R9
|
|
MOVQ DI, R10
|
|
MOVQ DI, R11
|
|
SHRQ $0x08, R11
|
|
SHLQ $0x10, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x32, R10
|
|
SHLQ $0x10, R11
|
|
IMULQ R9, R11
|
|
SHRQ $0x32, R11
|
|
MOVL (AX)(R10*4), SI
|
|
MOVL (AX)(R11*4), R8
|
|
MOVL DX, (AX)(R10*4)
|
|
LEAL 1(DX), R10
|
|
MOVL R10, (AX)(R11*4)
|
|
MOVQ DI, R10
|
|
SHRQ $0x10, R10
|
|
SHLQ $0x10, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x32, R10
|
|
MOVL DX, R9
|
|
SUBL 16(SP), R9
|
|
MOVL 1(BX)(R9*1), R11
|
|
MOVQ DI, R9
|
|
SHRQ $0x08, R9
|
|
CMPL R9, R11
|
|
JNE no_repeat_found_encodeBlockAsm
|
|
LEAL 1(DX), DI
|
|
MOVL 12(SP), R8
|
|
MOVL DI, SI
|
|
SUBL 16(SP), SI
|
|
JZ repeat_extend_back_end_encodeBlockAsm
|
|
|
|
repeat_extend_back_loop_encodeBlockAsm:
|
|
CMPL DI, R8
|
|
JBE repeat_extend_back_end_encodeBlockAsm
|
|
MOVB -1(BX)(SI*1), R9
|
|
MOVB -1(BX)(DI*1), R10
|
|
CMPB R9, R10
|
|
JNE repeat_extend_back_end_encodeBlockAsm
|
|
LEAL -1(DI), DI
|
|
DECL SI
|
|
JNZ repeat_extend_back_loop_encodeBlockAsm
|
|
|
|
repeat_extend_back_end_encodeBlockAsm:
|
|
MOVL DI, SI
|
|
SUBL 12(SP), SI
|
|
LEAQ 5(CX)(SI*1), SI
|
|
CMPQ SI, (SP)
|
|
JB repeat_dst_size_check_encodeBlockAsm
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
repeat_dst_size_check_encodeBlockAsm:
|
|
MOVL 12(SP), SI
|
|
CMPL SI, DI
|
|
JEQ emit_literal_done_repeat_emit_encodeBlockAsm
|
|
MOVL DI, R9
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(SI*1), R10
|
|
SUBL SI, R9
|
|
LEAL -1(R9), SI
|
|
CMPL SI, $0x3c
|
|
JB one_byte_repeat_emit_encodeBlockAsm
|
|
CMPL SI, $0x00000100
|
|
JB two_bytes_repeat_emit_encodeBlockAsm
|
|
CMPL SI, $0x00010000
|
|
JB three_bytes_repeat_emit_encodeBlockAsm
|
|
CMPL SI, $0x01000000
|
|
JB four_bytes_repeat_emit_encodeBlockAsm
|
|
MOVB $0xfc, (CX)
|
|
MOVL SI, 1(CX)
|
|
ADDQ $0x05, CX
|
|
JMP memmove_long_repeat_emit_encodeBlockAsm
|
|
|
|
four_bytes_repeat_emit_encodeBlockAsm:
|
|
MOVL SI, R11
|
|
SHRL $0x10, R11
|
|
MOVB $0xf8, (CX)
|
|
MOVW SI, 1(CX)
|
|
MOVB R11, 3(CX)
|
|
ADDQ $0x04, CX
|
|
JMP memmove_long_repeat_emit_encodeBlockAsm
|
|
|
|
three_bytes_repeat_emit_encodeBlockAsm:
|
|
MOVB $0xf4, (CX)
|
|
MOVW SI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_repeat_emit_encodeBlockAsm
|
|
|
|
two_bytes_repeat_emit_encodeBlockAsm:
|
|
MOVB $0xf0, (CX)
|
|
MOVB SI, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL SI, $0x40
|
|
JB memmove_repeat_emit_encodeBlockAsm
|
|
JMP memmove_long_repeat_emit_encodeBlockAsm
|
|
|
|
one_byte_repeat_emit_encodeBlockAsm:
|
|
SHLB $0x02, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_repeat_emit_encodeBlockAsm:
|
|
LEAQ (CX)(R9*1), SI
|
|
|
|
// genMemMoveShort
|
|
CMPQ R9, $0x08
|
|
JBE emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_8
|
|
CMPQ R9, $0x10
|
|
JBE emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_8through16
|
|
CMPQ R9, $0x20
|
|
JBE emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_17through32
|
|
JMP emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_33through64
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_8:
|
|
MOVQ (R10), R11
|
|
MOVQ R11, (CX)
|
|
JMP memmove_end_copy_repeat_emit_encodeBlockAsm
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_8through16:
|
|
MOVQ (R10), R11
|
|
MOVQ -8(R10)(R9*1), R10
|
|
MOVQ R11, (CX)
|
|
MOVQ R10, -8(CX)(R9*1)
|
|
JMP memmove_end_copy_repeat_emit_encodeBlockAsm
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_17through32:
|
|
MOVOU (R10), X0
|
|
MOVOU -16(R10)(R9*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(R9*1)
|
|
JMP memmove_end_copy_repeat_emit_encodeBlockAsm
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_33through64:
|
|
MOVOU (R10), X0
|
|
MOVOU 16(R10), X1
|
|
MOVOU -32(R10)(R9*1), X2
|
|
MOVOU -16(R10)(R9*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
|
|
memmove_end_copy_repeat_emit_encodeBlockAsm:
|
|
MOVQ SI, CX
|
|
JMP emit_literal_done_repeat_emit_encodeBlockAsm
|
|
|
|
memmove_long_repeat_emit_encodeBlockAsm:
|
|
LEAQ (CX)(R9*1), SI
|
|
|
|
// genMemMoveLong
|
|
MOVOU (R10), X0
|
|
MOVOU 16(R10), X1
|
|
MOVOU -32(R10)(R9*1), X2
|
|
MOVOU -16(R10)(R9*1), X3
|
|
MOVQ R9, R12
|
|
SHRQ $0x05, R12
|
|
MOVQ CX, R11
|
|
ANDL $0x0000001f, R11
|
|
MOVQ $0x00000040, R13
|
|
SUBQ R11, R13
|
|
DECQ R12
|
|
JA emit_lit_memmove_long_repeat_emit_encodeBlockAsmlarge_forward_sse_loop_32
|
|
LEAQ -32(R10)(R13*1), R11
|
|
LEAQ -32(CX)(R13*1), R14
|
|
|
|
emit_lit_memmove_long_repeat_emit_encodeBlockAsmlarge_big_loop_back:
|
|
MOVOU (R11), X4
|
|
MOVOU 16(R11), X5
|
|
MOVOA X4, (R14)
|
|
MOVOA X5, 16(R14)
|
|
ADDQ $0x20, R14
|
|
ADDQ $0x20, R11
|
|
ADDQ $0x20, R13
|
|
DECQ R12
|
|
JNA emit_lit_memmove_long_repeat_emit_encodeBlockAsmlarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_repeat_emit_encodeBlockAsmlarge_forward_sse_loop_32:
|
|
MOVOU -32(R10)(R13*1), X4
|
|
MOVOU -16(R10)(R13*1), X5
|
|
MOVOA X4, -32(CX)(R13*1)
|
|
MOVOA X5, -16(CX)(R13*1)
|
|
ADDQ $0x20, R13
|
|
CMPQ R9, R13
|
|
JAE emit_lit_memmove_long_repeat_emit_encodeBlockAsmlarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
MOVQ SI, CX
|
|
|
|
emit_literal_done_repeat_emit_encodeBlockAsm:
|
|
ADDL $0x05, DX
|
|
MOVL DX, SI
|
|
SUBL 16(SP), SI
|
|
MOVQ src_len+32(FP), R9
|
|
SUBL DX, R9
|
|
LEAQ (BX)(DX*1), R10
|
|
LEAQ (BX)(SI*1), SI
|
|
|
|
// matchLen
|
|
XORL R12, R12
|
|
|
|
matchlen_loopback_16_repeat_extend_encodeBlockAsm:
|
|
CMPL R9, $0x10
|
|
JB matchlen_match8_repeat_extend_encodeBlockAsm
|
|
MOVQ (R10)(R12*1), R11
|
|
MOVQ 8(R10)(R12*1), R13
|
|
XORQ (SI)(R12*1), R11
|
|
JNZ matchlen_bsf_8_repeat_extend_encodeBlockAsm
|
|
XORQ 8(SI)(R12*1), R13
|
|
JNZ matchlen_bsf_16repeat_extend_encodeBlockAsm
|
|
LEAL -16(R9), R9
|
|
LEAL 16(R12), R12
|
|
JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm
|
|
|
|
matchlen_bsf_16repeat_extend_encodeBlockAsm:
|
|
TZCNTQ R13, R13
|
|
SARQ $0x03, R13
|
|
LEAL 8(R12)(R13*1), R12
|
|
JMP repeat_extend_forward_end_encodeBlockAsm
|
|
|
|
matchlen_match8_repeat_extend_encodeBlockAsm:
|
|
CMPL R9, $0x08
|
|
JB matchlen_match4_repeat_extend_encodeBlockAsm
|
|
MOVQ (R10)(R12*1), R11
|
|
XORQ (SI)(R12*1), R11
|
|
JNZ matchlen_bsf_8_repeat_extend_encodeBlockAsm
|
|
LEAL -8(R9), R9
|
|
LEAL 8(R12), R12
|
|
JMP matchlen_match4_repeat_extend_encodeBlockAsm
|
|
|
|
matchlen_bsf_8_repeat_extend_encodeBlockAsm:
|
|
TZCNTQ R11, R11
|
|
SARQ $0x03, R11
|
|
LEAL (R12)(R11*1), R12
|
|
JMP repeat_extend_forward_end_encodeBlockAsm
|
|
|
|
matchlen_match4_repeat_extend_encodeBlockAsm:
|
|
CMPL R9, $0x04
|
|
JB matchlen_match2_repeat_extend_encodeBlockAsm
|
|
MOVL (R10)(R12*1), R11
|
|
CMPL (SI)(R12*1), R11
|
|
JNE matchlen_match2_repeat_extend_encodeBlockAsm
|
|
LEAL -4(R9), R9
|
|
LEAL 4(R12), R12
|
|
|
|
matchlen_match2_repeat_extend_encodeBlockAsm:
|
|
CMPL R9, $0x01
|
|
JE matchlen_match1_repeat_extend_encodeBlockAsm
|
|
JB repeat_extend_forward_end_encodeBlockAsm
|
|
MOVW (R10)(R12*1), R11
|
|
CMPW (SI)(R12*1), R11
|
|
JNE matchlen_match1_repeat_extend_encodeBlockAsm
|
|
LEAL 2(R12), R12
|
|
SUBL $0x02, R9
|
|
JZ repeat_extend_forward_end_encodeBlockAsm
|
|
|
|
matchlen_match1_repeat_extend_encodeBlockAsm:
|
|
MOVB (R10)(R12*1), R11
|
|
CMPB (SI)(R12*1), R11
|
|
JNE repeat_extend_forward_end_encodeBlockAsm
|
|
LEAL 1(R12), R12
|
|
|
|
repeat_extend_forward_end_encodeBlockAsm:
|
|
ADDL R12, DX
|
|
MOVL DX, SI
|
|
SUBL DI, SI
|
|
MOVL 16(SP), DI
|
|
TESTL R8, R8
|
|
JZ repeat_as_copy_encodeBlockAsm
|
|
|
|
// emitRepeat
|
|
emit_repeat_again_match_repeat_encodeBlockAsm:
|
|
MOVL SI, R8
|
|
LEAL -4(SI), SI
|
|
CMPL R8, $0x08
|
|
JBE repeat_two_match_repeat_encodeBlockAsm
|
|
CMPL R8, $0x0c
|
|
JAE cant_repeat_two_offset_match_repeat_encodeBlockAsm
|
|
CMPL DI, $0x00000800
|
|
JB repeat_two_offset_match_repeat_encodeBlockAsm
|
|
|
|
cant_repeat_two_offset_match_repeat_encodeBlockAsm:
|
|
CMPL SI, $0x00000104
|
|
JB repeat_three_match_repeat_encodeBlockAsm
|
|
CMPL SI, $0x00010100
|
|
JB repeat_four_match_repeat_encodeBlockAsm
|
|
CMPL SI, $0x0100ffff
|
|
JB repeat_five_match_repeat_encodeBlockAsm
|
|
LEAL -16842747(SI), SI
|
|
MOVL $0xfffb001d, (CX)
|
|
MOVB $0xff, 4(CX)
|
|
ADDQ $0x05, CX
|
|
JMP emit_repeat_again_match_repeat_encodeBlockAsm
|
|
|
|
repeat_five_match_repeat_encodeBlockAsm:
|
|
LEAL -65536(SI), SI
|
|
MOVL SI, DI
|
|
MOVW $0x001d, (CX)
|
|
MOVW SI, 2(CX)
|
|
SARL $0x10, DI
|
|
MOVB DI, 4(CX)
|
|
ADDQ $0x05, CX
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
repeat_four_match_repeat_encodeBlockAsm:
|
|
LEAL -256(SI), SI
|
|
MOVW $0x0019, (CX)
|
|
MOVW SI, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
repeat_three_match_repeat_encodeBlockAsm:
|
|
LEAL -4(SI), SI
|
|
MOVW $0x0015, (CX)
|
|
MOVB SI, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
repeat_two_match_repeat_encodeBlockAsm:
|
|
SHLL $0x02, SI
|
|
ORL $0x01, SI
|
|
MOVW SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
repeat_two_offset_match_repeat_encodeBlockAsm:
|
|
XORQ R8, R8
|
|
LEAL 1(R8)(SI*4), SI
|
|
MOVB DI, 1(CX)
|
|
SARL $0x08, DI
|
|
SHLL $0x05, DI
|
|
ORL DI, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
repeat_as_copy_encodeBlockAsm:
|
|
// emitCopy
|
|
CMPL DI, $0x00010000
|
|
JB two_byte_offset_repeat_as_copy_encodeBlockAsm
|
|
CMPL SI, $0x40
|
|
JBE four_bytes_remain_repeat_as_copy_encodeBlockAsm
|
|
MOVB $0xff, (CX)
|
|
MOVL DI, 1(CX)
|
|
LEAL -64(SI), SI
|
|
ADDQ $0x05, CX
|
|
CMPL SI, $0x04
|
|
JB four_bytes_remain_repeat_as_copy_encodeBlockAsm
|
|
|
|
// emitRepeat
|
|
emit_repeat_again_repeat_as_copy_encodeBlockAsm_emit_copy:
|
|
MOVL SI, R8
|
|
LEAL -4(SI), SI
|
|
CMPL R8, $0x08
|
|
JBE repeat_two_repeat_as_copy_encodeBlockAsm_emit_copy
|
|
CMPL R8, $0x0c
|
|
JAE cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy
|
|
CMPL DI, $0x00000800
|
|
JB repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy
|
|
|
|
cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy:
|
|
CMPL SI, $0x00000104
|
|
JB repeat_three_repeat_as_copy_encodeBlockAsm_emit_copy
|
|
CMPL SI, $0x00010100
|
|
JB repeat_four_repeat_as_copy_encodeBlockAsm_emit_copy
|
|
CMPL SI, $0x0100ffff
|
|
JB repeat_five_repeat_as_copy_encodeBlockAsm_emit_copy
|
|
LEAL -16842747(SI), SI
|
|
MOVL $0xfffb001d, (CX)
|
|
MOVB $0xff, 4(CX)
|
|
ADDQ $0x05, CX
|
|
JMP emit_repeat_again_repeat_as_copy_encodeBlockAsm_emit_copy
|
|
|
|
repeat_five_repeat_as_copy_encodeBlockAsm_emit_copy:
|
|
LEAL -65536(SI), SI
|
|
MOVL SI, DI
|
|
MOVW $0x001d, (CX)
|
|
MOVW SI, 2(CX)
|
|
SARL $0x10, DI
|
|
MOVB DI, 4(CX)
|
|
ADDQ $0x05, CX
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
repeat_four_repeat_as_copy_encodeBlockAsm_emit_copy:
|
|
LEAL -256(SI), SI
|
|
MOVW $0x0019, (CX)
|
|
MOVW SI, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
repeat_three_repeat_as_copy_encodeBlockAsm_emit_copy:
|
|
LEAL -4(SI), SI
|
|
MOVW $0x0015, (CX)
|
|
MOVB SI, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
repeat_two_repeat_as_copy_encodeBlockAsm_emit_copy:
|
|
SHLL $0x02, SI
|
|
ORL $0x01, SI
|
|
MOVW SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy:
|
|
XORQ R8, R8
|
|
LEAL 1(R8)(SI*4), SI
|
|
MOVB DI, 1(CX)
|
|
SARL $0x08, DI
|
|
SHLL $0x05, DI
|
|
ORL DI, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
four_bytes_remain_repeat_as_copy_encodeBlockAsm:
|
|
TESTL SI, SI
|
|
JZ repeat_end_emit_encodeBlockAsm
|
|
XORL R8, R8
|
|
LEAL -1(R8)(SI*4), SI
|
|
MOVB SI, (CX)
|
|
MOVL DI, 1(CX)
|
|
ADDQ $0x05, CX
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
two_byte_offset_repeat_as_copy_encodeBlockAsm:
|
|
CMPL SI, $0x40
|
|
JBE two_byte_offset_short_repeat_as_copy_encodeBlockAsm
|
|
CMPL DI, $0x00000800
|
|
JAE long_offset_short_repeat_as_copy_encodeBlockAsm
|
|
MOVL $0x00000001, R8
|
|
LEAL 16(R8), R8
|
|
MOVB DI, 1(CX)
|
|
MOVL DI, R9
|
|
SHRL $0x08, R9
|
|
SHLL $0x05, R9
|
|
ORL R9, R8
|
|
MOVB R8, (CX)
|
|
ADDQ $0x02, CX
|
|
SUBL $0x08, SI
|
|
|
|
// emitRepeat
|
|
LEAL -4(SI), SI
|
|
JMP cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b
|
|
|
|
emit_repeat_again_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b:
|
|
MOVL SI, R8
|
|
LEAL -4(SI), SI
|
|
CMPL R8, $0x08
|
|
JBE repeat_two_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b
|
|
CMPL R8, $0x0c
|
|
JAE cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b
|
|
CMPL DI, $0x00000800
|
|
JB repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b
|
|
|
|
cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b:
|
|
CMPL SI, $0x00000104
|
|
JB repeat_three_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b
|
|
CMPL SI, $0x00010100
|
|
JB repeat_four_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b
|
|
CMPL SI, $0x0100ffff
|
|
JB repeat_five_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b
|
|
LEAL -16842747(SI), SI
|
|
MOVL $0xfffb001d, (CX)
|
|
MOVB $0xff, 4(CX)
|
|
ADDQ $0x05, CX
|
|
JMP emit_repeat_again_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b
|
|
|
|
repeat_five_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b:
|
|
LEAL -65536(SI), SI
|
|
MOVL SI, DI
|
|
MOVW $0x001d, (CX)
|
|
MOVW SI, 2(CX)
|
|
SARL $0x10, DI
|
|
MOVB DI, 4(CX)
|
|
ADDQ $0x05, CX
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
repeat_four_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b:
|
|
LEAL -256(SI), SI
|
|
MOVW $0x0019, (CX)
|
|
MOVW SI, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
repeat_three_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b:
|
|
LEAL -4(SI), SI
|
|
MOVW $0x0015, (CX)
|
|
MOVB SI, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
repeat_two_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b:
|
|
SHLL $0x02, SI
|
|
ORL $0x01, SI
|
|
MOVW SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b:
|
|
XORQ R8, R8
|
|
LEAL 1(R8)(SI*4), SI
|
|
MOVB DI, 1(CX)
|
|
SARL $0x08, DI
|
|
SHLL $0x05, DI
|
|
ORL DI, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
long_offset_short_repeat_as_copy_encodeBlockAsm:
|
|
MOVB $0xee, (CX)
|
|
MOVW DI, 1(CX)
|
|
LEAL -60(SI), SI
|
|
ADDQ $0x03, CX
|
|
|
|
// emitRepeat
|
|
emit_repeat_again_repeat_as_copy_encodeBlockAsm_emit_copy_short:
|
|
MOVL SI, R8
|
|
LEAL -4(SI), SI
|
|
CMPL R8, $0x08
|
|
JBE repeat_two_repeat_as_copy_encodeBlockAsm_emit_copy_short
|
|
CMPL R8, $0x0c
|
|
JAE cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short
|
|
CMPL DI, $0x00000800
|
|
JB repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short
|
|
|
|
cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short:
|
|
CMPL SI, $0x00000104
|
|
JB repeat_three_repeat_as_copy_encodeBlockAsm_emit_copy_short
|
|
CMPL SI, $0x00010100
|
|
JB repeat_four_repeat_as_copy_encodeBlockAsm_emit_copy_short
|
|
CMPL SI, $0x0100ffff
|
|
JB repeat_five_repeat_as_copy_encodeBlockAsm_emit_copy_short
|
|
LEAL -16842747(SI), SI
|
|
MOVL $0xfffb001d, (CX)
|
|
MOVB $0xff, 4(CX)
|
|
ADDQ $0x05, CX
|
|
JMP emit_repeat_again_repeat_as_copy_encodeBlockAsm_emit_copy_short
|
|
|
|
repeat_five_repeat_as_copy_encodeBlockAsm_emit_copy_short:
|
|
LEAL -65536(SI), SI
|
|
MOVL SI, DI
|
|
MOVW $0x001d, (CX)
|
|
MOVW SI, 2(CX)
|
|
SARL $0x10, DI
|
|
MOVB DI, 4(CX)
|
|
ADDQ $0x05, CX
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
repeat_four_repeat_as_copy_encodeBlockAsm_emit_copy_short:
|
|
LEAL -256(SI), SI
|
|
MOVW $0x0019, (CX)
|
|
MOVW SI, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
repeat_three_repeat_as_copy_encodeBlockAsm_emit_copy_short:
|
|
LEAL -4(SI), SI
|
|
MOVW $0x0015, (CX)
|
|
MOVB SI, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
repeat_two_repeat_as_copy_encodeBlockAsm_emit_copy_short:
|
|
SHLL $0x02, SI
|
|
ORL $0x01, SI
|
|
MOVW SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short:
|
|
XORQ R8, R8
|
|
LEAL 1(R8)(SI*4), SI
|
|
MOVB DI, 1(CX)
|
|
SARL $0x08, DI
|
|
SHLL $0x05, DI
|
|
ORL DI, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
two_byte_offset_short_repeat_as_copy_encodeBlockAsm:
|
|
MOVL SI, R8
|
|
SHLL $0x02, R8
|
|
CMPL SI, $0x0c
|
|
JAE emit_copy_three_repeat_as_copy_encodeBlockAsm
|
|
CMPL DI, $0x00000800
|
|
JAE emit_copy_three_repeat_as_copy_encodeBlockAsm
|
|
LEAL -15(R8), R8
|
|
MOVB DI, 1(CX)
|
|
SHRL $0x08, DI
|
|
SHLL $0x05, DI
|
|
ORL DI, R8
|
|
MOVB R8, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
emit_copy_three_repeat_as_copy_encodeBlockAsm:
|
|
LEAL -2(R8), R8
|
|
MOVB R8, (CX)
|
|
MOVW DI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
|
|
repeat_end_emit_encodeBlockAsm:
|
|
MOVL DX, 12(SP)
|
|
JMP search_loop_encodeBlockAsm
|
|
|
|
no_repeat_found_encodeBlockAsm:
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ candidate_match_encodeBlockAsm
|
|
SHRQ $0x08, DI
|
|
MOVL (AX)(R10*4), SI
|
|
LEAL 2(DX), R9
|
|
CMPL (BX)(R8*1), DI
|
|
JEQ candidate2_match_encodeBlockAsm
|
|
MOVL R9, (AX)(R10*4)
|
|
SHRQ $0x08, DI
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ candidate3_match_encodeBlockAsm
|
|
MOVL 20(SP), DX
|
|
JMP search_loop_encodeBlockAsm
|
|
|
|
candidate3_match_encodeBlockAsm:
|
|
ADDL $0x02, DX
|
|
JMP candidate_match_encodeBlockAsm
|
|
|
|
candidate2_match_encodeBlockAsm:
|
|
MOVL R9, (AX)(R10*4)
|
|
INCL DX
|
|
MOVL R8, SI
|
|
|
|
candidate_match_encodeBlockAsm:
|
|
MOVL 12(SP), DI
|
|
TESTL SI, SI
|
|
JZ match_extend_back_end_encodeBlockAsm
|
|
|
|
match_extend_back_loop_encodeBlockAsm:
|
|
CMPL DX, DI
|
|
JBE match_extend_back_end_encodeBlockAsm
|
|
MOVB -1(BX)(SI*1), R8
|
|
MOVB -1(BX)(DX*1), R9
|
|
CMPB R8, R9
|
|
JNE match_extend_back_end_encodeBlockAsm
|
|
LEAL -1(DX), DX
|
|
DECL SI
|
|
JZ match_extend_back_end_encodeBlockAsm
|
|
JMP match_extend_back_loop_encodeBlockAsm
|
|
|
|
match_extend_back_end_encodeBlockAsm:
|
|
MOVL DX, DI
|
|
SUBL 12(SP), DI
|
|
LEAQ 5(CX)(DI*1), DI
|
|
CMPQ DI, (SP)
|
|
JB match_dst_size_check_encodeBlockAsm
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeBlockAsm:
|
|
MOVL DX, DI
|
|
MOVL 12(SP), R8
|
|
CMPL R8, DI
|
|
JEQ emit_literal_done_match_emit_encodeBlockAsm
|
|
MOVL DI, R9
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(R8*1), DI
|
|
SUBL R8, R9
|
|
LEAL -1(R9), R8
|
|
CMPL R8, $0x3c
|
|
JB one_byte_match_emit_encodeBlockAsm
|
|
CMPL R8, $0x00000100
|
|
JB two_bytes_match_emit_encodeBlockAsm
|
|
CMPL R8, $0x00010000
|
|
JB three_bytes_match_emit_encodeBlockAsm
|
|
CMPL R8, $0x01000000
|
|
JB four_bytes_match_emit_encodeBlockAsm
|
|
MOVB $0xfc, (CX)
|
|
MOVL R8, 1(CX)
|
|
ADDQ $0x05, CX
|
|
JMP memmove_long_match_emit_encodeBlockAsm
|
|
|
|
four_bytes_match_emit_encodeBlockAsm:
|
|
MOVL R8, R10
|
|
SHRL $0x10, R10
|
|
MOVB $0xf8, (CX)
|
|
MOVW R8, 1(CX)
|
|
MOVB R10, 3(CX)
|
|
ADDQ $0x04, CX
|
|
JMP memmove_long_match_emit_encodeBlockAsm
|
|
|
|
three_bytes_match_emit_encodeBlockAsm:
|
|
MOVB $0xf4, (CX)
|
|
MOVW R8, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_match_emit_encodeBlockAsm
|
|
|
|
two_bytes_match_emit_encodeBlockAsm:
|
|
MOVB $0xf0, (CX)
|
|
MOVB R8, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL R8, $0x40
|
|
JB memmove_match_emit_encodeBlockAsm
|
|
JMP memmove_long_match_emit_encodeBlockAsm
|
|
|
|
one_byte_match_emit_encodeBlockAsm:
|
|
SHLB $0x02, R8
|
|
MOVB R8, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_match_emit_encodeBlockAsm:
|
|
LEAQ (CX)(R9*1), R8
|
|
|
|
// genMemMoveShort
|
|
CMPQ R9, $0x08
|
|
JBE emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_8
|
|
CMPQ R9, $0x10
|
|
JBE emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_8through16
|
|
CMPQ R9, $0x20
|
|
JBE emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_8:
|
|
MOVQ (DI), R10
|
|
MOVQ R10, (CX)
|
|
JMP memmove_end_copy_match_emit_encodeBlockAsm
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_8through16:
|
|
MOVQ (DI), R10
|
|
MOVQ -8(DI)(R9*1), DI
|
|
MOVQ R10, (CX)
|
|
MOVQ DI, -8(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeBlockAsm
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_17through32:
|
|
MOVOU (DI), X0
|
|
MOVOU -16(DI)(R9*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeBlockAsm
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_33through64:
|
|
MOVOU (DI), X0
|
|
MOVOU 16(DI), X1
|
|
MOVOU -32(DI)(R9*1), X2
|
|
MOVOU -16(DI)(R9*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
|
|
memmove_end_copy_match_emit_encodeBlockAsm:
|
|
MOVQ R8, CX
|
|
JMP emit_literal_done_match_emit_encodeBlockAsm
|
|
|
|
memmove_long_match_emit_encodeBlockAsm:
|
|
LEAQ (CX)(R9*1), R8
|
|
|
|
// genMemMoveLong
|
|
MOVOU (DI), X0
|
|
MOVOU 16(DI), X1
|
|
MOVOU -32(DI)(R9*1), X2
|
|
MOVOU -16(DI)(R9*1), X3
|
|
MOVQ R9, R11
|
|
SHRQ $0x05, R11
|
|
MOVQ CX, R10
|
|
ANDL $0x0000001f, R10
|
|
MOVQ $0x00000040, R12
|
|
SUBQ R10, R12
|
|
DECQ R11
|
|
JA emit_lit_memmove_long_match_emit_encodeBlockAsmlarge_forward_sse_loop_32
|
|
LEAQ -32(DI)(R12*1), R10
|
|
LEAQ -32(CX)(R12*1), R13
|
|
|
|
emit_lit_memmove_long_match_emit_encodeBlockAsmlarge_big_loop_back:
|
|
MOVOU (R10), X4
|
|
MOVOU 16(R10), X5
|
|
MOVOA X4, (R13)
|
|
MOVOA X5, 16(R13)
|
|
ADDQ $0x20, R13
|
|
ADDQ $0x20, R10
|
|
ADDQ $0x20, R12
|
|
DECQ R11
|
|
JNA emit_lit_memmove_long_match_emit_encodeBlockAsmlarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_match_emit_encodeBlockAsmlarge_forward_sse_loop_32:
|
|
MOVOU -32(DI)(R12*1), X4
|
|
MOVOU -16(DI)(R12*1), X5
|
|
MOVOA X4, -32(CX)(R12*1)
|
|
MOVOA X5, -16(CX)(R12*1)
|
|
ADDQ $0x20, R12
|
|
CMPQ R9, R12
|
|
JAE emit_lit_memmove_long_match_emit_encodeBlockAsmlarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
MOVQ R8, CX
|
|
|
|
emit_literal_done_match_emit_encodeBlockAsm:
|
|
match_nolit_loop_encodeBlockAsm:
|
|
MOVL DX, DI
|
|
SUBL SI, DI
|
|
MOVL DI, 16(SP)
|
|
ADDL $0x04, DX
|
|
ADDL $0x04, SI
|
|
MOVQ src_len+32(FP), DI
|
|
SUBL DX, DI
|
|
LEAQ (BX)(DX*1), R8
|
|
LEAQ (BX)(SI*1), SI
|
|
|
|
// matchLen
|
|
XORL R10, R10
|
|
|
|
matchlen_loopback_16_match_nolit_encodeBlockAsm:
|
|
CMPL DI, $0x10
|
|
JB matchlen_match8_match_nolit_encodeBlockAsm
|
|
MOVQ (R8)(R10*1), R9
|
|
MOVQ 8(R8)(R10*1), R11
|
|
XORQ (SI)(R10*1), R9
|
|
JNZ matchlen_bsf_8_match_nolit_encodeBlockAsm
|
|
XORQ 8(SI)(R10*1), R11
|
|
JNZ matchlen_bsf_16match_nolit_encodeBlockAsm
|
|
LEAL -16(DI), DI
|
|
LEAL 16(R10), R10
|
|
JMP matchlen_loopback_16_match_nolit_encodeBlockAsm
|
|
|
|
matchlen_bsf_16match_nolit_encodeBlockAsm:
|
|
TZCNTQ R11, R11
|
|
SARQ $0x03, R11
|
|
LEAL 8(R10)(R11*1), R10
|
|
JMP match_nolit_end_encodeBlockAsm
|
|
|
|
matchlen_match8_match_nolit_encodeBlockAsm:
|
|
CMPL DI, $0x08
|
|
JB matchlen_match4_match_nolit_encodeBlockAsm
|
|
MOVQ (R8)(R10*1), R9
|
|
XORQ (SI)(R10*1), R9
|
|
JNZ matchlen_bsf_8_match_nolit_encodeBlockAsm
|
|
LEAL -8(DI), DI
|
|
LEAL 8(R10), R10
|
|
JMP matchlen_match4_match_nolit_encodeBlockAsm
|
|
|
|
matchlen_bsf_8_match_nolit_encodeBlockAsm:
|
|
TZCNTQ R9, R9
|
|
SARQ $0x03, R9
|
|
LEAL (R10)(R9*1), R10
|
|
JMP match_nolit_end_encodeBlockAsm
|
|
|
|
matchlen_match4_match_nolit_encodeBlockAsm:
|
|
CMPL DI, $0x04
|
|
JB matchlen_match2_match_nolit_encodeBlockAsm
|
|
MOVL (R8)(R10*1), R9
|
|
CMPL (SI)(R10*1), R9
|
|
JNE matchlen_match2_match_nolit_encodeBlockAsm
|
|
LEAL -4(DI), DI
|
|
LEAL 4(R10), R10
|
|
|
|
matchlen_match2_match_nolit_encodeBlockAsm:
|
|
CMPL DI, $0x01
|
|
JE matchlen_match1_match_nolit_encodeBlockAsm
|
|
JB match_nolit_end_encodeBlockAsm
|
|
MOVW (R8)(R10*1), R9
|
|
CMPW (SI)(R10*1), R9
|
|
JNE matchlen_match1_match_nolit_encodeBlockAsm
|
|
LEAL 2(R10), R10
|
|
SUBL $0x02, DI
|
|
JZ match_nolit_end_encodeBlockAsm
|
|
|
|
matchlen_match1_match_nolit_encodeBlockAsm:
|
|
MOVB (R8)(R10*1), R9
|
|
CMPB (SI)(R10*1), R9
|
|
JNE match_nolit_end_encodeBlockAsm
|
|
LEAL 1(R10), R10
|
|
|
|
match_nolit_end_encodeBlockAsm:
|
|
ADDL R10, DX
|
|
MOVL 16(SP), SI
|
|
ADDL $0x04, R10
|
|
MOVL DX, 12(SP)
|
|
|
|
// emitCopy
|
|
CMPL SI, $0x00010000
|
|
JB two_byte_offset_match_nolit_encodeBlockAsm
|
|
CMPL R10, $0x40
|
|
JBE four_bytes_remain_match_nolit_encodeBlockAsm
|
|
MOVB $0xff, (CX)
|
|
MOVL SI, 1(CX)
|
|
LEAL -64(R10), R10
|
|
ADDQ $0x05, CX
|
|
CMPL R10, $0x04
|
|
JB four_bytes_remain_match_nolit_encodeBlockAsm
|
|
|
|
// emitRepeat
|
|
emit_repeat_again_match_nolit_encodeBlockAsm_emit_copy:
|
|
MOVL R10, DI
|
|
LEAL -4(R10), R10
|
|
CMPL DI, $0x08
|
|
JBE repeat_two_match_nolit_encodeBlockAsm_emit_copy
|
|
CMPL DI, $0x0c
|
|
JAE cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy
|
|
CMPL SI, $0x00000800
|
|
JB repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy:
|
|
CMPL R10, $0x00000104
|
|
JB repeat_three_match_nolit_encodeBlockAsm_emit_copy
|
|
CMPL R10, $0x00010100
|
|
JB repeat_four_match_nolit_encodeBlockAsm_emit_copy
|
|
CMPL R10, $0x0100ffff
|
|
JB repeat_five_match_nolit_encodeBlockAsm_emit_copy
|
|
LEAL -16842747(R10), R10
|
|
MOVL $0xfffb001d, (CX)
|
|
MOVB $0xff, 4(CX)
|
|
ADDQ $0x05, CX
|
|
JMP emit_repeat_again_match_nolit_encodeBlockAsm_emit_copy
|
|
|
|
repeat_five_match_nolit_encodeBlockAsm_emit_copy:
|
|
LEAL -65536(R10), R10
|
|
MOVL R10, SI
|
|
MOVW $0x001d, (CX)
|
|
MOVW R10, 2(CX)
|
|
SARL $0x10, SI
|
|
MOVB SI, 4(CX)
|
|
ADDQ $0x05, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm
|
|
|
|
repeat_four_match_nolit_encodeBlockAsm_emit_copy:
|
|
LEAL -256(R10), R10
|
|
MOVW $0x0019, (CX)
|
|
MOVW R10, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm
|
|
|
|
repeat_three_match_nolit_encodeBlockAsm_emit_copy:
|
|
LEAL -4(R10), R10
|
|
MOVW $0x0015, (CX)
|
|
MOVB R10, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm
|
|
|
|
repeat_two_match_nolit_encodeBlockAsm_emit_copy:
|
|
SHLL $0x02, R10
|
|
ORL $0x01, R10
|
|
MOVW R10, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm
|
|
|
|
repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy:
|
|
XORQ DI, DI
|
|
LEAL 1(DI)(R10*4), R10
|
|
MOVB SI, 1(CX)
|
|
SARL $0x08, SI
|
|
SHLL $0x05, SI
|
|
ORL SI, R10
|
|
MOVB R10, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm
|
|
|
|
four_bytes_remain_match_nolit_encodeBlockAsm:
|
|
TESTL R10, R10
|
|
JZ match_nolit_emitcopy_end_encodeBlockAsm
|
|
XORL DI, DI
|
|
LEAL -1(DI)(R10*4), R10
|
|
MOVB R10, (CX)
|
|
MOVL SI, 1(CX)
|
|
ADDQ $0x05, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm
|
|
|
|
two_byte_offset_match_nolit_encodeBlockAsm:
|
|
CMPL R10, $0x40
|
|
JBE two_byte_offset_short_match_nolit_encodeBlockAsm
|
|
CMPL SI, $0x00000800
|
|
JAE long_offset_short_match_nolit_encodeBlockAsm
|
|
MOVL $0x00000001, DI
|
|
LEAL 16(DI), DI
|
|
MOVB SI, 1(CX)
|
|
MOVL SI, R8
|
|
SHRL $0x08, R8
|
|
SHLL $0x05, R8
|
|
ORL R8, DI
|
|
MOVB DI, (CX)
|
|
ADDQ $0x02, CX
|
|
SUBL $0x08, R10
|
|
|
|
// emitRepeat
|
|
LEAL -4(R10), R10
|
|
JMP cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short_2b
|
|
|
|
emit_repeat_again_match_nolit_encodeBlockAsm_emit_copy_short_2b:
|
|
MOVL R10, DI
|
|
LEAL -4(R10), R10
|
|
CMPL DI, $0x08
|
|
JBE repeat_two_match_nolit_encodeBlockAsm_emit_copy_short_2b
|
|
CMPL DI, $0x0c
|
|
JAE cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short_2b
|
|
CMPL SI, $0x00000800
|
|
JB repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short_2b
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short_2b:
|
|
CMPL R10, $0x00000104
|
|
JB repeat_three_match_nolit_encodeBlockAsm_emit_copy_short_2b
|
|
CMPL R10, $0x00010100
|
|
JB repeat_four_match_nolit_encodeBlockAsm_emit_copy_short_2b
|
|
CMPL R10, $0x0100ffff
|
|
JB repeat_five_match_nolit_encodeBlockAsm_emit_copy_short_2b
|
|
LEAL -16842747(R10), R10
|
|
MOVL $0xfffb001d, (CX)
|
|
MOVB $0xff, 4(CX)
|
|
ADDQ $0x05, CX
|
|
JMP emit_repeat_again_match_nolit_encodeBlockAsm_emit_copy_short_2b
|
|
|
|
repeat_five_match_nolit_encodeBlockAsm_emit_copy_short_2b:
|
|
LEAL -65536(R10), R10
|
|
MOVL R10, SI
|
|
MOVW $0x001d, (CX)
|
|
MOVW R10, 2(CX)
|
|
SARL $0x10, SI
|
|
MOVB SI, 4(CX)
|
|
ADDQ $0x05, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm
|
|
|
|
repeat_four_match_nolit_encodeBlockAsm_emit_copy_short_2b:
|
|
LEAL -256(R10), R10
|
|
MOVW $0x0019, (CX)
|
|
MOVW R10, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm
|
|
|
|
repeat_three_match_nolit_encodeBlockAsm_emit_copy_short_2b:
|
|
LEAL -4(R10), R10
|
|
MOVW $0x0015, (CX)
|
|
MOVB R10, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm
|
|
|
|
repeat_two_match_nolit_encodeBlockAsm_emit_copy_short_2b:
|
|
SHLL $0x02, R10
|
|
ORL $0x01, R10
|
|
MOVW R10, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm
|
|
|
|
repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short_2b:
|
|
XORQ DI, DI
|
|
LEAL 1(DI)(R10*4), R10
|
|
MOVB SI, 1(CX)
|
|
SARL $0x08, SI
|
|
SHLL $0x05, SI
|
|
ORL SI, R10
|
|
MOVB R10, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm
|
|
|
|
long_offset_short_match_nolit_encodeBlockAsm:
|
|
MOVB $0xee, (CX)
|
|
MOVW SI, 1(CX)
|
|
LEAL -60(R10), R10
|
|
ADDQ $0x03, CX
|
|
|
|
// emitRepeat
|
|
emit_repeat_again_match_nolit_encodeBlockAsm_emit_copy_short:
|
|
MOVL R10, DI
|
|
LEAL -4(R10), R10
|
|
CMPL DI, $0x08
|
|
JBE repeat_two_match_nolit_encodeBlockAsm_emit_copy_short
|
|
CMPL DI, $0x0c
|
|
JAE cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short
|
|
CMPL SI, $0x00000800
|
|
JB repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short:
|
|
CMPL R10, $0x00000104
|
|
JB repeat_three_match_nolit_encodeBlockAsm_emit_copy_short
|
|
CMPL R10, $0x00010100
|
|
JB repeat_four_match_nolit_encodeBlockAsm_emit_copy_short
|
|
CMPL R10, $0x0100ffff
|
|
JB repeat_five_match_nolit_encodeBlockAsm_emit_copy_short
|
|
LEAL -16842747(R10), R10
|
|
MOVL $0xfffb001d, (CX)
|
|
MOVB $0xff, 4(CX)
|
|
ADDQ $0x05, CX
|
|
JMP emit_repeat_again_match_nolit_encodeBlockAsm_emit_copy_short
|
|
|
|
repeat_five_match_nolit_encodeBlockAsm_emit_copy_short:
|
|
LEAL -65536(R10), R10
|
|
MOVL R10, SI
|
|
MOVW $0x001d, (CX)
|
|
MOVW R10, 2(CX)
|
|
SARL $0x10, SI
|
|
MOVB SI, 4(CX)
|
|
ADDQ $0x05, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm
|
|
|
|
repeat_four_match_nolit_encodeBlockAsm_emit_copy_short:
|
|
LEAL -256(R10), R10
|
|
MOVW $0x0019, (CX)
|
|
MOVW R10, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm
|
|
|
|
repeat_three_match_nolit_encodeBlockAsm_emit_copy_short:
|
|
LEAL -4(R10), R10
|
|
MOVW $0x0015, (CX)
|
|
MOVB R10, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm
|
|
|
|
repeat_two_match_nolit_encodeBlockAsm_emit_copy_short:
|
|
SHLL $0x02, R10
|
|
ORL $0x01, R10
|
|
MOVW R10, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm
|
|
|
|
repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short:
|
|
XORQ DI, DI
|
|
LEAL 1(DI)(R10*4), R10
|
|
MOVB SI, 1(CX)
|
|
SARL $0x08, SI
|
|
SHLL $0x05, SI
|
|
ORL SI, R10
|
|
MOVB R10, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm
|
|
|
|
two_byte_offset_short_match_nolit_encodeBlockAsm:
|
|
MOVL R10, DI
|
|
SHLL $0x02, DI
|
|
CMPL R10, $0x0c
|
|
JAE emit_copy_three_match_nolit_encodeBlockAsm
|
|
CMPL SI, $0x00000800
|
|
JAE emit_copy_three_match_nolit_encodeBlockAsm
|
|
LEAL -15(DI), DI
|
|
MOVB SI, 1(CX)
|
|
SHRL $0x08, SI
|
|
SHLL $0x05, SI
|
|
ORL SI, DI
|
|
MOVB DI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm
|
|
|
|
emit_copy_three_match_nolit_encodeBlockAsm:
|
|
LEAL -2(DI), DI
|
|
MOVB DI, (CX)
|
|
MOVW SI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
|
|
match_nolit_emitcopy_end_encodeBlockAsm:
|
|
CMPL DX, 8(SP)
|
|
JAE emit_remainder_encodeBlockAsm
|
|
MOVQ -2(BX)(DX*1), DI
|
|
CMPQ CX, (SP)
|
|
JB match_nolit_dst_ok_encodeBlockAsm
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeBlockAsm:
|
|
MOVQ $0x0000cf1bbcdcbf9b, R9
|
|
MOVQ DI, R8
|
|
SHRQ $0x10, DI
|
|
MOVQ DI, SI
|
|
SHLQ $0x10, R8
|
|
IMULQ R9, R8
|
|
SHRQ $0x32, R8
|
|
SHLQ $0x10, SI
|
|
IMULQ R9, SI
|
|
SHRQ $0x32, SI
|
|
LEAL -2(DX), R9
|
|
LEAQ (AX)(SI*4), R10
|
|
MOVL (R10), SI
|
|
MOVL R9, (AX)(R8*4)
|
|
MOVL DX, (R10)
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ match_nolit_loop_encodeBlockAsm
|
|
INCL DX
|
|
JMP search_loop_encodeBlockAsm
|
|
|
|
emit_remainder_encodeBlockAsm:
|
|
MOVQ src_len+32(FP), AX
|
|
SUBL 12(SP), AX
|
|
LEAQ 5(CX)(AX*1), AX
|
|
CMPQ AX, (SP)
|
|
JB emit_remainder_ok_encodeBlockAsm
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeBlockAsm:
|
|
MOVQ src_len+32(FP), AX
|
|
MOVL 12(SP), DX
|
|
CMPL DX, AX
|
|
JEQ emit_literal_done_emit_remainder_encodeBlockAsm
|
|
MOVL AX, SI
|
|
MOVL AX, 12(SP)
|
|
LEAQ (BX)(DX*1), AX
|
|
SUBL DX, SI
|
|
LEAL -1(SI), DX
|
|
CMPL DX, $0x3c
|
|
JB one_byte_emit_remainder_encodeBlockAsm
|
|
CMPL DX, $0x00000100
|
|
JB two_bytes_emit_remainder_encodeBlockAsm
|
|
CMPL DX, $0x00010000
|
|
JB three_bytes_emit_remainder_encodeBlockAsm
|
|
CMPL DX, $0x01000000
|
|
JB four_bytes_emit_remainder_encodeBlockAsm
|
|
MOVB $0xfc, (CX)
|
|
MOVL DX, 1(CX)
|
|
ADDQ $0x05, CX
|
|
JMP memmove_long_emit_remainder_encodeBlockAsm
|
|
|
|
four_bytes_emit_remainder_encodeBlockAsm:
|
|
MOVL DX, BX
|
|
SHRL $0x10, BX
|
|
MOVB $0xf8, (CX)
|
|
MOVW DX, 1(CX)
|
|
MOVB BL, 3(CX)
|
|
ADDQ $0x04, CX
|
|
JMP memmove_long_emit_remainder_encodeBlockAsm
|
|
|
|
three_bytes_emit_remainder_encodeBlockAsm:
|
|
MOVB $0xf4, (CX)
|
|
MOVW DX, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_emit_remainder_encodeBlockAsm
|
|
|
|
two_bytes_emit_remainder_encodeBlockAsm:
|
|
MOVB $0xf0, (CX)
|
|
MOVB DL, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL DX, $0x40
|
|
JB memmove_emit_remainder_encodeBlockAsm
|
|
JMP memmove_long_emit_remainder_encodeBlockAsm
|
|
|
|
one_byte_emit_remainder_encodeBlockAsm:
|
|
SHLB $0x02, DL
|
|
MOVB DL, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_emit_remainder_encodeBlockAsm:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveShort
|
|
CMPQ BX, $0x03
|
|
JB emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_1or2
|
|
JE emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_3
|
|
CMPQ BX, $0x08
|
|
JB emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_4through7
|
|
CMPQ BX, $0x10
|
|
JBE emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_8through16
|
|
CMPQ BX, $0x20
|
|
JBE emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_1or2:
|
|
MOVB (AX), SI
|
|
MOVB -1(AX)(BX*1), AL
|
|
MOVB SI, (CX)
|
|
MOVB AL, -1(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_3:
|
|
MOVW (AX), SI
|
|
MOVB 2(AX), AL
|
|
MOVW SI, (CX)
|
|
MOVB AL, 2(CX)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_4through7:
|
|
MOVL (AX), SI
|
|
MOVL -4(AX)(BX*1), AX
|
|
MOVL SI, (CX)
|
|
MOVL AX, -4(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_8through16:
|
|
MOVQ (AX), SI
|
|
MOVQ -8(AX)(BX*1), AX
|
|
MOVQ SI, (CX)
|
|
MOVQ AX, -8(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_17through32:
|
|
MOVOU (AX), X0
|
|
MOVOU -16(AX)(BX*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_33through64:
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
|
|
memmove_end_copy_emit_remainder_encodeBlockAsm:
|
|
MOVQ DX, CX
|
|
JMP emit_literal_done_emit_remainder_encodeBlockAsm
|
|
|
|
memmove_long_emit_remainder_encodeBlockAsm:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveLong
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVQ BX, DI
|
|
SHRQ $0x05, DI
|
|
MOVQ CX, SI
|
|
ANDL $0x0000001f, SI
|
|
MOVQ $0x00000040, R8
|
|
SUBQ SI, R8
|
|
DECQ DI
|
|
JA emit_lit_memmove_long_emit_remainder_encodeBlockAsmlarge_forward_sse_loop_32
|
|
LEAQ -32(AX)(R8*1), SI
|
|
LEAQ -32(CX)(R8*1), R9
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeBlockAsmlarge_big_loop_back:
|
|
MOVOU (SI), X4
|
|
MOVOU 16(SI), X5
|
|
MOVOA X4, (R9)
|
|
MOVOA X5, 16(R9)
|
|
ADDQ $0x20, R9
|
|
ADDQ $0x20, SI
|
|
ADDQ $0x20, R8
|
|
DECQ DI
|
|
JNA emit_lit_memmove_long_emit_remainder_encodeBlockAsmlarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeBlockAsmlarge_forward_sse_loop_32:
|
|
MOVOU -32(AX)(R8*1), X4
|
|
MOVOU -16(AX)(R8*1), X5
|
|
MOVOA X4, -32(CX)(R8*1)
|
|
MOVOA X5, -16(CX)(R8*1)
|
|
ADDQ $0x20, R8
|
|
CMPQ BX, R8
|
|
JAE emit_lit_memmove_long_emit_remainder_encodeBlockAsmlarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
MOVQ DX, CX
|
|
|
|
emit_literal_done_emit_remainder_encodeBlockAsm:
|
|
MOVQ dst_base+0(FP), AX
|
|
SUBQ AX, CX
|
|
MOVQ CX, ret+56(FP)
|
|
RET
|
|
|
|
// func encodeBlockAsm4MB(dst []byte, src []byte, tmp *[65536]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeBlockAsm4MB(SB), $24-64
|
|
MOVQ tmp+48(FP), AX
|
|
MOVQ dst_base+0(FP), CX
|
|
MOVQ $0x00000200, DX
|
|
MOVQ AX, BX
|
|
PXOR X0, X0
|
|
|
|
zero_loop_encodeBlockAsm4MB:
|
|
MOVOU X0, (BX)
|
|
MOVOU X0, 16(BX)
|
|
MOVOU X0, 32(BX)
|
|
MOVOU X0, 48(BX)
|
|
MOVOU X0, 64(BX)
|
|
MOVOU X0, 80(BX)
|
|
MOVOU X0, 96(BX)
|
|
MOVOU X0, 112(BX)
|
|
ADDQ $0x80, BX
|
|
DECQ DX
|
|
JNZ zero_loop_encodeBlockAsm4MB
|
|
MOVL $0x00000000, 12(SP)
|
|
MOVQ src_len+32(FP), DX
|
|
LEAQ -9(DX), BX
|
|
LEAQ -8(DX), SI
|
|
MOVL SI, 8(SP)
|
|
SHRQ $0x05, DX
|
|
SUBL DX, BX
|
|
LEAQ (CX)(BX*1), BX
|
|
MOVQ BX, (SP)
|
|
MOVL $0x00000001, DX
|
|
MOVL DX, 16(SP)
|
|
MOVQ src_base+24(FP), BX
|
|
|
|
search_loop_encodeBlockAsm4MB:
|
|
MOVL DX, SI
|
|
SUBL 12(SP), SI
|
|
SHRL $0x06, SI
|
|
LEAL 4(DX)(SI*1), SI
|
|
CMPL SI, 8(SP)
|
|
JAE emit_remainder_encodeBlockAsm4MB
|
|
MOVQ (BX)(DX*1), DI
|
|
MOVL SI, 20(SP)
|
|
MOVQ $0x0000cf1bbcdcbf9b, R9
|
|
MOVQ DI, R10
|
|
MOVQ DI, R11
|
|
SHRQ $0x08, R11
|
|
SHLQ $0x10, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x32, R10
|
|
SHLQ $0x10, R11
|
|
IMULQ R9, R11
|
|
SHRQ $0x32, R11
|
|
MOVL (AX)(R10*4), SI
|
|
MOVL (AX)(R11*4), R8
|
|
MOVL DX, (AX)(R10*4)
|
|
LEAL 1(DX), R10
|
|
MOVL R10, (AX)(R11*4)
|
|
MOVQ DI, R10
|
|
SHRQ $0x10, R10
|
|
SHLQ $0x10, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x32, R10
|
|
MOVL DX, R9
|
|
SUBL 16(SP), R9
|
|
MOVL 1(BX)(R9*1), R11
|
|
MOVQ DI, R9
|
|
SHRQ $0x08, R9
|
|
CMPL R9, R11
|
|
JNE no_repeat_found_encodeBlockAsm4MB
|
|
LEAL 1(DX), DI
|
|
MOVL 12(SP), R8
|
|
MOVL DI, SI
|
|
SUBL 16(SP), SI
|
|
JZ repeat_extend_back_end_encodeBlockAsm4MB
|
|
|
|
repeat_extend_back_loop_encodeBlockAsm4MB:
|
|
CMPL DI, R8
|
|
JBE repeat_extend_back_end_encodeBlockAsm4MB
|
|
MOVB -1(BX)(SI*1), R9
|
|
MOVB -1(BX)(DI*1), R10
|
|
CMPB R9, R10
|
|
JNE repeat_extend_back_end_encodeBlockAsm4MB
|
|
LEAL -1(DI), DI
|
|
DECL SI
|
|
JNZ repeat_extend_back_loop_encodeBlockAsm4MB
|
|
|
|
repeat_extend_back_end_encodeBlockAsm4MB:
|
|
MOVL DI, SI
|
|
SUBL 12(SP), SI
|
|
LEAQ 4(CX)(SI*1), SI
|
|
CMPQ SI, (SP)
|
|
JB repeat_dst_size_check_encodeBlockAsm4MB
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
repeat_dst_size_check_encodeBlockAsm4MB:
|
|
MOVL 12(SP), SI
|
|
CMPL SI, DI
|
|
JEQ emit_literal_done_repeat_emit_encodeBlockAsm4MB
|
|
MOVL DI, R9
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(SI*1), R10
|
|
SUBL SI, R9
|
|
LEAL -1(R9), SI
|
|
CMPL SI, $0x3c
|
|
JB one_byte_repeat_emit_encodeBlockAsm4MB
|
|
CMPL SI, $0x00000100
|
|
JB two_bytes_repeat_emit_encodeBlockAsm4MB
|
|
CMPL SI, $0x00010000
|
|
JB three_bytes_repeat_emit_encodeBlockAsm4MB
|
|
MOVL SI, R11
|
|
SHRL $0x10, R11
|
|
MOVB $0xf8, (CX)
|
|
MOVW SI, 1(CX)
|
|
MOVB R11, 3(CX)
|
|
ADDQ $0x04, CX
|
|
JMP memmove_long_repeat_emit_encodeBlockAsm4MB
|
|
|
|
three_bytes_repeat_emit_encodeBlockAsm4MB:
|
|
MOVB $0xf4, (CX)
|
|
MOVW SI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_repeat_emit_encodeBlockAsm4MB
|
|
|
|
two_bytes_repeat_emit_encodeBlockAsm4MB:
|
|
MOVB $0xf0, (CX)
|
|
MOVB SI, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL SI, $0x40
|
|
JB memmove_repeat_emit_encodeBlockAsm4MB
|
|
JMP memmove_long_repeat_emit_encodeBlockAsm4MB
|
|
|
|
one_byte_repeat_emit_encodeBlockAsm4MB:
|
|
SHLB $0x02, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_repeat_emit_encodeBlockAsm4MB:
|
|
LEAQ (CX)(R9*1), SI
|
|
|
|
// genMemMoveShort
|
|
CMPQ R9, $0x08
|
|
JBE emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_8
|
|
CMPQ R9, $0x10
|
|
JBE emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_8through16
|
|
CMPQ R9, $0x20
|
|
JBE emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_17through32
|
|
JMP emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_33through64
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_8:
|
|
MOVQ (R10), R11
|
|
MOVQ R11, (CX)
|
|
JMP memmove_end_copy_repeat_emit_encodeBlockAsm4MB
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_8through16:
|
|
MOVQ (R10), R11
|
|
MOVQ -8(R10)(R9*1), R10
|
|
MOVQ R11, (CX)
|
|
MOVQ R10, -8(CX)(R9*1)
|
|
JMP memmove_end_copy_repeat_emit_encodeBlockAsm4MB
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_17through32:
|
|
MOVOU (R10), X0
|
|
MOVOU -16(R10)(R9*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(R9*1)
|
|
JMP memmove_end_copy_repeat_emit_encodeBlockAsm4MB
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_33through64:
|
|
MOVOU (R10), X0
|
|
MOVOU 16(R10), X1
|
|
MOVOU -32(R10)(R9*1), X2
|
|
MOVOU -16(R10)(R9*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
|
|
memmove_end_copy_repeat_emit_encodeBlockAsm4MB:
|
|
MOVQ SI, CX
|
|
JMP emit_literal_done_repeat_emit_encodeBlockAsm4MB
|
|
|
|
memmove_long_repeat_emit_encodeBlockAsm4MB:
|
|
LEAQ (CX)(R9*1), SI
|
|
|
|
// genMemMoveLong
|
|
MOVOU (R10), X0
|
|
MOVOU 16(R10), X1
|
|
MOVOU -32(R10)(R9*1), X2
|
|
MOVOU -16(R10)(R9*1), X3
|
|
MOVQ R9, R12
|
|
SHRQ $0x05, R12
|
|
MOVQ CX, R11
|
|
ANDL $0x0000001f, R11
|
|
MOVQ $0x00000040, R13
|
|
SUBQ R11, R13
|
|
DECQ R12
|
|
JA emit_lit_memmove_long_repeat_emit_encodeBlockAsm4MBlarge_forward_sse_loop_32
|
|
LEAQ -32(R10)(R13*1), R11
|
|
LEAQ -32(CX)(R13*1), R14
|
|
|
|
emit_lit_memmove_long_repeat_emit_encodeBlockAsm4MBlarge_big_loop_back:
|
|
MOVOU (R11), X4
|
|
MOVOU 16(R11), X5
|
|
MOVOA X4, (R14)
|
|
MOVOA X5, 16(R14)
|
|
ADDQ $0x20, R14
|
|
ADDQ $0x20, R11
|
|
ADDQ $0x20, R13
|
|
DECQ R12
|
|
JNA emit_lit_memmove_long_repeat_emit_encodeBlockAsm4MBlarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_repeat_emit_encodeBlockAsm4MBlarge_forward_sse_loop_32:
|
|
MOVOU -32(R10)(R13*1), X4
|
|
MOVOU -16(R10)(R13*1), X5
|
|
MOVOA X4, -32(CX)(R13*1)
|
|
MOVOA X5, -16(CX)(R13*1)
|
|
ADDQ $0x20, R13
|
|
CMPQ R9, R13
|
|
JAE emit_lit_memmove_long_repeat_emit_encodeBlockAsm4MBlarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
MOVQ SI, CX
|
|
|
|
emit_literal_done_repeat_emit_encodeBlockAsm4MB:
|
|
ADDL $0x05, DX
|
|
MOVL DX, SI
|
|
SUBL 16(SP), SI
|
|
MOVQ src_len+32(FP), R9
|
|
SUBL DX, R9
|
|
LEAQ (BX)(DX*1), R10
|
|
LEAQ (BX)(SI*1), SI
|
|
|
|
// matchLen
|
|
XORL R12, R12
|
|
|
|
matchlen_loopback_16_repeat_extend_encodeBlockAsm4MB:
|
|
CMPL R9, $0x10
|
|
JB matchlen_match8_repeat_extend_encodeBlockAsm4MB
|
|
MOVQ (R10)(R12*1), R11
|
|
MOVQ 8(R10)(R12*1), R13
|
|
XORQ (SI)(R12*1), R11
|
|
JNZ matchlen_bsf_8_repeat_extend_encodeBlockAsm4MB
|
|
XORQ 8(SI)(R12*1), R13
|
|
JNZ matchlen_bsf_16repeat_extend_encodeBlockAsm4MB
|
|
LEAL -16(R9), R9
|
|
LEAL 16(R12), R12
|
|
JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm4MB
|
|
|
|
matchlen_bsf_16repeat_extend_encodeBlockAsm4MB:
|
|
TZCNTQ R13, R13
|
|
SARQ $0x03, R13
|
|
LEAL 8(R12)(R13*1), R12
|
|
JMP repeat_extend_forward_end_encodeBlockAsm4MB
|
|
|
|
matchlen_match8_repeat_extend_encodeBlockAsm4MB:
|
|
CMPL R9, $0x08
|
|
JB matchlen_match4_repeat_extend_encodeBlockAsm4MB
|
|
MOVQ (R10)(R12*1), R11
|
|
XORQ (SI)(R12*1), R11
|
|
JNZ matchlen_bsf_8_repeat_extend_encodeBlockAsm4MB
|
|
LEAL -8(R9), R9
|
|
LEAL 8(R12), R12
|
|
JMP matchlen_match4_repeat_extend_encodeBlockAsm4MB
|
|
|
|
matchlen_bsf_8_repeat_extend_encodeBlockAsm4MB:
|
|
TZCNTQ R11, R11
|
|
SARQ $0x03, R11
|
|
LEAL (R12)(R11*1), R12
|
|
JMP repeat_extend_forward_end_encodeBlockAsm4MB
|
|
|
|
matchlen_match4_repeat_extend_encodeBlockAsm4MB:
|
|
CMPL R9, $0x04
|
|
JB matchlen_match2_repeat_extend_encodeBlockAsm4MB
|
|
MOVL (R10)(R12*1), R11
|
|
CMPL (SI)(R12*1), R11
|
|
JNE matchlen_match2_repeat_extend_encodeBlockAsm4MB
|
|
LEAL -4(R9), R9
|
|
LEAL 4(R12), R12
|
|
|
|
matchlen_match2_repeat_extend_encodeBlockAsm4MB:
|
|
CMPL R9, $0x01
|
|
JE matchlen_match1_repeat_extend_encodeBlockAsm4MB
|
|
JB repeat_extend_forward_end_encodeBlockAsm4MB
|
|
MOVW (R10)(R12*1), R11
|
|
CMPW (SI)(R12*1), R11
|
|
JNE matchlen_match1_repeat_extend_encodeBlockAsm4MB
|
|
LEAL 2(R12), R12
|
|
SUBL $0x02, R9
|
|
JZ repeat_extend_forward_end_encodeBlockAsm4MB
|
|
|
|
matchlen_match1_repeat_extend_encodeBlockAsm4MB:
|
|
MOVB (R10)(R12*1), R11
|
|
CMPB (SI)(R12*1), R11
|
|
JNE repeat_extend_forward_end_encodeBlockAsm4MB
|
|
LEAL 1(R12), R12
|
|
|
|
repeat_extend_forward_end_encodeBlockAsm4MB:
|
|
ADDL R12, DX
|
|
MOVL DX, SI
|
|
SUBL DI, SI
|
|
MOVL 16(SP), DI
|
|
TESTL R8, R8
|
|
JZ repeat_as_copy_encodeBlockAsm4MB
|
|
|
|
// emitRepeat
|
|
MOVL SI, R8
|
|
LEAL -4(SI), SI
|
|
CMPL R8, $0x08
|
|
JBE repeat_two_match_repeat_encodeBlockAsm4MB
|
|
CMPL R8, $0x0c
|
|
JAE cant_repeat_two_offset_match_repeat_encodeBlockAsm4MB
|
|
CMPL DI, $0x00000800
|
|
JB repeat_two_offset_match_repeat_encodeBlockAsm4MB
|
|
|
|
cant_repeat_two_offset_match_repeat_encodeBlockAsm4MB:
|
|
CMPL SI, $0x00000104
|
|
JB repeat_three_match_repeat_encodeBlockAsm4MB
|
|
CMPL SI, $0x00010100
|
|
JB repeat_four_match_repeat_encodeBlockAsm4MB
|
|
LEAL -65536(SI), SI
|
|
MOVL SI, DI
|
|
MOVW $0x001d, (CX)
|
|
MOVW SI, 2(CX)
|
|
SARL $0x10, DI
|
|
MOVB DI, 4(CX)
|
|
ADDQ $0x05, CX
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
repeat_four_match_repeat_encodeBlockAsm4MB:
|
|
LEAL -256(SI), SI
|
|
MOVW $0x0019, (CX)
|
|
MOVW SI, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
repeat_three_match_repeat_encodeBlockAsm4MB:
|
|
LEAL -4(SI), SI
|
|
MOVW $0x0015, (CX)
|
|
MOVB SI, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
repeat_two_match_repeat_encodeBlockAsm4MB:
|
|
SHLL $0x02, SI
|
|
ORL $0x01, SI
|
|
MOVW SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
repeat_two_offset_match_repeat_encodeBlockAsm4MB:
|
|
XORQ R8, R8
|
|
LEAL 1(R8)(SI*4), SI
|
|
MOVB DI, 1(CX)
|
|
SARL $0x08, DI
|
|
SHLL $0x05, DI
|
|
ORL DI, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
repeat_as_copy_encodeBlockAsm4MB:
|
|
// emitCopy
|
|
CMPL DI, $0x00010000
|
|
JB two_byte_offset_repeat_as_copy_encodeBlockAsm4MB
|
|
CMPL SI, $0x40
|
|
JBE four_bytes_remain_repeat_as_copy_encodeBlockAsm4MB
|
|
MOVB $0xff, (CX)
|
|
MOVL DI, 1(CX)
|
|
LEAL -64(SI), SI
|
|
ADDQ $0x05, CX
|
|
CMPL SI, $0x04
|
|
JB four_bytes_remain_repeat_as_copy_encodeBlockAsm4MB
|
|
|
|
// emitRepeat
|
|
MOVL SI, R8
|
|
LEAL -4(SI), SI
|
|
CMPL R8, $0x08
|
|
JBE repeat_two_repeat_as_copy_encodeBlockAsm4MB_emit_copy
|
|
CMPL R8, $0x0c
|
|
JAE cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy
|
|
CMPL DI, $0x00000800
|
|
JB repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy
|
|
|
|
cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy:
|
|
CMPL SI, $0x00000104
|
|
JB repeat_three_repeat_as_copy_encodeBlockAsm4MB_emit_copy
|
|
CMPL SI, $0x00010100
|
|
JB repeat_four_repeat_as_copy_encodeBlockAsm4MB_emit_copy
|
|
LEAL -65536(SI), SI
|
|
MOVL SI, DI
|
|
MOVW $0x001d, (CX)
|
|
MOVW SI, 2(CX)
|
|
SARL $0x10, DI
|
|
MOVB DI, 4(CX)
|
|
ADDQ $0x05, CX
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
repeat_four_repeat_as_copy_encodeBlockAsm4MB_emit_copy:
|
|
LEAL -256(SI), SI
|
|
MOVW $0x0019, (CX)
|
|
MOVW SI, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
repeat_three_repeat_as_copy_encodeBlockAsm4MB_emit_copy:
|
|
LEAL -4(SI), SI
|
|
MOVW $0x0015, (CX)
|
|
MOVB SI, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
repeat_two_repeat_as_copy_encodeBlockAsm4MB_emit_copy:
|
|
SHLL $0x02, SI
|
|
ORL $0x01, SI
|
|
MOVW SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy:
|
|
XORQ R8, R8
|
|
LEAL 1(R8)(SI*4), SI
|
|
MOVB DI, 1(CX)
|
|
SARL $0x08, DI
|
|
SHLL $0x05, DI
|
|
ORL DI, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
four_bytes_remain_repeat_as_copy_encodeBlockAsm4MB:
|
|
TESTL SI, SI
|
|
JZ repeat_end_emit_encodeBlockAsm4MB
|
|
XORL R8, R8
|
|
LEAL -1(R8)(SI*4), SI
|
|
MOVB SI, (CX)
|
|
MOVL DI, 1(CX)
|
|
ADDQ $0x05, CX
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
two_byte_offset_repeat_as_copy_encodeBlockAsm4MB:
|
|
CMPL SI, $0x40
|
|
JBE two_byte_offset_short_repeat_as_copy_encodeBlockAsm4MB
|
|
CMPL DI, $0x00000800
|
|
JAE long_offset_short_repeat_as_copy_encodeBlockAsm4MB
|
|
MOVL $0x00000001, R8
|
|
LEAL 16(R8), R8
|
|
MOVB DI, 1(CX)
|
|
SHRL $0x08, DI
|
|
SHLL $0x05, DI
|
|
ORL DI, R8
|
|
MOVB R8, (CX)
|
|
ADDQ $0x02, CX
|
|
SUBL $0x08, SI
|
|
|
|
// emitRepeat
|
|
LEAL -4(SI), SI
|
|
JMP cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b
|
|
MOVL SI, R8
|
|
LEAL -4(SI), SI
|
|
CMPL R8, $0x08
|
|
JBE repeat_two_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b
|
|
CMPL R8, $0x0c
|
|
JAE cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b
|
|
CMPL DI, $0x00000800
|
|
JB repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b
|
|
|
|
cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b:
|
|
CMPL SI, $0x00000104
|
|
JB repeat_three_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b
|
|
CMPL SI, $0x00010100
|
|
JB repeat_four_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b
|
|
LEAL -65536(SI), SI
|
|
MOVL SI, DI
|
|
MOVW $0x001d, (CX)
|
|
MOVW SI, 2(CX)
|
|
SARL $0x10, DI
|
|
MOVB DI, 4(CX)
|
|
ADDQ $0x05, CX
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
repeat_four_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b:
|
|
LEAL -256(SI), SI
|
|
MOVW $0x0019, (CX)
|
|
MOVW SI, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
repeat_three_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b:
|
|
LEAL -4(SI), SI
|
|
MOVW $0x0015, (CX)
|
|
MOVB SI, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
repeat_two_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b:
|
|
SHLL $0x02, SI
|
|
ORL $0x01, SI
|
|
MOVW SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b:
|
|
XORQ R8, R8
|
|
LEAL 1(R8)(SI*4), SI
|
|
MOVB DI, 1(CX)
|
|
SARL $0x08, DI
|
|
SHLL $0x05, DI
|
|
ORL DI, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
long_offset_short_repeat_as_copy_encodeBlockAsm4MB:
|
|
MOVB $0xee, (CX)
|
|
MOVW DI, 1(CX)
|
|
LEAL -60(SI), SI
|
|
ADDQ $0x03, CX
|
|
|
|
// emitRepeat
|
|
MOVL SI, R8
|
|
LEAL -4(SI), SI
|
|
CMPL R8, $0x08
|
|
JBE repeat_two_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short
|
|
CMPL R8, $0x0c
|
|
JAE cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short
|
|
CMPL DI, $0x00000800
|
|
JB repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short
|
|
|
|
cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short:
|
|
CMPL SI, $0x00000104
|
|
JB repeat_three_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short
|
|
CMPL SI, $0x00010100
|
|
JB repeat_four_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short
|
|
LEAL -65536(SI), SI
|
|
MOVL SI, DI
|
|
MOVW $0x001d, (CX)
|
|
MOVW SI, 2(CX)
|
|
SARL $0x10, DI
|
|
MOVB DI, 4(CX)
|
|
ADDQ $0x05, CX
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
repeat_four_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short:
|
|
LEAL -256(SI), SI
|
|
MOVW $0x0019, (CX)
|
|
MOVW SI, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
repeat_three_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short:
|
|
LEAL -4(SI), SI
|
|
MOVW $0x0015, (CX)
|
|
MOVB SI, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
repeat_two_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short:
|
|
SHLL $0x02, SI
|
|
ORL $0x01, SI
|
|
MOVW SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short:
|
|
XORQ R8, R8
|
|
LEAL 1(R8)(SI*4), SI
|
|
MOVB DI, 1(CX)
|
|
SARL $0x08, DI
|
|
SHLL $0x05, DI
|
|
ORL DI, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
two_byte_offset_short_repeat_as_copy_encodeBlockAsm4MB:
|
|
MOVL SI, R8
|
|
SHLL $0x02, R8
|
|
CMPL SI, $0x0c
|
|
JAE emit_copy_three_repeat_as_copy_encodeBlockAsm4MB
|
|
CMPL DI, $0x00000800
|
|
JAE emit_copy_three_repeat_as_copy_encodeBlockAsm4MB
|
|
LEAL -15(R8), R8
|
|
MOVB DI, 1(CX)
|
|
SHRL $0x08, DI
|
|
SHLL $0x05, DI
|
|
ORL DI, R8
|
|
MOVB R8, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
emit_copy_three_repeat_as_copy_encodeBlockAsm4MB:
|
|
LEAL -2(R8), R8
|
|
MOVB R8, (CX)
|
|
MOVW DI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
|
|
repeat_end_emit_encodeBlockAsm4MB:
|
|
MOVL DX, 12(SP)
|
|
JMP search_loop_encodeBlockAsm4MB
|
|
|
|
no_repeat_found_encodeBlockAsm4MB:
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ candidate_match_encodeBlockAsm4MB
|
|
SHRQ $0x08, DI
|
|
MOVL (AX)(R10*4), SI
|
|
LEAL 2(DX), R9
|
|
CMPL (BX)(R8*1), DI
|
|
JEQ candidate2_match_encodeBlockAsm4MB
|
|
MOVL R9, (AX)(R10*4)
|
|
SHRQ $0x08, DI
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ candidate3_match_encodeBlockAsm4MB
|
|
MOVL 20(SP), DX
|
|
JMP search_loop_encodeBlockAsm4MB
|
|
|
|
candidate3_match_encodeBlockAsm4MB:
|
|
ADDL $0x02, DX
|
|
JMP candidate_match_encodeBlockAsm4MB
|
|
|
|
candidate2_match_encodeBlockAsm4MB:
|
|
MOVL R9, (AX)(R10*4)
|
|
INCL DX
|
|
MOVL R8, SI
|
|
|
|
candidate_match_encodeBlockAsm4MB:
|
|
MOVL 12(SP), DI
|
|
TESTL SI, SI
|
|
JZ match_extend_back_end_encodeBlockAsm4MB
|
|
|
|
match_extend_back_loop_encodeBlockAsm4MB:
|
|
CMPL DX, DI
|
|
JBE match_extend_back_end_encodeBlockAsm4MB
|
|
MOVB -1(BX)(SI*1), R8
|
|
MOVB -1(BX)(DX*1), R9
|
|
CMPB R8, R9
|
|
JNE match_extend_back_end_encodeBlockAsm4MB
|
|
LEAL -1(DX), DX
|
|
DECL SI
|
|
JZ match_extend_back_end_encodeBlockAsm4MB
|
|
JMP match_extend_back_loop_encodeBlockAsm4MB
|
|
|
|
match_extend_back_end_encodeBlockAsm4MB:
|
|
MOVL DX, DI
|
|
SUBL 12(SP), DI
|
|
LEAQ 4(CX)(DI*1), DI
|
|
CMPQ DI, (SP)
|
|
JB match_dst_size_check_encodeBlockAsm4MB
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeBlockAsm4MB:
|
|
MOVL DX, DI
|
|
MOVL 12(SP), R8
|
|
CMPL R8, DI
|
|
JEQ emit_literal_done_match_emit_encodeBlockAsm4MB
|
|
MOVL DI, R9
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(R8*1), DI
|
|
SUBL R8, R9
|
|
LEAL -1(R9), R8
|
|
CMPL R8, $0x3c
|
|
JB one_byte_match_emit_encodeBlockAsm4MB
|
|
CMPL R8, $0x00000100
|
|
JB two_bytes_match_emit_encodeBlockAsm4MB
|
|
CMPL R8, $0x00010000
|
|
JB three_bytes_match_emit_encodeBlockAsm4MB
|
|
MOVL R8, R10
|
|
SHRL $0x10, R10
|
|
MOVB $0xf8, (CX)
|
|
MOVW R8, 1(CX)
|
|
MOVB R10, 3(CX)
|
|
ADDQ $0x04, CX
|
|
JMP memmove_long_match_emit_encodeBlockAsm4MB
|
|
|
|
three_bytes_match_emit_encodeBlockAsm4MB:
|
|
MOVB $0xf4, (CX)
|
|
MOVW R8, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_match_emit_encodeBlockAsm4MB
|
|
|
|
two_bytes_match_emit_encodeBlockAsm4MB:
|
|
MOVB $0xf0, (CX)
|
|
MOVB R8, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL R8, $0x40
|
|
JB memmove_match_emit_encodeBlockAsm4MB
|
|
JMP memmove_long_match_emit_encodeBlockAsm4MB
|
|
|
|
one_byte_match_emit_encodeBlockAsm4MB:
|
|
SHLB $0x02, R8
|
|
MOVB R8, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_match_emit_encodeBlockAsm4MB:
|
|
LEAQ (CX)(R9*1), R8
|
|
|
|
// genMemMoveShort
|
|
CMPQ R9, $0x08
|
|
JBE emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_8
|
|
CMPQ R9, $0x10
|
|
JBE emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_8through16
|
|
CMPQ R9, $0x20
|
|
JBE emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_8:
|
|
MOVQ (DI), R10
|
|
MOVQ R10, (CX)
|
|
JMP memmove_end_copy_match_emit_encodeBlockAsm4MB
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_8through16:
|
|
MOVQ (DI), R10
|
|
MOVQ -8(DI)(R9*1), DI
|
|
MOVQ R10, (CX)
|
|
MOVQ DI, -8(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeBlockAsm4MB
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_17through32:
|
|
MOVOU (DI), X0
|
|
MOVOU -16(DI)(R9*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeBlockAsm4MB
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_33through64:
|
|
MOVOU (DI), X0
|
|
MOVOU 16(DI), X1
|
|
MOVOU -32(DI)(R9*1), X2
|
|
MOVOU -16(DI)(R9*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
|
|
memmove_end_copy_match_emit_encodeBlockAsm4MB:
|
|
MOVQ R8, CX
|
|
JMP emit_literal_done_match_emit_encodeBlockAsm4MB
|
|
|
|
memmove_long_match_emit_encodeBlockAsm4MB:
|
|
LEAQ (CX)(R9*1), R8
|
|
|
|
// genMemMoveLong
|
|
MOVOU (DI), X0
|
|
MOVOU 16(DI), X1
|
|
MOVOU -32(DI)(R9*1), X2
|
|
MOVOU -16(DI)(R9*1), X3
|
|
MOVQ R9, R11
|
|
SHRQ $0x05, R11
|
|
MOVQ CX, R10
|
|
ANDL $0x0000001f, R10
|
|
MOVQ $0x00000040, R12
|
|
SUBQ R10, R12
|
|
DECQ R11
|
|
JA emit_lit_memmove_long_match_emit_encodeBlockAsm4MBlarge_forward_sse_loop_32
|
|
LEAQ -32(DI)(R12*1), R10
|
|
LEAQ -32(CX)(R12*1), R13
|
|
|
|
emit_lit_memmove_long_match_emit_encodeBlockAsm4MBlarge_big_loop_back:
|
|
MOVOU (R10), X4
|
|
MOVOU 16(R10), X5
|
|
MOVOA X4, (R13)
|
|
MOVOA X5, 16(R13)
|
|
ADDQ $0x20, R13
|
|
ADDQ $0x20, R10
|
|
ADDQ $0x20, R12
|
|
DECQ R11
|
|
JNA emit_lit_memmove_long_match_emit_encodeBlockAsm4MBlarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_match_emit_encodeBlockAsm4MBlarge_forward_sse_loop_32:
|
|
MOVOU -32(DI)(R12*1), X4
|
|
MOVOU -16(DI)(R12*1), X5
|
|
MOVOA X4, -32(CX)(R12*1)
|
|
MOVOA X5, -16(CX)(R12*1)
|
|
ADDQ $0x20, R12
|
|
CMPQ R9, R12
|
|
JAE emit_lit_memmove_long_match_emit_encodeBlockAsm4MBlarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
MOVQ R8, CX
|
|
|
|
emit_literal_done_match_emit_encodeBlockAsm4MB:
|
|
match_nolit_loop_encodeBlockAsm4MB:
|
|
MOVL DX, DI
|
|
SUBL SI, DI
|
|
MOVL DI, 16(SP)
|
|
ADDL $0x04, DX
|
|
ADDL $0x04, SI
|
|
MOVQ src_len+32(FP), DI
|
|
SUBL DX, DI
|
|
LEAQ (BX)(DX*1), R8
|
|
LEAQ (BX)(SI*1), SI
|
|
|
|
// matchLen
|
|
XORL R10, R10
|
|
|
|
matchlen_loopback_16_match_nolit_encodeBlockAsm4MB:
|
|
CMPL DI, $0x10
|
|
JB matchlen_match8_match_nolit_encodeBlockAsm4MB
|
|
MOVQ (R8)(R10*1), R9
|
|
MOVQ 8(R8)(R10*1), R11
|
|
XORQ (SI)(R10*1), R9
|
|
JNZ matchlen_bsf_8_match_nolit_encodeBlockAsm4MB
|
|
XORQ 8(SI)(R10*1), R11
|
|
JNZ matchlen_bsf_16match_nolit_encodeBlockAsm4MB
|
|
LEAL -16(DI), DI
|
|
LEAL 16(R10), R10
|
|
JMP matchlen_loopback_16_match_nolit_encodeBlockAsm4MB
|
|
|
|
matchlen_bsf_16match_nolit_encodeBlockAsm4MB:
|
|
TZCNTQ R11, R11
|
|
SARQ $0x03, R11
|
|
LEAL 8(R10)(R11*1), R10
|
|
JMP match_nolit_end_encodeBlockAsm4MB
|
|
|
|
matchlen_match8_match_nolit_encodeBlockAsm4MB:
|
|
CMPL DI, $0x08
|
|
JB matchlen_match4_match_nolit_encodeBlockAsm4MB
|
|
MOVQ (R8)(R10*1), R9
|
|
XORQ (SI)(R10*1), R9
|
|
JNZ matchlen_bsf_8_match_nolit_encodeBlockAsm4MB
|
|
LEAL -8(DI), DI
|
|
LEAL 8(R10), R10
|
|
JMP matchlen_match4_match_nolit_encodeBlockAsm4MB
|
|
|
|
matchlen_bsf_8_match_nolit_encodeBlockAsm4MB:
|
|
TZCNTQ R9, R9
|
|
SARQ $0x03, R9
|
|
LEAL (R10)(R9*1), R10
|
|
JMP match_nolit_end_encodeBlockAsm4MB
|
|
|
|
matchlen_match4_match_nolit_encodeBlockAsm4MB:
|
|
CMPL DI, $0x04
|
|
JB matchlen_match2_match_nolit_encodeBlockAsm4MB
|
|
MOVL (R8)(R10*1), R9
|
|
CMPL (SI)(R10*1), R9
|
|
JNE matchlen_match2_match_nolit_encodeBlockAsm4MB
|
|
LEAL -4(DI), DI
|
|
LEAL 4(R10), R10
|
|
|
|
matchlen_match2_match_nolit_encodeBlockAsm4MB:
|
|
CMPL DI, $0x01
|
|
JE matchlen_match1_match_nolit_encodeBlockAsm4MB
|
|
JB match_nolit_end_encodeBlockAsm4MB
|
|
MOVW (R8)(R10*1), R9
|
|
CMPW (SI)(R10*1), R9
|
|
JNE matchlen_match1_match_nolit_encodeBlockAsm4MB
|
|
LEAL 2(R10), R10
|
|
SUBL $0x02, DI
|
|
JZ match_nolit_end_encodeBlockAsm4MB
|
|
|
|
matchlen_match1_match_nolit_encodeBlockAsm4MB:
|
|
MOVB (R8)(R10*1), R9
|
|
CMPB (SI)(R10*1), R9
|
|
JNE match_nolit_end_encodeBlockAsm4MB
|
|
LEAL 1(R10), R10
|
|
|
|
match_nolit_end_encodeBlockAsm4MB:
|
|
ADDL R10, DX
|
|
MOVL 16(SP), SI
|
|
ADDL $0x04, R10
|
|
MOVL DX, 12(SP)
|
|
|
|
// emitCopy
|
|
CMPL SI, $0x00010000
|
|
JB two_byte_offset_match_nolit_encodeBlockAsm4MB
|
|
CMPL R10, $0x40
|
|
JBE four_bytes_remain_match_nolit_encodeBlockAsm4MB
|
|
MOVB $0xff, (CX)
|
|
MOVL SI, 1(CX)
|
|
LEAL -64(R10), R10
|
|
ADDQ $0x05, CX
|
|
CMPL R10, $0x04
|
|
JB four_bytes_remain_match_nolit_encodeBlockAsm4MB
|
|
|
|
// emitRepeat
|
|
MOVL R10, DI
|
|
LEAL -4(R10), R10
|
|
CMPL DI, $0x08
|
|
JBE repeat_two_match_nolit_encodeBlockAsm4MB_emit_copy
|
|
CMPL DI, $0x0c
|
|
JAE cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy
|
|
CMPL SI, $0x00000800
|
|
JB repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy:
|
|
CMPL R10, $0x00000104
|
|
JB repeat_three_match_nolit_encodeBlockAsm4MB_emit_copy
|
|
CMPL R10, $0x00010100
|
|
JB repeat_four_match_nolit_encodeBlockAsm4MB_emit_copy
|
|
LEAL -65536(R10), R10
|
|
MOVL R10, SI
|
|
MOVW $0x001d, (CX)
|
|
MOVW R10, 2(CX)
|
|
SARL $0x10, SI
|
|
MOVB SI, 4(CX)
|
|
ADDQ $0x05, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm4MB
|
|
|
|
repeat_four_match_nolit_encodeBlockAsm4MB_emit_copy:
|
|
LEAL -256(R10), R10
|
|
MOVW $0x0019, (CX)
|
|
MOVW R10, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm4MB
|
|
|
|
repeat_three_match_nolit_encodeBlockAsm4MB_emit_copy:
|
|
LEAL -4(R10), R10
|
|
MOVW $0x0015, (CX)
|
|
MOVB R10, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm4MB
|
|
|
|
repeat_two_match_nolit_encodeBlockAsm4MB_emit_copy:
|
|
SHLL $0x02, R10
|
|
ORL $0x01, R10
|
|
MOVW R10, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm4MB
|
|
|
|
repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy:
|
|
XORQ DI, DI
|
|
LEAL 1(DI)(R10*4), R10
|
|
MOVB SI, 1(CX)
|
|
SARL $0x08, SI
|
|
SHLL $0x05, SI
|
|
ORL SI, R10
|
|
MOVB R10, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm4MB
|
|
|
|
four_bytes_remain_match_nolit_encodeBlockAsm4MB:
|
|
TESTL R10, R10
|
|
JZ match_nolit_emitcopy_end_encodeBlockAsm4MB
|
|
XORL DI, DI
|
|
LEAL -1(DI)(R10*4), R10
|
|
MOVB R10, (CX)
|
|
MOVL SI, 1(CX)
|
|
ADDQ $0x05, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm4MB
|
|
|
|
two_byte_offset_match_nolit_encodeBlockAsm4MB:
|
|
CMPL R10, $0x40
|
|
JBE two_byte_offset_short_match_nolit_encodeBlockAsm4MB
|
|
CMPL SI, $0x00000800
|
|
JAE long_offset_short_match_nolit_encodeBlockAsm4MB
|
|
MOVL $0x00000001, DI
|
|
LEAL 16(DI), DI
|
|
MOVB SI, 1(CX)
|
|
SHRL $0x08, SI
|
|
SHLL $0x05, SI
|
|
ORL SI, DI
|
|
MOVB DI, (CX)
|
|
ADDQ $0x02, CX
|
|
SUBL $0x08, R10
|
|
|
|
// emitRepeat
|
|
LEAL -4(R10), R10
|
|
JMP cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b
|
|
MOVL R10, DI
|
|
LEAL -4(R10), R10
|
|
CMPL DI, $0x08
|
|
JBE repeat_two_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b
|
|
CMPL DI, $0x0c
|
|
JAE cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b
|
|
CMPL SI, $0x00000800
|
|
JB repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b:
|
|
CMPL R10, $0x00000104
|
|
JB repeat_three_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b
|
|
CMPL R10, $0x00010100
|
|
JB repeat_four_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b
|
|
LEAL -65536(R10), R10
|
|
MOVL R10, SI
|
|
MOVW $0x001d, (CX)
|
|
MOVW R10, 2(CX)
|
|
SARL $0x10, SI
|
|
MOVB SI, 4(CX)
|
|
ADDQ $0x05, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm4MB
|
|
|
|
repeat_four_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b:
|
|
LEAL -256(R10), R10
|
|
MOVW $0x0019, (CX)
|
|
MOVW R10, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm4MB
|
|
|
|
repeat_three_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b:
|
|
LEAL -4(R10), R10
|
|
MOVW $0x0015, (CX)
|
|
MOVB R10, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm4MB
|
|
|
|
repeat_two_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b:
|
|
SHLL $0x02, R10
|
|
ORL $0x01, R10
|
|
MOVW R10, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm4MB
|
|
|
|
repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b:
|
|
XORQ DI, DI
|
|
LEAL 1(DI)(R10*4), R10
|
|
MOVB SI, 1(CX)
|
|
SARL $0x08, SI
|
|
SHLL $0x05, SI
|
|
ORL SI, R10
|
|
MOVB R10, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm4MB
|
|
|
|
long_offset_short_match_nolit_encodeBlockAsm4MB:
|
|
MOVB $0xee, (CX)
|
|
MOVW SI, 1(CX)
|
|
LEAL -60(R10), R10
|
|
ADDQ $0x03, CX
|
|
|
|
// emitRepeat
|
|
MOVL R10, DI
|
|
LEAL -4(R10), R10
|
|
CMPL DI, $0x08
|
|
JBE repeat_two_match_nolit_encodeBlockAsm4MB_emit_copy_short
|
|
CMPL DI, $0x0c
|
|
JAE cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short
|
|
CMPL SI, $0x00000800
|
|
JB repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short:
|
|
CMPL R10, $0x00000104
|
|
JB repeat_three_match_nolit_encodeBlockAsm4MB_emit_copy_short
|
|
CMPL R10, $0x00010100
|
|
JB repeat_four_match_nolit_encodeBlockAsm4MB_emit_copy_short
|
|
LEAL -65536(R10), R10
|
|
MOVL R10, SI
|
|
MOVW $0x001d, (CX)
|
|
MOVW R10, 2(CX)
|
|
SARL $0x10, SI
|
|
MOVB SI, 4(CX)
|
|
ADDQ $0x05, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm4MB
|
|
|
|
repeat_four_match_nolit_encodeBlockAsm4MB_emit_copy_short:
|
|
LEAL -256(R10), R10
|
|
MOVW $0x0019, (CX)
|
|
MOVW R10, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm4MB
|
|
|
|
repeat_three_match_nolit_encodeBlockAsm4MB_emit_copy_short:
|
|
LEAL -4(R10), R10
|
|
MOVW $0x0015, (CX)
|
|
MOVB R10, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm4MB
|
|
|
|
repeat_two_match_nolit_encodeBlockAsm4MB_emit_copy_short:
|
|
SHLL $0x02, R10
|
|
ORL $0x01, R10
|
|
MOVW R10, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm4MB
|
|
|
|
repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short:
|
|
XORQ DI, DI
|
|
LEAL 1(DI)(R10*4), R10
|
|
MOVB SI, 1(CX)
|
|
SARL $0x08, SI
|
|
SHLL $0x05, SI
|
|
ORL SI, R10
|
|
MOVB R10, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm4MB
|
|
|
|
two_byte_offset_short_match_nolit_encodeBlockAsm4MB:
|
|
MOVL R10, DI
|
|
SHLL $0x02, DI
|
|
CMPL R10, $0x0c
|
|
JAE emit_copy_three_match_nolit_encodeBlockAsm4MB
|
|
CMPL SI, $0x00000800
|
|
JAE emit_copy_three_match_nolit_encodeBlockAsm4MB
|
|
LEAL -15(DI), DI
|
|
MOVB SI, 1(CX)
|
|
SHRL $0x08, SI
|
|
SHLL $0x05, SI
|
|
ORL SI, DI
|
|
MOVB DI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm4MB
|
|
|
|
emit_copy_three_match_nolit_encodeBlockAsm4MB:
|
|
LEAL -2(DI), DI
|
|
MOVB DI, (CX)
|
|
MOVW SI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
|
|
match_nolit_emitcopy_end_encodeBlockAsm4MB:
|
|
CMPL DX, 8(SP)
|
|
JAE emit_remainder_encodeBlockAsm4MB
|
|
MOVQ -2(BX)(DX*1), DI
|
|
CMPQ CX, (SP)
|
|
JB match_nolit_dst_ok_encodeBlockAsm4MB
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeBlockAsm4MB:
|
|
MOVQ $0x0000cf1bbcdcbf9b, R9
|
|
MOVQ DI, R8
|
|
SHRQ $0x10, DI
|
|
MOVQ DI, SI
|
|
SHLQ $0x10, R8
|
|
IMULQ R9, R8
|
|
SHRQ $0x32, R8
|
|
SHLQ $0x10, SI
|
|
IMULQ R9, SI
|
|
SHRQ $0x32, SI
|
|
LEAL -2(DX), R9
|
|
LEAQ (AX)(SI*4), R10
|
|
MOVL (R10), SI
|
|
MOVL R9, (AX)(R8*4)
|
|
MOVL DX, (R10)
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ match_nolit_loop_encodeBlockAsm4MB
|
|
INCL DX
|
|
JMP search_loop_encodeBlockAsm4MB
|
|
|
|
emit_remainder_encodeBlockAsm4MB:
|
|
MOVQ src_len+32(FP), AX
|
|
SUBL 12(SP), AX
|
|
LEAQ 4(CX)(AX*1), AX
|
|
CMPQ AX, (SP)
|
|
JB emit_remainder_ok_encodeBlockAsm4MB
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeBlockAsm4MB:
|
|
MOVQ src_len+32(FP), AX
|
|
MOVL 12(SP), DX
|
|
CMPL DX, AX
|
|
JEQ emit_literal_done_emit_remainder_encodeBlockAsm4MB
|
|
MOVL AX, SI
|
|
MOVL AX, 12(SP)
|
|
LEAQ (BX)(DX*1), AX
|
|
SUBL DX, SI
|
|
LEAL -1(SI), DX
|
|
CMPL DX, $0x3c
|
|
JB one_byte_emit_remainder_encodeBlockAsm4MB
|
|
CMPL DX, $0x00000100
|
|
JB two_bytes_emit_remainder_encodeBlockAsm4MB
|
|
CMPL DX, $0x00010000
|
|
JB three_bytes_emit_remainder_encodeBlockAsm4MB
|
|
MOVL DX, BX
|
|
SHRL $0x10, BX
|
|
MOVB $0xf8, (CX)
|
|
MOVW DX, 1(CX)
|
|
MOVB BL, 3(CX)
|
|
ADDQ $0x04, CX
|
|
JMP memmove_long_emit_remainder_encodeBlockAsm4MB
|
|
|
|
three_bytes_emit_remainder_encodeBlockAsm4MB:
|
|
MOVB $0xf4, (CX)
|
|
MOVW DX, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_emit_remainder_encodeBlockAsm4MB
|
|
|
|
two_bytes_emit_remainder_encodeBlockAsm4MB:
|
|
MOVB $0xf0, (CX)
|
|
MOVB DL, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL DX, $0x40
|
|
JB memmove_emit_remainder_encodeBlockAsm4MB
|
|
JMP memmove_long_emit_remainder_encodeBlockAsm4MB
|
|
|
|
one_byte_emit_remainder_encodeBlockAsm4MB:
|
|
SHLB $0x02, DL
|
|
MOVB DL, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_emit_remainder_encodeBlockAsm4MB:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveShort
|
|
CMPQ BX, $0x03
|
|
JB emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_1or2
|
|
JE emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_3
|
|
CMPQ BX, $0x08
|
|
JB emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_4through7
|
|
CMPQ BX, $0x10
|
|
JBE emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_8through16
|
|
CMPQ BX, $0x20
|
|
JBE emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_1or2:
|
|
MOVB (AX), SI
|
|
MOVB -1(AX)(BX*1), AL
|
|
MOVB SI, (CX)
|
|
MOVB AL, -1(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm4MB
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_3:
|
|
MOVW (AX), SI
|
|
MOVB 2(AX), AL
|
|
MOVW SI, (CX)
|
|
MOVB AL, 2(CX)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm4MB
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_4through7:
|
|
MOVL (AX), SI
|
|
MOVL -4(AX)(BX*1), AX
|
|
MOVL SI, (CX)
|
|
MOVL AX, -4(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm4MB
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_8through16:
|
|
MOVQ (AX), SI
|
|
MOVQ -8(AX)(BX*1), AX
|
|
MOVQ SI, (CX)
|
|
MOVQ AX, -8(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm4MB
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_17through32:
|
|
MOVOU (AX), X0
|
|
MOVOU -16(AX)(BX*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm4MB
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_33through64:
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
|
|
memmove_end_copy_emit_remainder_encodeBlockAsm4MB:
|
|
MOVQ DX, CX
|
|
JMP emit_literal_done_emit_remainder_encodeBlockAsm4MB
|
|
|
|
memmove_long_emit_remainder_encodeBlockAsm4MB:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveLong
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVQ BX, DI
|
|
SHRQ $0x05, DI
|
|
MOVQ CX, SI
|
|
ANDL $0x0000001f, SI
|
|
MOVQ $0x00000040, R8
|
|
SUBQ SI, R8
|
|
DECQ DI
|
|
JA emit_lit_memmove_long_emit_remainder_encodeBlockAsm4MBlarge_forward_sse_loop_32
|
|
LEAQ -32(AX)(R8*1), SI
|
|
LEAQ -32(CX)(R8*1), R9
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeBlockAsm4MBlarge_big_loop_back:
|
|
MOVOU (SI), X4
|
|
MOVOU 16(SI), X5
|
|
MOVOA X4, (R9)
|
|
MOVOA X5, 16(R9)
|
|
ADDQ $0x20, R9
|
|
ADDQ $0x20, SI
|
|
ADDQ $0x20, R8
|
|
DECQ DI
|
|
JNA emit_lit_memmove_long_emit_remainder_encodeBlockAsm4MBlarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeBlockAsm4MBlarge_forward_sse_loop_32:
|
|
MOVOU -32(AX)(R8*1), X4
|
|
MOVOU -16(AX)(R8*1), X5
|
|
MOVOA X4, -32(CX)(R8*1)
|
|
MOVOA X5, -16(CX)(R8*1)
|
|
ADDQ $0x20, R8
|
|
CMPQ BX, R8
|
|
JAE emit_lit_memmove_long_emit_remainder_encodeBlockAsm4MBlarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
MOVQ DX, CX
|
|
|
|
emit_literal_done_emit_remainder_encodeBlockAsm4MB:
|
|
MOVQ dst_base+0(FP), AX
|
|
SUBQ AX, CX
|
|
MOVQ CX, ret+56(FP)
|
|
RET
|
|
|
|
// func encodeBlockAsm12B(dst []byte, src []byte, tmp *[16384]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeBlockAsm12B(SB), $24-64
|
|
MOVQ tmp+48(FP), AX
|
|
MOVQ dst_base+0(FP), CX
|
|
MOVQ $0x00000080, DX
|
|
MOVQ AX, BX
|
|
PXOR X0, X0
|
|
|
|
zero_loop_encodeBlockAsm12B:
|
|
MOVOU X0, (BX)
|
|
MOVOU X0, 16(BX)
|
|
MOVOU X0, 32(BX)
|
|
MOVOU X0, 48(BX)
|
|
MOVOU X0, 64(BX)
|
|
MOVOU X0, 80(BX)
|
|
MOVOU X0, 96(BX)
|
|
MOVOU X0, 112(BX)
|
|
ADDQ $0x80, BX
|
|
DECQ DX
|
|
JNZ zero_loop_encodeBlockAsm12B
|
|
MOVL $0x00000000, 12(SP)
|
|
MOVQ src_len+32(FP), DX
|
|
LEAQ -9(DX), BX
|
|
LEAQ -8(DX), SI
|
|
MOVL SI, 8(SP)
|
|
SHRQ $0x05, DX
|
|
SUBL DX, BX
|
|
LEAQ (CX)(BX*1), BX
|
|
MOVQ BX, (SP)
|
|
MOVL $0x00000001, DX
|
|
MOVL DX, 16(SP)
|
|
MOVQ src_base+24(FP), BX
|
|
|
|
search_loop_encodeBlockAsm12B:
|
|
MOVL DX, SI
|
|
SUBL 12(SP), SI
|
|
SHRL $0x05, SI
|
|
LEAL 4(DX)(SI*1), SI
|
|
CMPL SI, 8(SP)
|
|
JAE emit_remainder_encodeBlockAsm12B
|
|
MOVQ (BX)(DX*1), DI
|
|
MOVL SI, 20(SP)
|
|
MOVQ $0x000000cf1bbcdcbb, R9
|
|
MOVQ DI, R10
|
|
MOVQ DI, R11
|
|
SHRQ $0x08, R11
|
|
SHLQ $0x18, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x34, R10
|
|
SHLQ $0x18, R11
|
|
IMULQ R9, R11
|
|
SHRQ $0x34, R11
|
|
MOVL (AX)(R10*4), SI
|
|
MOVL (AX)(R11*4), R8
|
|
MOVL DX, (AX)(R10*4)
|
|
LEAL 1(DX), R10
|
|
MOVL R10, (AX)(R11*4)
|
|
MOVQ DI, R10
|
|
SHRQ $0x10, R10
|
|
SHLQ $0x18, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x34, R10
|
|
MOVL DX, R9
|
|
SUBL 16(SP), R9
|
|
MOVL 1(BX)(R9*1), R11
|
|
MOVQ DI, R9
|
|
SHRQ $0x08, R9
|
|
CMPL R9, R11
|
|
JNE no_repeat_found_encodeBlockAsm12B
|
|
LEAL 1(DX), DI
|
|
MOVL 12(SP), R8
|
|
MOVL DI, SI
|
|
SUBL 16(SP), SI
|
|
JZ repeat_extend_back_end_encodeBlockAsm12B
|
|
|
|
repeat_extend_back_loop_encodeBlockAsm12B:
|
|
CMPL DI, R8
|
|
JBE repeat_extend_back_end_encodeBlockAsm12B
|
|
MOVB -1(BX)(SI*1), R9
|
|
MOVB -1(BX)(DI*1), R10
|
|
CMPB R9, R10
|
|
JNE repeat_extend_back_end_encodeBlockAsm12B
|
|
LEAL -1(DI), DI
|
|
DECL SI
|
|
JNZ repeat_extend_back_loop_encodeBlockAsm12B
|
|
|
|
repeat_extend_back_end_encodeBlockAsm12B:
|
|
MOVL DI, SI
|
|
SUBL 12(SP), SI
|
|
LEAQ 3(CX)(SI*1), SI
|
|
CMPQ SI, (SP)
|
|
JB repeat_dst_size_check_encodeBlockAsm12B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
repeat_dst_size_check_encodeBlockAsm12B:
|
|
MOVL 12(SP), SI
|
|
CMPL SI, DI
|
|
JEQ emit_literal_done_repeat_emit_encodeBlockAsm12B
|
|
MOVL DI, R9
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(SI*1), R10
|
|
SUBL SI, R9
|
|
LEAL -1(R9), SI
|
|
CMPL SI, $0x3c
|
|
JB one_byte_repeat_emit_encodeBlockAsm12B
|
|
CMPL SI, $0x00000100
|
|
JB two_bytes_repeat_emit_encodeBlockAsm12B
|
|
JB three_bytes_repeat_emit_encodeBlockAsm12B
|
|
|
|
three_bytes_repeat_emit_encodeBlockAsm12B:
|
|
MOVB $0xf4, (CX)
|
|
MOVW SI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_repeat_emit_encodeBlockAsm12B
|
|
|
|
two_bytes_repeat_emit_encodeBlockAsm12B:
|
|
MOVB $0xf0, (CX)
|
|
MOVB SI, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL SI, $0x40
|
|
JB memmove_repeat_emit_encodeBlockAsm12B
|
|
JMP memmove_long_repeat_emit_encodeBlockAsm12B
|
|
|
|
one_byte_repeat_emit_encodeBlockAsm12B:
|
|
SHLB $0x02, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_repeat_emit_encodeBlockAsm12B:
|
|
LEAQ (CX)(R9*1), SI
|
|
|
|
// genMemMoveShort
|
|
CMPQ R9, $0x08
|
|
JBE emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_8
|
|
CMPQ R9, $0x10
|
|
JBE emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_8through16
|
|
CMPQ R9, $0x20
|
|
JBE emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_17through32
|
|
JMP emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_8:
|
|
MOVQ (R10), R11
|
|
MOVQ R11, (CX)
|
|
JMP memmove_end_copy_repeat_emit_encodeBlockAsm12B
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_8through16:
|
|
MOVQ (R10), R11
|
|
MOVQ -8(R10)(R9*1), R10
|
|
MOVQ R11, (CX)
|
|
MOVQ R10, -8(CX)(R9*1)
|
|
JMP memmove_end_copy_repeat_emit_encodeBlockAsm12B
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_17through32:
|
|
MOVOU (R10), X0
|
|
MOVOU -16(R10)(R9*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(R9*1)
|
|
JMP memmove_end_copy_repeat_emit_encodeBlockAsm12B
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_33through64:
|
|
MOVOU (R10), X0
|
|
MOVOU 16(R10), X1
|
|
MOVOU -32(R10)(R9*1), X2
|
|
MOVOU -16(R10)(R9*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
|
|
memmove_end_copy_repeat_emit_encodeBlockAsm12B:
|
|
MOVQ SI, CX
|
|
JMP emit_literal_done_repeat_emit_encodeBlockAsm12B
|
|
|
|
memmove_long_repeat_emit_encodeBlockAsm12B:
|
|
LEAQ (CX)(R9*1), SI
|
|
|
|
// genMemMoveLong
|
|
MOVOU (R10), X0
|
|
MOVOU 16(R10), X1
|
|
MOVOU -32(R10)(R9*1), X2
|
|
MOVOU -16(R10)(R9*1), X3
|
|
MOVQ R9, R12
|
|
SHRQ $0x05, R12
|
|
MOVQ CX, R11
|
|
ANDL $0x0000001f, R11
|
|
MOVQ $0x00000040, R13
|
|
SUBQ R11, R13
|
|
DECQ R12
|
|
JA emit_lit_memmove_long_repeat_emit_encodeBlockAsm12Blarge_forward_sse_loop_32
|
|
LEAQ -32(R10)(R13*1), R11
|
|
LEAQ -32(CX)(R13*1), R14
|
|
|
|
emit_lit_memmove_long_repeat_emit_encodeBlockAsm12Blarge_big_loop_back:
|
|
MOVOU (R11), X4
|
|
MOVOU 16(R11), X5
|
|
MOVOA X4, (R14)
|
|
MOVOA X5, 16(R14)
|
|
ADDQ $0x20, R14
|
|
ADDQ $0x20, R11
|
|
ADDQ $0x20, R13
|
|
DECQ R12
|
|
JNA emit_lit_memmove_long_repeat_emit_encodeBlockAsm12Blarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_repeat_emit_encodeBlockAsm12Blarge_forward_sse_loop_32:
|
|
MOVOU -32(R10)(R13*1), X4
|
|
MOVOU -16(R10)(R13*1), X5
|
|
MOVOA X4, -32(CX)(R13*1)
|
|
MOVOA X5, -16(CX)(R13*1)
|
|
ADDQ $0x20, R13
|
|
CMPQ R9, R13
|
|
JAE emit_lit_memmove_long_repeat_emit_encodeBlockAsm12Blarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
MOVQ SI, CX
|
|
|
|
emit_literal_done_repeat_emit_encodeBlockAsm12B:
|
|
ADDL $0x05, DX
|
|
MOVL DX, SI
|
|
SUBL 16(SP), SI
|
|
MOVQ src_len+32(FP), R9
|
|
SUBL DX, R9
|
|
LEAQ (BX)(DX*1), R10
|
|
LEAQ (BX)(SI*1), SI
|
|
|
|
// matchLen
|
|
XORL R12, R12
|
|
|
|
matchlen_loopback_16_repeat_extend_encodeBlockAsm12B:
|
|
CMPL R9, $0x10
|
|
JB matchlen_match8_repeat_extend_encodeBlockAsm12B
|
|
MOVQ (R10)(R12*1), R11
|
|
MOVQ 8(R10)(R12*1), R13
|
|
XORQ (SI)(R12*1), R11
|
|
JNZ matchlen_bsf_8_repeat_extend_encodeBlockAsm12B
|
|
XORQ 8(SI)(R12*1), R13
|
|
JNZ matchlen_bsf_16repeat_extend_encodeBlockAsm12B
|
|
LEAL -16(R9), R9
|
|
LEAL 16(R12), R12
|
|
JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm12B
|
|
|
|
matchlen_bsf_16repeat_extend_encodeBlockAsm12B:
|
|
TZCNTQ R13, R13
|
|
SARQ $0x03, R13
|
|
LEAL 8(R12)(R13*1), R12
|
|
JMP repeat_extend_forward_end_encodeBlockAsm12B
|
|
|
|
matchlen_match8_repeat_extend_encodeBlockAsm12B:
|
|
CMPL R9, $0x08
|
|
JB matchlen_match4_repeat_extend_encodeBlockAsm12B
|
|
MOVQ (R10)(R12*1), R11
|
|
XORQ (SI)(R12*1), R11
|
|
JNZ matchlen_bsf_8_repeat_extend_encodeBlockAsm12B
|
|
LEAL -8(R9), R9
|
|
LEAL 8(R12), R12
|
|
JMP matchlen_match4_repeat_extend_encodeBlockAsm12B
|
|
|
|
matchlen_bsf_8_repeat_extend_encodeBlockAsm12B:
|
|
TZCNTQ R11, R11
|
|
SARQ $0x03, R11
|
|
LEAL (R12)(R11*1), R12
|
|
JMP repeat_extend_forward_end_encodeBlockAsm12B
|
|
|
|
matchlen_match4_repeat_extend_encodeBlockAsm12B:
|
|
CMPL R9, $0x04
|
|
JB matchlen_match2_repeat_extend_encodeBlockAsm12B
|
|
MOVL (R10)(R12*1), R11
|
|
CMPL (SI)(R12*1), R11
|
|
JNE matchlen_match2_repeat_extend_encodeBlockAsm12B
|
|
LEAL -4(R9), R9
|
|
LEAL 4(R12), R12
|
|
|
|
matchlen_match2_repeat_extend_encodeBlockAsm12B:
|
|
CMPL R9, $0x01
|
|
JE matchlen_match1_repeat_extend_encodeBlockAsm12B
|
|
JB repeat_extend_forward_end_encodeBlockAsm12B
|
|
MOVW (R10)(R12*1), R11
|
|
CMPW (SI)(R12*1), R11
|
|
JNE matchlen_match1_repeat_extend_encodeBlockAsm12B
|
|
LEAL 2(R12), R12
|
|
SUBL $0x02, R9
|
|
JZ repeat_extend_forward_end_encodeBlockAsm12B
|
|
|
|
matchlen_match1_repeat_extend_encodeBlockAsm12B:
|
|
MOVB (R10)(R12*1), R11
|
|
CMPB (SI)(R12*1), R11
|
|
JNE repeat_extend_forward_end_encodeBlockAsm12B
|
|
LEAL 1(R12), R12
|
|
|
|
repeat_extend_forward_end_encodeBlockAsm12B:
|
|
ADDL R12, DX
|
|
MOVL DX, SI
|
|
SUBL DI, SI
|
|
MOVL 16(SP), DI
|
|
TESTL R8, R8
|
|
JZ repeat_as_copy_encodeBlockAsm12B
|
|
|
|
// emitRepeat
|
|
MOVL SI, R8
|
|
LEAL -4(SI), SI
|
|
CMPL R8, $0x08
|
|
JBE repeat_two_match_repeat_encodeBlockAsm12B
|
|
CMPL R8, $0x0c
|
|
JAE cant_repeat_two_offset_match_repeat_encodeBlockAsm12B
|
|
CMPL DI, $0x00000800
|
|
JB repeat_two_offset_match_repeat_encodeBlockAsm12B
|
|
|
|
cant_repeat_two_offset_match_repeat_encodeBlockAsm12B:
|
|
CMPL SI, $0x00000104
|
|
JB repeat_three_match_repeat_encodeBlockAsm12B
|
|
LEAL -256(SI), SI
|
|
MOVW $0x0019, (CX)
|
|
MOVW SI, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP repeat_end_emit_encodeBlockAsm12B
|
|
|
|
repeat_three_match_repeat_encodeBlockAsm12B:
|
|
LEAL -4(SI), SI
|
|
MOVW $0x0015, (CX)
|
|
MOVB SI, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP repeat_end_emit_encodeBlockAsm12B
|
|
|
|
repeat_two_match_repeat_encodeBlockAsm12B:
|
|
SHLL $0x02, SI
|
|
ORL $0x01, SI
|
|
MOVW SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm12B
|
|
|
|
repeat_two_offset_match_repeat_encodeBlockAsm12B:
|
|
XORQ R8, R8
|
|
LEAL 1(R8)(SI*4), SI
|
|
MOVB DI, 1(CX)
|
|
SARL $0x08, DI
|
|
SHLL $0x05, DI
|
|
ORL DI, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm12B
|
|
|
|
repeat_as_copy_encodeBlockAsm12B:
|
|
// emitCopy
|
|
CMPL SI, $0x40
|
|
JBE two_byte_offset_short_repeat_as_copy_encodeBlockAsm12B
|
|
CMPL DI, $0x00000800
|
|
JAE long_offset_short_repeat_as_copy_encodeBlockAsm12B
|
|
MOVL $0x00000001, R8
|
|
LEAL 16(R8), R8
|
|
MOVB DI, 1(CX)
|
|
SHRL $0x08, DI
|
|
SHLL $0x05, DI
|
|
ORL DI, R8
|
|
MOVB R8, (CX)
|
|
ADDQ $0x02, CX
|
|
SUBL $0x08, SI
|
|
|
|
// emitRepeat
|
|
LEAL -4(SI), SI
|
|
JMP cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b
|
|
MOVL SI, R8
|
|
LEAL -4(SI), SI
|
|
CMPL R8, $0x08
|
|
JBE repeat_two_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b
|
|
CMPL R8, $0x0c
|
|
JAE cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b
|
|
CMPL DI, $0x00000800
|
|
JB repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b
|
|
|
|
cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b:
|
|
CMPL SI, $0x00000104
|
|
JB repeat_three_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b
|
|
LEAL -256(SI), SI
|
|
MOVW $0x0019, (CX)
|
|
MOVW SI, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP repeat_end_emit_encodeBlockAsm12B
|
|
|
|
repeat_three_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b:
|
|
LEAL -4(SI), SI
|
|
MOVW $0x0015, (CX)
|
|
MOVB SI, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP repeat_end_emit_encodeBlockAsm12B
|
|
|
|
repeat_two_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b:
|
|
SHLL $0x02, SI
|
|
ORL $0x01, SI
|
|
MOVW SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm12B
|
|
|
|
repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b:
|
|
XORQ R8, R8
|
|
LEAL 1(R8)(SI*4), SI
|
|
MOVB DI, 1(CX)
|
|
SARL $0x08, DI
|
|
SHLL $0x05, DI
|
|
ORL DI, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm12B
|
|
|
|
long_offset_short_repeat_as_copy_encodeBlockAsm12B:
|
|
MOVB $0xee, (CX)
|
|
MOVW DI, 1(CX)
|
|
LEAL -60(SI), SI
|
|
ADDQ $0x03, CX
|
|
|
|
// emitRepeat
|
|
MOVL SI, R8
|
|
LEAL -4(SI), SI
|
|
CMPL R8, $0x08
|
|
JBE repeat_two_repeat_as_copy_encodeBlockAsm12B_emit_copy_short
|
|
CMPL R8, $0x0c
|
|
JAE cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short
|
|
CMPL DI, $0x00000800
|
|
JB repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short
|
|
|
|
cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short:
|
|
CMPL SI, $0x00000104
|
|
JB repeat_three_repeat_as_copy_encodeBlockAsm12B_emit_copy_short
|
|
LEAL -256(SI), SI
|
|
MOVW $0x0019, (CX)
|
|
MOVW SI, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP repeat_end_emit_encodeBlockAsm12B
|
|
|
|
repeat_three_repeat_as_copy_encodeBlockAsm12B_emit_copy_short:
|
|
LEAL -4(SI), SI
|
|
MOVW $0x0015, (CX)
|
|
MOVB SI, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP repeat_end_emit_encodeBlockAsm12B
|
|
|
|
repeat_two_repeat_as_copy_encodeBlockAsm12B_emit_copy_short:
|
|
SHLL $0x02, SI
|
|
ORL $0x01, SI
|
|
MOVW SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm12B
|
|
|
|
repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short:
|
|
XORQ R8, R8
|
|
LEAL 1(R8)(SI*4), SI
|
|
MOVB DI, 1(CX)
|
|
SARL $0x08, DI
|
|
SHLL $0x05, DI
|
|
ORL DI, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm12B
|
|
|
|
two_byte_offset_short_repeat_as_copy_encodeBlockAsm12B:
|
|
MOVL SI, R8
|
|
SHLL $0x02, R8
|
|
CMPL SI, $0x0c
|
|
JAE emit_copy_three_repeat_as_copy_encodeBlockAsm12B
|
|
CMPL DI, $0x00000800
|
|
JAE emit_copy_three_repeat_as_copy_encodeBlockAsm12B
|
|
LEAL -15(R8), R8
|
|
MOVB DI, 1(CX)
|
|
SHRL $0x08, DI
|
|
SHLL $0x05, DI
|
|
ORL DI, R8
|
|
MOVB R8, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm12B
|
|
|
|
emit_copy_three_repeat_as_copy_encodeBlockAsm12B:
|
|
LEAL -2(R8), R8
|
|
MOVB R8, (CX)
|
|
MOVW DI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
|
|
repeat_end_emit_encodeBlockAsm12B:
|
|
MOVL DX, 12(SP)
|
|
JMP search_loop_encodeBlockAsm12B
|
|
|
|
no_repeat_found_encodeBlockAsm12B:
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ candidate_match_encodeBlockAsm12B
|
|
SHRQ $0x08, DI
|
|
MOVL (AX)(R10*4), SI
|
|
LEAL 2(DX), R9
|
|
CMPL (BX)(R8*1), DI
|
|
JEQ candidate2_match_encodeBlockAsm12B
|
|
MOVL R9, (AX)(R10*4)
|
|
SHRQ $0x08, DI
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ candidate3_match_encodeBlockAsm12B
|
|
MOVL 20(SP), DX
|
|
JMP search_loop_encodeBlockAsm12B
|
|
|
|
candidate3_match_encodeBlockAsm12B:
|
|
ADDL $0x02, DX
|
|
JMP candidate_match_encodeBlockAsm12B
|
|
|
|
candidate2_match_encodeBlockAsm12B:
|
|
MOVL R9, (AX)(R10*4)
|
|
INCL DX
|
|
MOVL R8, SI
|
|
|
|
candidate_match_encodeBlockAsm12B:
|
|
MOVL 12(SP), DI
|
|
TESTL SI, SI
|
|
JZ match_extend_back_end_encodeBlockAsm12B
|
|
|
|
match_extend_back_loop_encodeBlockAsm12B:
|
|
CMPL DX, DI
|
|
JBE match_extend_back_end_encodeBlockAsm12B
|
|
MOVB -1(BX)(SI*1), R8
|
|
MOVB -1(BX)(DX*1), R9
|
|
CMPB R8, R9
|
|
JNE match_extend_back_end_encodeBlockAsm12B
|
|
LEAL -1(DX), DX
|
|
DECL SI
|
|
JZ match_extend_back_end_encodeBlockAsm12B
|
|
JMP match_extend_back_loop_encodeBlockAsm12B
|
|
|
|
match_extend_back_end_encodeBlockAsm12B:
|
|
MOVL DX, DI
|
|
SUBL 12(SP), DI
|
|
LEAQ 3(CX)(DI*1), DI
|
|
CMPQ DI, (SP)
|
|
JB match_dst_size_check_encodeBlockAsm12B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeBlockAsm12B:
|
|
MOVL DX, DI
|
|
MOVL 12(SP), R8
|
|
CMPL R8, DI
|
|
JEQ emit_literal_done_match_emit_encodeBlockAsm12B
|
|
MOVL DI, R9
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(R8*1), DI
|
|
SUBL R8, R9
|
|
LEAL -1(R9), R8
|
|
CMPL R8, $0x3c
|
|
JB one_byte_match_emit_encodeBlockAsm12B
|
|
CMPL R8, $0x00000100
|
|
JB two_bytes_match_emit_encodeBlockAsm12B
|
|
JB three_bytes_match_emit_encodeBlockAsm12B
|
|
|
|
three_bytes_match_emit_encodeBlockAsm12B:
|
|
MOVB $0xf4, (CX)
|
|
MOVW R8, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_match_emit_encodeBlockAsm12B
|
|
|
|
two_bytes_match_emit_encodeBlockAsm12B:
|
|
MOVB $0xf0, (CX)
|
|
MOVB R8, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL R8, $0x40
|
|
JB memmove_match_emit_encodeBlockAsm12B
|
|
JMP memmove_long_match_emit_encodeBlockAsm12B
|
|
|
|
one_byte_match_emit_encodeBlockAsm12B:
|
|
SHLB $0x02, R8
|
|
MOVB R8, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_match_emit_encodeBlockAsm12B:
|
|
LEAQ (CX)(R9*1), R8
|
|
|
|
// genMemMoveShort
|
|
CMPQ R9, $0x08
|
|
JBE emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_8
|
|
CMPQ R9, $0x10
|
|
JBE emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_8through16
|
|
CMPQ R9, $0x20
|
|
JBE emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_8:
|
|
MOVQ (DI), R10
|
|
MOVQ R10, (CX)
|
|
JMP memmove_end_copy_match_emit_encodeBlockAsm12B
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_8through16:
|
|
MOVQ (DI), R10
|
|
MOVQ -8(DI)(R9*1), DI
|
|
MOVQ R10, (CX)
|
|
MOVQ DI, -8(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeBlockAsm12B
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_17through32:
|
|
MOVOU (DI), X0
|
|
MOVOU -16(DI)(R9*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeBlockAsm12B
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_33through64:
|
|
MOVOU (DI), X0
|
|
MOVOU 16(DI), X1
|
|
MOVOU -32(DI)(R9*1), X2
|
|
MOVOU -16(DI)(R9*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
|
|
memmove_end_copy_match_emit_encodeBlockAsm12B:
|
|
MOVQ R8, CX
|
|
JMP emit_literal_done_match_emit_encodeBlockAsm12B
|
|
|
|
memmove_long_match_emit_encodeBlockAsm12B:
|
|
LEAQ (CX)(R9*1), R8
|
|
|
|
// genMemMoveLong
|
|
MOVOU (DI), X0
|
|
MOVOU 16(DI), X1
|
|
MOVOU -32(DI)(R9*1), X2
|
|
MOVOU -16(DI)(R9*1), X3
|
|
MOVQ R9, R11
|
|
SHRQ $0x05, R11
|
|
MOVQ CX, R10
|
|
ANDL $0x0000001f, R10
|
|
MOVQ $0x00000040, R12
|
|
SUBQ R10, R12
|
|
DECQ R11
|
|
JA emit_lit_memmove_long_match_emit_encodeBlockAsm12Blarge_forward_sse_loop_32
|
|
LEAQ -32(DI)(R12*1), R10
|
|
LEAQ -32(CX)(R12*1), R13
|
|
|
|
emit_lit_memmove_long_match_emit_encodeBlockAsm12Blarge_big_loop_back:
|
|
MOVOU (R10), X4
|
|
MOVOU 16(R10), X5
|
|
MOVOA X4, (R13)
|
|
MOVOA X5, 16(R13)
|
|
ADDQ $0x20, R13
|
|
ADDQ $0x20, R10
|
|
ADDQ $0x20, R12
|
|
DECQ R11
|
|
JNA emit_lit_memmove_long_match_emit_encodeBlockAsm12Blarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_match_emit_encodeBlockAsm12Blarge_forward_sse_loop_32:
|
|
MOVOU -32(DI)(R12*1), X4
|
|
MOVOU -16(DI)(R12*1), X5
|
|
MOVOA X4, -32(CX)(R12*1)
|
|
MOVOA X5, -16(CX)(R12*1)
|
|
ADDQ $0x20, R12
|
|
CMPQ R9, R12
|
|
JAE emit_lit_memmove_long_match_emit_encodeBlockAsm12Blarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
MOVQ R8, CX
|
|
|
|
emit_literal_done_match_emit_encodeBlockAsm12B:
|
|
match_nolit_loop_encodeBlockAsm12B:
|
|
MOVL DX, DI
|
|
SUBL SI, DI
|
|
MOVL DI, 16(SP)
|
|
ADDL $0x04, DX
|
|
ADDL $0x04, SI
|
|
MOVQ src_len+32(FP), DI
|
|
SUBL DX, DI
|
|
LEAQ (BX)(DX*1), R8
|
|
LEAQ (BX)(SI*1), SI
|
|
|
|
// matchLen
|
|
XORL R10, R10
|
|
|
|
matchlen_loopback_16_match_nolit_encodeBlockAsm12B:
|
|
CMPL DI, $0x10
|
|
JB matchlen_match8_match_nolit_encodeBlockAsm12B
|
|
MOVQ (R8)(R10*1), R9
|
|
MOVQ 8(R8)(R10*1), R11
|
|
XORQ (SI)(R10*1), R9
|
|
JNZ matchlen_bsf_8_match_nolit_encodeBlockAsm12B
|
|
XORQ 8(SI)(R10*1), R11
|
|
JNZ matchlen_bsf_16match_nolit_encodeBlockAsm12B
|
|
LEAL -16(DI), DI
|
|
LEAL 16(R10), R10
|
|
JMP matchlen_loopback_16_match_nolit_encodeBlockAsm12B
|
|
|
|
matchlen_bsf_16match_nolit_encodeBlockAsm12B:
|
|
TZCNTQ R11, R11
|
|
SARQ $0x03, R11
|
|
LEAL 8(R10)(R11*1), R10
|
|
JMP match_nolit_end_encodeBlockAsm12B
|
|
|
|
matchlen_match8_match_nolit_encodeBlockAsm12B:
|
|
CMPL DI, $0x08
|
|
JB matchlen_match4_match_nolit_encodeBlockAsm12B
|
|
MOVQ (R8)(R10*1), R9
|
|
XORQ (SI)(R10*1), R9
|
|
JNZ matchlen_bsf_8_match_nolit_encodeBlockAsm12B
|
|
LEAL -8(DI), DI
|
|
LEAL 8(R10), R10
|
|
JMP matchlen_match4_match_nolit_encodeBlockAsm12B
|
|
|
|
matchlen_bsf_8_match_nolit_encodeBlockAsm12B:
|
|
TZCNTQ R9, R9
|
|
SARQ $0x03, R9
|
|
LEAL (R10)(R9*1), R10
|
|
JMP match_nolit_end_encodeBlockAsm12B
|
|
|
|
matchlen_match4_match_nolit_encodeBlockAsm12B:
|
|
CMPL DI, $0x04
|
|
JB matchlen_match2_match_nolit_encodeBlockAsm12B
|
|
MOVL (R8)(R10*1), R9
|
|
CMPL (SI)(R10*1), R9
|
|
JNE matchlen_match2_match_nolit_encodeBlockAsm12B
|
|
LEAL -4(DI), DI
|
|
LEAL 4(R10), R10
|
|
|
|
matchlen_match2_match_nolit_encodeBlockAsm12B:
|
|
CMPL DI, $0x01
|
|
JE matchlen_match1_match_nolit_encodeBlockAsm12B
|
|
JB match_nolit_end_encodeBlockAsm12B
|
|
MOVW (R8)(R10*1), R9
|
|
CMPW (SI)(R10*1), R9
|
|
JNE matchlen_match1_match_nolit_encodeBlockAsm12B
|
|
LEAL 2(R10), R10
|
|
SUBL $0x02, DI
|
|
JZ match_nolit_end_encodeBlockAsm12B
|
|
|
|
matchlen_match1_match_nolit_encodeBlockAsm12B:
|
|
MOVB (R8)(R10*1), R9
|
|
CMPB (SI)(R10*1), R9
|
|
JNE match_nolit_end_encodeBlockAsm12B
|
|
LEAL 1(R10), R10
|
|
|
|
match_nolit_end_encodeBlockAsm12B:
|
|
ADDL R10, DX
|
|
MOVL 16(SP), SI
|
|
ADDL $0x04, R10
|
|
MOVL DX, 12(SP)
|
|
|
|
// emitCopy
|
|
CMPL R10, $0x40
|
|
JBE two_byte_offset_short_match_nolit_encodeBlockAsm12B
|
|
CMPL SI, $0x00000800
|
|
JAE long_offset_short_match_nolit_encodeBlockAsm12B
|
|
MOVL $0x00000001, DI
|
|
LEAL 16(DI), DI
|
|
MOVB SI, 1(CX)
|
|
SHRL $0x08, SI
|
|
SHLL $0x05, SI
|
|
ORL SI, DI
|
|
MOVB DI, (CX)
|
|
ADDQ $0x02, CX
|
|
SUBL $0x08, R10
|
|
|
|
// emitRepeat
|
|
LEAL -4(R10), R10
|
|
JMP cant_repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short_2b
|
|
MOVL R10, DI
|
|
LEAL -4(R10), R10
|
|
CMPL DI, $0x08
|
|
JBE repeat_two_match_nolit_encodeBlockAsm12B_emit_copy_short_2b
|
|
CMPL DI, $0x0c
|
|
JAE cant_repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short_2b
|
|
CMPL SI, $0x00000800
|
|
JB repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short_2b
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short_2b:
|
|
CMPL R10, $0x00000104
|
|
JB repeat_three_match_nolit_encodeBlockAsm12B_emit_copy_short_2b
|
|
LEAL -256(R10), R10
|
|
MOVW $0x0019, (CX)
|
|
MOVW R10, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm12B
|
|
|
|
repeat_three_match_nolit_encodeBlockAsm12B_emit_copy_short_2b:
|
|
LEAL -4(R10), R10
|
|
MOVW $0x0015, (CX)
|
|
MOVB R10, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm12B
|
|
|
|
repeat_two_match_nolit_encodeBlockAsm12B_emit_copy_short_2b:
|
|
SHLL $0x02, R10
|
|
ORL $0x01, R10
|
|
MOVW R10, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm12B
|
|
|
|
repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short_2b:
|
|
XORQ DI, DI
|
|
LEAL 1(DI)(R10*4), R10
|
|
MOVB SI, 1(CX)
|
|
SARL $0x08, SI
|
|
SHLL $0x05, SI
|
|
ORL SI, R10
|
|
MOVB R10, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm12B
|
|
|
|
long_offset_short_match_nolit_encodeBlockAsm12B:
|
|
MOVB $0xee, (CX)
|
|
MOVW SI, 1(CX)
|
|
LEAL -60(R10), R10
|
|
ADDQ $0x03, CX
|
|
|
|
// emitRepeat
|
|
MOVL R10, DI
|
|
LEAL -4(R10), R10
|
|
CMPL DI, $0x08
|
|
JBE repeat_two_match_nolit_encodeBlockAsm12B_emit_copy_short
|
|
CMPL DI, $0x0c
|
|
JAE cant_repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short
|
|
CMPL SI, $0x00000800
|
|
JB repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short:
|
|
CMPL R10, $0x00000104
|
|
JB repeat_three_match_nolit_encodeBlockAsm12B_emit_copy_short
|
|
LEAL -256(R10), R10
|
|
MOVW $0x0019, (CX)
|
|
MOVW R10, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm12B
|
|
|
|
repeat_three_match_nolit_encodeBlockAsm12B_emit_copy_short:
|
|
LEAL -4(R10), R10
|
|
MOVW $0x0015, (CX)
|
|
MOVB R10, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm12B
|
|
|
|
repeat_two_match_nolit_encodeBlockAsm12B_emit_copy_short:
|
|
SHLL $0x02, R10
|
|
ORL $0x01, R10
|
|
MOVW R10, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm12B
|
|
|
|
repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short:
|
|
XORQ DI, DI
|
|
LEAL 1(DI)(R10*4), R10
|
|
MOVB SI, 1(CX)
|
|
SARL $0x08, SI
|
|
SHLL $0x05, SI
|
|
ORL SI, R10
|
|
MOVB R10, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm12B
|
|
|
|
two_byte_offset_short_match_nolit_encodeBlockAsm12B:
|
|
MOVL R10, DI
|
|
SHLL $0x02, DI
|
|
CMPL R10, $0x0c
|
|
JAE emit_copy_three_match_nolit_encodeBlockAsm12B
|
|
CMPL SI, $0x00000800
|
|
JAE emit_copy_three_match_nolit_encodeBlockAsm12B
|
|
LEAL -15(DI), DI
|
|
MOVB SI, 1(CX)
|
|
SHRL $0x08, SI
|
|
SHLL $0x05, SI
|
|
ORL SI, DI
|
|
MOVB DI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm12B
|
|
|
|
emit_copy_three_match_nolit_encodeBlockAsm12B:
|
|
LEAL -2(DI), DI
|
|
MOVB DI, (CX)
|
|
MOVW SI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
|
|
match_nolit_emitcopy_end_encodeBlockAsm12B:
|
|
CMPL DX, 8(SP)
|
|
JAE emit_remainder_encodeBlockAsm12B
|
|
MOVQ -2(BX)(DX*1), DI
|
|
CMPQ CX, (SP)
|
|
JB match_nolit_dst_ok_encodeBlockAsm12B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeBlockAsm12B:
|
|
MOVQ $0x000000cf1bbcdcbb, R9
|
|
MOVQ DI, R8
|
|
SHRQ $0x10, DI
|
|
MOVQ DI, SI
|
|
SHLQ $0x18, R8
|
|
IMULQ R9, R8
|
|
SHRQ $0x34, R8
|
|
SHLQ $0x18, SI
|
|
IMULQ R9, SI
|
|
SHRQ $0x34, SI
|
|
LEAL -2(DX), R9
|
|
LEAQ (AX)(SI*4), R10
|
|
MOVL (R10), SI
|
|
MOVL R9, (AX)(R8*4)
|
|
MOVL DX, (R10)
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ match_nolit_loop_encodeBlockAsm12B
|
|
INCL DX
|
|
JMP search_loop_encodeBlockAsm12B
|
|
|
|
emit_remainder_encodeBlockAsm12B:
|
|
MOVQ src_len+32(FP), AX
|
|
SUBL 12(SP), AX
|
|
LEAQ 3(CX)(AX*1), AX
|
|
CMPQ AX, (SP)
|
|
JB emit_remainder_ok_encodeBlockAsm12B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeBlockAsm12B:
|
|
MOVQ src_len+32(FP), AX
|
|
MOVL 12(SP), DX
|
|
CMPL DX, AX
|
|
JEQ emit_literal_done_emit_remainder_encodeBlockAsm12B
|
|
MOVL AX, SI
|
|
MOVL AX, 12(SP)
|
|
LEAQ (BX)(DX*1), AX
|
|
SUBL DX, SI
|
|
LEAL -1(SI), DX
|
|
CMPL DX, $0x3c
|
|
JB one_byte_emit_remainder_encodeBlockAsm12B
|
|
CMPL DX, $0x00000100
|
|
JB two_bytes_emit_remainder_encodeBlockAsm12B
|
|
JB three_bytes_emit_remainder_encodeBlockAsm12B
|
|
|
|
three_bytes_emit_remainder_encodeBlockAsm12B:
|
|
MOVB $0xf4, (CX)
|
|
MOVW DX, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_emit_remainder_encodeBlockAsm12B
|
|
|
|
two_bytes_emit_remainder_encodeBlockAsm12B:
|
|
MOVB $0xf0, (CX)
|
|
MOVB DL, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL DX, $0x40
|
|
JB memmove_emit_remainder_encodeBlockAsm12B
|
|
JMP memmove_long_emit_remainder_encodeBlockAsm12B
|
|
|
|
one_byte_emit_remainder_encodeBlockAsm12B:
|
|
SHLB $0x02, DL
|
|
MOVB DL, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_emit_remainder_encodeBlockAsm12B:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveShort
|
|
CMPQ BX, $0x03
|
|
JB emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_1or2
|
|
JE emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_3
|
|
CMPQ BX, $0x08
|
|
JB emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_4through7
|
|
CMPQ BX, $0x10
|
|
JBE emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_8through16
|
|
CMPQ BX, $0x20
|
|
JBE emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_1or2:
|
|
MOVB (AX), SI
|
|
MOVB -1(AX)(BX*1), AL
|
|
MOVB SI, (CX)
|
|
MOVB AL, -1(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_3:
|
|
MOVW (AX), SI
|
|
MOVB 2(AX), AL
|
|
MOVW SI, (CX)
|
|
MOVB AL, 2(CX)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_4through7:
|
|
MOVL (AX), SI
|
|
MOVL -4(AX)(BX*1), AX
|
|
MOVL SI, (CX)
|
|
MOVL AX, -4(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_8through16:
|
|
MOVQ (AX), SI
|
|
MOVQ -8(AX)(BX*1), AX
|
|
MOVQ SI, (CX)
|
|
MOVQ AX, -8(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_17through32:
|
|
MOVOU (AX), X0
|
|
MOVOU -16(AX)(BX*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_33through64:
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
|
|
memmove_end_copy_emit_remainder_encodeBlockAsm12B:
|
|
MOVQ DX, CX
|
|
JMP emit_literal_done_emit_remainder_encodeBlockAsm12B
|
|
|
|
memmove_long_emit_remainder_encodeBlockAsm12B:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveLong
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVQ BX, DI
|
|
SHRQ $0x05, DI
|
|
MOVQ CX, SI
|
|
ANDL $0x0000001f, SI
|
|
MOVQ $0x00000040, R8
|
|
SUBQ SI, R8
|
|
DECQ DI
|
|
JA emit_lit_memmove_long_emit_remainder_encodeBlockAsm12Blarge_forward_sse_loop_32
|
|
LEAQ -32(AX)(R8*1), SI
|
|
LEAQ -32(CX)(R8*1), R9
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeBlockAsm12Blarge_big_loop_back:
|
|
MOVOU (SI), X4
|
|
MOVOU 16(SI), X5
|
|
MOVOA X4, (R9)
|
|
MOVOA X5, 16(R9)
|
|
ADDQ $0x20, R9
|
|
ADDQ $0x20, SI
|
|
ADDQ $0x20, R8
|
|
DECQ DI
|
|
JNA emit_lit_memmove_long_emit_remainder_encodeBlockAsm12Blarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeBlockAsm12Blarge_forward_sse_loop_32:
|
|
MOVOU -32(AX)(R8*1), X4
|
|
MOVOU -16(AX)(R8*1), X5
|
|
MOVOA X4, -32(CX)(R8*1)
|
|
MOVOA X5, -16(CX)(R8*1)
|
|
ADDQ $0x20, R8
|
|
CMPQ BX, R8
|
|
JAE emit_lit_memmove_long_emit_remainder_encodeBlockAsm12Blarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
MOVQ DX, CX
|
|
|
|
emit_literal_done_emit_remainder_encodeBlockAsm12B:
|
|
MOVQ dst_base+0(FP), AX
|
|
SUBQ AX, CX
|
|
MOVQ CX, ret+56(FP)
|
|
RET
|
|
|
|
// func encodeBlockAsm10B(dst []byte, src []byte, tmp *[4096]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeBlockAsm10B(SB), $24-64
|
|
MOVQ tmp+48(FP), AX
|
|
MOVQ dst_base+0(FP), CX
|
|
MOVQ $0x00000020, DX
|
|
MOVQ AX, BX
|
|
PXOR X0, X0
|
|
|
|
zero_loop_encodeBlockAsm10B:
|
|
MOVOU X0, (BX)
|
|
MOVOU X0, 16(BX)
|
|
MOVOU X0, 32(BX)
|
|
MOVOU X0, 48(BX)
|
|
MOVOU X0, 64(BX)
|
|
MOVOU X0, 80(BX)
|
|
MOVOU X0, 96(BX)
|
|
MOVOU X0, 112(BX)
|
|
ADDQ $0x80, BX
|
|
DECQ DX
|
|
JNZ zero_loop_encodeBlockAsm10B
|
|
MOVL $0x00000000, 12(SP)
|
|
MOVQ src_len+32(FP), DX
|
|
LEAQ -9(DX), BX
|
|
LEAQ -8(DX), SI
|
|
MOVL SI, 8(SP)
|
|
SHRQ $0x05, DX
|
|
SUBL DX, BX
|
|
LEAQ (CX)(BX*1), BX
|
|
MOVQ BX, (SP)
|
|
MOVL $0x00000001, DX
|
|
MOVL DX, 16(SP)
|
|
MOVQ src_base+24(FP), BX
|
|
|
|
search_loop_encodeBlockAsm10B:
|
|
MOVL DX, SI
|
|
SUBL 12(SP), SI
|
|
SHRL $0x05, SI
|
|
LEAL 4(DX)(SI*1), SI
|
|
CMPL SI, 8(SP)
|
|
JAE emit_remainder_encodeBlockAsm10B
|
|
MOVQ (BX)(DX*1), DI
|
|
MOVL SI, 20(SP)
|
|
MOVQ $0x9e3779b1, R9
|
|
MOVQ DI, R10
|
|
MOVQ DI, R11
|
|
SHRQ $0x08, R11
|
|
SHLQ $0x20, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x36, R10
|
|
SHLQ $0x20, R11
|
|
IMULQ R9, R11
|
|
SHRQ $0x36, R11
|
|
MOVL (AX)(R10*4), SI
|
|
MOVL (AX)(R11*4), R8
|
|
MOVL DX, (AX)(R10*4)
|
|
LEAL 1(DX), R10
|
|
MOVL R10, (AX)(R11*4)
|
|
MOVQ DI, R10
|
|
SHRQ $0x10, R10
|
|
SHLQ $0x20, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x36, R10
|
|
MOVL DX, R9
|
|
SUBL 16(SP), R9
|
|
MOVL 1(BX)(R9*1), R11
|
|
MOVQ DI, R9
|
|
SHRQ $0x08, R9
|
|
CMPL R9, R11
|
|
JNE no_repeat_found_encodeBlockAsm10B
|
|
LEAL 1(DX), DI
|
|
MOVL 12(SP), R8
|
|
MOVL DI, SI
|
|
SUBL 16(SP), SI
|
|
JZ repeat_extend_back_end_encodeBlockAsm10B
|
|
|
|
repeat_extend_back_loop_encodeBlockAsm10B:
|
|
CMPL DI, R8
|
|
JBE repeat_extend_back_end_encodeBlockAsm10B
|
|
MOVB -1(BX)(SI*1), R9
|
|
MOVB -1(BX)(DI*1), R10
|
|
CMPB R9, R10
|
|
JNE repeat_extend_back_end_encodeBlockAsm10B
|
|
LEAL -1(DI), DI
|
|
DECL SI
|
|
JNZ repeat_extend_back_loop_encodeBlockAsm10B
|
|
|
|
repeat_extend_back_end_encodeBlockAsm10B:
|
|
MOVL DI, SI
|
|
SUBL 12(SP), SI
|
|
LEAQ 3(CX)(SI*1), SI
|
|
CMPQ SI, (SP)
|
|
JB repeat_dst_size_check_encodeBlockAsm10B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
repeat_dst_size_check_encodeBlockAsm10B:
|
|
MOVL 12(SP), SI
|
|
CMPL SI, DI
|
|
JEQ emit_literal_done_repeat_emit_encodeBlockAsm10B
|
|
MOVL DI, R9
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(SI*1), R10
|
|
SUBL SI, R9
|
|
LEAL -1(R9), SI
|
|
CMPL SI, $0x3c
|
|
JB one_byte_repeat_emit_encodeBlockAsm10B
|
|
CMPL SI, $0x00000100
|
|
JB two_bytes_repeat_emit_encodeBlockAsm10B
|
|
JB three_bytes_repeat_emit_encodeBlockAsm10B
|
|
|
|
three_bytes_repeat_emit_encodeBlockAsm10B:
|
|
MOVB $0xf4, (CX)
|
|
MOVW SI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_repeat_emit_encodeBlockAsm10B
|
|
|
|
two_bytes_repeat_emit_encodeBlockAsm10B:
|
|
MOVB $0xf0, (CX)
|
|
MOVB SI, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL SI, $0x40
|
|
JB memmove_repeat_emit_encodeBlockAsm10B
|
|
JMP memmove_long_repeat_emit_encodeBlockAsm10B
|
|
|
|
one_byte_repeat_emit_encodeBlockAsm10B:
|
|
SHLB $0x02, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_repeat_emit_encodeBlockAsm10B:
|
|
LEAQ (CX)(R9*1), SI
|
|
|
|
// genMemMoveShort
|
|
CMPQ R9, $0x08
|
|
JBE emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_8
|
|
CMPQ R9, $0x10
|
|
JBE emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_8through16
|
|
CMPQ R9, $0x20
|
|
JBE emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_17through32
|
|
JMP emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_8:
|
|
MOVQ (R10), R11
|
|
MOVQ R11, (CX)
|
|
JMP memmove_end_copy_repeat_emit_encodeBlockAsm10B
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_8through16:
|
|
MOVQ (R10), R11
|
|
MOVQ -8(R10)(R9*1), R10
|
|
MOVQ R11, (CX)
|
|
MOVQ R10, -8(CX)(R9*1)
|
|
JMP memmove_end_copy_repeat_emit_encodeBlockAsm10B
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_17through32:
|
|
MOVOU (R10), X0
|
|
MOVOU -16(R10)(R9*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(R9*1)
|
|
JMP memmove_end_copy_repeat_emit_encodeBlockAsm10B
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_33through64:
|
|
MOVOU (R10), X0
|
|
MOVOU 16(R10), X1
|
|
MOVOU -32(R10)(R9*1), X2
|
|
MOVOU -16(R10)(R9*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
|
|
memmove_end_copy_repeat_emit_encodeBlockAsm10B:
|
|
MOVQ SI, CX
|
|
JMP emit_literal_done_repeat_emit_encodeBlockAsm10B
|
|
|
|
memmove_long_repeat_emit_encodeBlockAsm10B:
|
|
LEAQ (CX)(R9*1), SI
|
|
|
|
// genMemMoveLong
|
|
MOVOU (R10), X0
|
|
MOVOU 16(R10), X1
|
|
MOVOU -32(R10)(R9*1), X2
|
|
MOVOU -16(R10)(R9*1), X3
|
|
MOVQ R9, R12
|
|
SHRQ $0x05, R12
|
|
MOVQ CX, R11
|
|
ANDL $0x0000001f, R11
|
|
MOVQ $0x00000040, R13
|
|
SUBQ R11, R13
|
|
DECQ R12
|
|
JA emit_lit_memmove_long_repeat_emit_encodeBlockAsm10Blarge_forward_sse_loop_32
|
|
LEAQ -32(R10)(R13*1), R11
|
|
LEAQ -32(CX)(R13*1), R14
|
|
|
|
emit_lit_memmove_long_repeat_emit_encodeBlockAsm10Blarge_big_loop_back:
|
|
MOVOU (R11), X4
|
|
MOVOU 16(R11), X5
|
|
MOVOA X4, (R14)
|
|
MOVOA X5, 16(R14)
|
|
ADDQ $0x20, R14
|
|
ADDQ $0x20, R11
|
|
ADDQ $0x20, R13
|
|
DECQ R12
|
|
JNA emit_lit_memmove_long_repeat_emit_encodeBlockAsm10Blarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_repeat_emit_encodeBlockAsm10Blarge_forward_sse_loop_32:
|
|
MOVOU -32(R10)(R13*1), X4
|
|
MOVOU -16(R10)(R13*1), X5
|
|
MOVOA X4, -32(CX)(R13*1)
|
|
MOVOA X5, -16(CX)(R13*1)
|
|
ADDQ $0x20, R13
|
|
CMPQ R9, R13
|
|
JAE emit_lit_memmove_long_repeat_emit_encodeBlockAsm10Blarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
MOVQ SI, CX
|
|
|
|
emit_literal_done_repeat_emit_encodeBlockAsm10B:
|
|
ADDL $0x05, DX
|
|
MOVL DX, SI
|
|
SUBL 16(SP), SI
|
|
MOVQ src_len+32(FP), R9
|
|
SUBL DX, R9
|
|
LEAQ (BX)(DX*1), R10
|
|
LEAQ (BX)(SI*1), SI
|
|
|
|
// matchLen
|
|
XORL R12, R12
|
|
|
|
matchlen_loopback_16_repeat_extend_encodeBlockAsm10B:
|
|
CMPL R9, $0x10
|
|
JB matchlen_match8_repeat_extend_encodeBlockAsm10B
|
|
MOVQ (R10)(R12*1), R11
|
|
MOVQ 8(R10)(R12*1), R13
|
|
XORQ (SI)(R12*1), R11
|
|
JNZ matchlen_bsf_8_repeat_extend_encodeBlockAsm10B
|
|
XORQ 8(SI)(R12*1), R13
|
|
JNZ matchlen_bsf_16repeat_extend_encodeBlockAsm10B
|
|
LEAL -16(R9), R9
|
|
LEAL 16(R12), R12
|
|
JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm10B
|
|
|
|
matchlen_bsf_16repeat_extend_encodeBlockAsm10B:
|
|
TZCNTQ R13, R13
|
|
SARQ $0x03, R13
|
|
LEAL 8(R12)(R13*1), R12
|
|
JMP repeat_extend_forward_end_encodeBlockAsm10B
|
|
|
|
matchlen_match8_repeat_extend_encodeBlockAsm10B:
|
|
CMPL R9, $0x08
|
|
JB matchlen_match4_repeat_extend_encodeBlockAsm10B
|
|
MOVQ (R10)(R12*1), R11
|
|
XORQ (SI)(R12*1), R11
|
|
JNZ matchlen_bsf_8_repeat_extend_encodeBlockAsm10B
|
|
LEAL -8(R9), R9
|
|
LEAL 8(R12), R12
|
|
JMP matchlen_match4_repeat_extend_encodeBlockAsm10B
|
|
|
|
matchlen_bsf_8_repeat_extend_encodeBlockAsm10B:
|
|
TZCNTQ R11, R11
|
|
SARQ $0x03, R11
|
|
LEAL (R12)(R11*1), R12
|
|
JMP repeat_extend_forward_end_encodeBlockAsm10B
|
|
|
|
matchlen_match4_repeat_extend_encodeBlockAsm10B:
|
|
CMPL R9, $0x04
|
|
JB matchlen_match2_repeat_extend_encodeBlockAsm10B
|
|
MOVL (R10)(R12*1), R11
|
|
CMPL (SI)(R12*1), R11
|
|
JNE matchlen_match2_repeat_extend_encodeBlockAsm10B
|
|
LEAL -4(R9), R9
|
|
LEAL 4(R12), R12
|
|
|
|
matchlen_match2_repeat_extend_encodeBlockAsm10B:
|
|
CMPL R9, $0x01
|
|
JE matchlen_match1_repeat_extend_encodeBlockAsm10B
|
|
JB repeat_extend_forward_end_encodeBlockAsm10B
|
|
MOVW (R10)(R12*1), R11
|
|
CMPW (SI)(R12*1), R11
|
|
JNE matchlen_match1_repeat_extend_encodeBlockAsm10B
|
|
LEAL 2(R12), R12
|
|
SUBL $0x02, R9
|
|
JZ repeat_extend_forward_end_encodeBlockAsm10B
|
|
|
|
matchlen_match1_repeat_extend_encodeBlockAsm10B:
|
|
MOVB (R10)(R12*1), R11
|
|
CMPB (SI)(R12*1), R11
|
|
JNE repeat_extend_forward_end_encodeBlockAsm10B
|
|
LEAL 1(R12), R12
|
|
|
|
repeat_extend_forward_end_encodeBlockAsm10B:
|
|
ADDL R12, DX
|
|
MOVL DX, SI
|
|
SUBL DI, SI
|
|
MOVL 16(SP), DI
|
|
TESTL R8, R8
|
|
JZ repeat_as_copy_encodeBlockAsm10B
|
|
|
|
// emitRepeat
|
|
MOVL SI, R8
|
|
LEAL -4(SI), SI
|
|
CMPL R8, $0x08
|
|
JBE repeat_two_match_repeat_encodeBlockAsm10B
|
|
CMPL R8, $0x0c
|
|
JAE cant_repeat_two_offset_match_repeat_encodeBlockAsm10B
|
|
CMPL DI, $0x00000800
|
|
JB repeat_two_offset_match_repeat_encodeBlockAsm10B
|
|
|
|
cant_repeat_two_offset_match_repeat_encodeBlockAsm10B:
|
|
CMPL SI, $0x00000104
|
|
JB repeat_three_match_repeat_encodeBlockAsm10B
|
|
LEAL -256(SI), SI
|
|
MOVW $0x0019, (CX)
|
|
MOVW SI, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP repeat_end_emit_encodeBlockAsm10B
|
|
|
|
repeat_three_match_repeat_encodeBlockAsm10B:
|
|
LEAL -4(SI), SI
|
|
MOVW $0x0015, (CX)
|
|
MOVB SI, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP repeat_end_emit_encodeBlockAsm10B
|
|
|
|
repeat_two_match_repeat_encodeBlockAsm10B:
|
|
SHLL $0x02, SI
|
|
ORL $0x01, SI
|
|
MOVW SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm10B
|
|
|
|
repeat_two_offset_match_repeat_encodeBlockAsm10B:
|
|
XORQ R8, R8
|
|
LEAL 1(R8)(SI*4), SI
|
|
MOVB DI, 1(CX)
|
|
SARL $0x08, DI
|
|
SHLL $0x05, DI
|
|
ORL DI, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm10B
|
|
|
|
repeat_as_copy_encodeBlockAsm10B:
|
|
// emitCopy
|
|
CMPL SI, $0x40
|
|
JBE two_byte_offset_short_repeat_as_copy_encodeBlockAsm10B
|
|
CMPL DI, $0x00000800
|
|
JAE long_offset_short_repeat_as_copy_encodeBlockAsm10B
|
|
MOVL $0x00000001, R8
|
|
LEAL 16(R8), R8
|
|
MOVB DI, 1(CX)
|
|
SHRL $0x08, DI
|
|
SHLL $0x05, DI
|
|
ORL DI, R8
|
|
MOVB R8, (CX)
|
|
ADDQ $0x02, CX
|
|
SUBL $0x08, SI
|
|
|
|
// emitRepeat
|
|
LEAL -4(SI), SI
|
|
JMP cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b
|
|
MOVL SI, R8
|
|
LEAL -4(SI), SI
|
|
CMPL R8, $0x08
|
|
JBE repeat_two_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b
|
|
CMPL R8, $0x0c
|
|
JAE cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b
|
|
CMPL DI, $0x00000800
|
|
JB repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b
|
|
|
|
cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b:
|
|
CMPL SI, $0x00000104
|
|
JB repeat_three_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b
|
|
LEAL -256(SI), SI
|
|
MOVW $0x0019, (CX)
|
|
MOVW SI, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP repeat_end_emit_encodeBlockAsm10B
|
|
|
|
repeat_three_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b:
|
|
LEAL -4(SI), SI
|
|
MOVW $0x0015, (CX)
|
|
MOVB SI, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP repeat_end_emit_encodeBlockAsm10B
|
|
|
|
repeat_two_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b:
|
|
SHLL $0x02, SI
|
|
ORL $0x01, SI
|
|
MOVW SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm10B
|
|
|
|
repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b:
|
|
XORQ R8, R8
|
|
LEAL 1(R8)(SI*4), SI
|
|
MOVB DI, 1(CX)
|
|
SARL $0x08, DI
|
|
SHLL $0x05, DI
|
|
ORL DI, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm10B
|
|
|
|
long_offset_short_repeat_as_copy_encodeBlockAsm10B:
|
|
MOVB $0xee, (CX)
|
|
MOVW DI, 1(CX)
|
|
LEAL -60(SI), SI
|
|
ADDQ $0x03, CX
|
|
|
|
// emitRepeat
|
|
MOVL SI, R8
|
|
LEAL -4(SI), SI
|
|
CMPL R8, $0x08
|
|
JBE repeat_two_repeat_as_copy_encodeBlockAsm10B_emit_copy_short
|
|
CMPL R8, $0x0c
|
|
JAE cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short
|
|
CMPL DI, $0x00000800
|
|
JB repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short
|
|
|
|
cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short:
|
|
CMPL SI, $0x00000104
|
|
JB repeat_three_repeat_as_copy_encodeBlockAsm10B_emit_copy_short
|
|
LEAL -256(SI), SI
|
|
MOVW $0x0019, (CX)
|
|
MOVW SI, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP repeat_end_emit_encodeBlockAsm10B
|
|
|
|
repeat_three_repeat_as_copy_encodeBlockAsm10B_emit_copy_short:
|
|
LEAL -4(SI), SI
|
|
MOVW $0x0015, (CX)
|
|
MOVB SI, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP repeat_end_emit_encodeBlockAsm10B
|
|
|
|
repeat_two_repeat_as_copy_encodeBlockAsm10B_emit_copy_short:
|
|
SHLL $0x02, SI
|
|
ORL $0x01, SI
|
|
MOVW SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm10B
|
|
|
|
repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short:
|
|
XORQ R8, R8
|
|
LEAL 1(R8)(SI*4), SI
|
|
MOVB DI, 1(CX)
|
|
SARL $0x08, DI
|
|
SHLL $0x05, DI
|
|
ORL DI, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm10B
|
|
|
|
two_byte_offset_short_repeat_as_copy_encodeBlockAsm10B:
|
|
MOVL SI, R8
|
|
SHLL $0x02, R8
|
|
CMPL SI, $0x0c
|
|
JAE emit_copy_three_repeat_as_copy_encodeBlockAsm10B
|
|
CMPL DI, $0x00000800
|
|
JAE emit_copy_three_repeat_as_copy_encodeBlockAsm10B
|
|
LEAL -15(R8), R8
|
|
MOVB DI, 1(CX)
|
|
SHRL $0x08, DI
|
|
SHLL $0x05, DI
|
|
ORL DI, R8
|
|
MOVB R8, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm10B
|
|
|
|
emit_copy_three_repeat_as_copy_encodeBlockAsm10B:
|
|
LEAL -2(R8), R8
|
|
MOVB R8, (CX)
|
|
MOVW DI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
|
|
repeat_end_emit_encodeBlockAsm10B:
|
|
MOVL DX, 12(SP)
|
|
JMP search_loop_encodeBlockAsm10B
|
|
|
|
no_repeat_found_encodeBlockAsm10B:
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ candidate_match_encodeBlockAsm10B
|
|
SHRQ $0x08, DI
|
|
MOVL (AX)(R10*4), SI
|
|
LEAL 2(DX), R9
|
|
CMPL (BX)(R8*1), DI
|
|
JEQ candidate2_match_encodeBlockAsm10B
|
|
MOVL R9, (AX)(R10*4)
|
|
SHRQ $0x08, DI
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ candidate3_match_encodeBlockAsm10B
|
|
MOVL 20(SP), DX
|
|
JMP search_loop_encodeBlockAsm10B
|
|
|
|
candidate3_match_encodeBlockAsm10B:
|
|
ADDL $0x02, DX
|
|
JMP candidate_match_encodeBlockAsm10B
|
|
|
|
candidate2_match_encodeBlockAsm10B:
|
|
MOVL R9, (AX)(R10*4)
|
|
INCL DX
|
|
MOVL R8, SI
|
|
|
|
candidate_match_encodeBlockAsm10B:
|
|
MOVL 12(SP), DI
|
|
TESTL SI, SI
|
|
JZ match_extend_back_end_encodeBlockAsm10B
|
|
|
|
match_extend_back_loop_encodeBlockAsm10B:
|
|
CMPL DX, DI
|
|
JBE match_extend_back_end_encodeBlockAsm10B
|
|
MOVB -1(BX)(SI*1), R8
|
|
MOVB -1(BX)(DX*1), R9
|
|
CMPB R8, R9
|
|
JNE match_extend_back_end_encodeBlockAsm10B
|
|
LEAL -1(DX), DX
|
|
DECL SI
|
|
JZ match_extend_back_end_encodeBlockAsm10B
|
|
JMP match_extend_back_loop_encodeBlockAsm10B
|
|
|
|
match_extend_back_end_encodeBlockAsm10B:
|
|
MOVL DX, DI
|
|
SUBL 12(SP), DI
|
|
LEAQ 3(CX)(DI*1), DI
|
|
CMPQ DI, (SP)
|
|
JB match_dst_size_check_encodeBlockAsm10B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeBlockAsm10B:
|
|
MOVL DX, DI
|
|
MOVL 12(SP), R8
|
|
CMPL R8, DI
|
|
JEQ emit_literal_done_match_emit_encodeBlockAsm10B
|
|
MOVL DI, R9
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(R8*1), DI
|
|
SUBL R8, R9
|
|
LEAL -1(R9), R8
|
|
CMPL R8, $0x3c
|
|
JB one_byte_match_emit_encodeBlockAsm10B
|
|
CMPL R8, $0x00000100
|
|
JB two_bytes_match_emit_encodeBlockAsm10B
|
|
JB three_bytes_match_emit_encodeBlockAsm10B
|
|
|
|
three_bytes_match_emit_encodeBlockAsm10B:
|
|
MOVB $0xf4, (CX)
|
|
MOVW R8, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_match_emit_encodeBlockAsm10B
|
|
|
|
two_bytes_match_emit_encodeBlockAsm10B:
|
|
MOVB $0xf0, (CX)
|
|
MOVB R8, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL R8, $0x40
|
|
JB memmove_match_emit_encodeBlockAsm10B
|
|
JMP memmove_long_match_emit_encodeBlockAsm10B
|
|
|
|
one_byte_match_emit_encodeBlockAsm10B:
|
|
SHLB $0x02, R8
|
|
MOVB R8, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_match_emit_encodeBlockAsm10B:
|
|
LEAQ (CX)(R9*1), R8
|
|
|
|
// genMemMoveShort
|
|
CMPQ R9, $0x08
|
|
JBE emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_8
|
|
CMPQ R9, $0x10
|
|
JBE emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_8through16
|
|
CMPQ R9, $0x20
|
|
JBE emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_8:
|
|
MOVQ (DI), R10
|
|
MOVQ R10, (CX)
|
|
JMP memmove_end_copy_match_emit_encodeBlockAsm10B
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_8through16:
|
|
MOVQ (DI), R10
|
|
MOVQ -8(DI)(R9*1), DI
|
|
MOVQ R10, (CX)
|
|
MOVQ DI, -8(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeBlockAsm10B
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_17through32:
|
|
MOVOU (DI), X0
|
|
MOVOU -16(DI)(R9*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeBlockAsm10B
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_33through64:
|
|
MOVOU (DI), X0
|
|
MOVOU 16(DI), X1
|
|
MOVOU -32(DI)(R9*1), X2
|
|
MOVOU -16(DI)(R9*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
|
|
memmove_end_copy_match_emit_encodeBlockAsm10B:
|
|
MOVQ R8, CX
|
|
JMP emit_literal_done_match_emit_encodeBlockAsm10B
|
|
|
|
memmove_long_match_emit_encodeBlockAsm10B:
|
|
LEAQ (CX)(R9*1), R8
|
|
|
|
// genMemMoveLong
|
|
MOVOU (DI), X0
|
|
MOVOU 16(DI), X1
|
|
MOVOU -32(DI)(R9*1), X2
|
|
MOVOU -16(DI)(R9*1), X3
|
|
MOVQ R9, R11
|
|
SHRQ $0x05, R11
|
|
MOVQ CX, R10
|
|
ANDL $0x0000001f, R10
|
|
MOVQ $0x00000040, R12
|
|
SUBQ R10, R12
|
|
DECQ R11
|
|
JA emit_lit_memmove_long_match_emit_encodeBlockAsm10Blarge_forward_sse_loop_32
|
|
LEAQ -32(DI)(R12*1), R10
|
|
LEAQ -32(CX)(R12*1), R13
|
|
|
|
emit_lit_memmove_long_match_emit_encodeBlockAsm10Blarge_big_loop_back:
|
|
MOVOU (R10), X4
|
|
MOVOU 16(R10), X5
|
|
MOVOA X4, (R13)
|
|
MOVOA X5, 16(R13)
|
|
ADDQ $0x20, R13
|
|
ADDQ $0x20, R10
|
|
ADDQ $0x20, R12
|
|
DECQ R11
|
|
JNA emit_lit_memmove_long_match_emit_encodeBlockAsm10Blarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_match_emit_encodeBlockAsm10Blarge_forward_sse_loop_32:
|
|
MOVOU -32(DI)(R12*1), X4
|
|
MOVOU -16(DI)(R12*1), X5
|
|
MOVOA X4, -32(CX)(R12*1)
|
|
MOVOA X5, -16(CX)(R12*1)
|
|
ADDQ $0x20, R12
|
|
CMPQ R9, R12
|
|
JAE emit_lit_memmove_long_match_emit_encodeBlockAsm10Blarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
MOVQ R8, CX
|
|
|
|
emit_literal_done_match_emit_encodeBlockAsm10B:
|
|
match_nolit_loop_encodeBlockAsm10B:
|
|
MOVL DX, DI
|
|
SUBL SI, DI
|
|
MOVL DI, 16(SP)
|
|
ADDL $0x04, DX
|
|
ADDL $0x04, SI
|
|
MOVQ src_len+32(FP), DI
|
|
SUBL DX, DI
|
|
LEAQ (BX)(DX*1), R8
|
|
LEAQ (BX)(SI*1), SI
|
|
|
|
// matchLen
|
|
XORL R10, R10
|
|
|
|
matchlen_loopback_16_match_nolit_encodeBlockAsm10B:
|
|
CMPL DI, $0x10
|
|
JB matchlen_match8_match_nolit_encodeBlockAsm10B
|
|
MOVQ (R8)(R10*1), R9
|
|
MOVQ 8(R8)(R10*1), R11
|
|
XORQ (SI)(R10*1), R9
|
|
JNZ matchlen_bsf_8_match_nolit_encodeBlockAsm10B
|
|
XORQ 8(SI)(R10*1), R11
|
|
JNZ matchlen_bsf_16match_nolit_encodeBlockAsm10B
|
|
LEAL -16(DI), DI
|
|
LEAL 16(R10), R10
|
|
JMP matchlen_loopback_16_match_nolit_encodeBlockAsm10B
|
|
|
|
matchlen_bsf_16match_nolit_encodeBlockAsm10B:
|
|
TZCNTQ R11, R11
|
|
SARQ $0x03, R11
|
|
LEAL 8(R10)(R11*1), R10
|
|
JMP match_nolit_end_encodeBlockAsm10B
|
|
|
|
matchlen_match8_match_nolit_encodeBlockAsm10B:
|
|
CMPL DI, $0x08
|
|
JB matchlen_match4_match_nolit_encodeBlockAsm10B
|
|
MOVQ (R8)(R10*1), R9
|
|
XORQ (SI)(R10*1), R9
|
|
JNZ matchlen_bsf_8_match_nolit_encodeBlockAsm10B
|
|
LEAL -8(DI), DI
|
|
LEAL 8(R10), R10
|
|
JMP matchlen_match4_match_nolit_encodeBlockAsm10B
|
|
|
|
matchlen_bsf_8_match_nolit_encodeBlockAsm10B:
|
|
TZCNTQ R9, R9
|
|
SARQ $0x03, R9
|
|
LEAL (R10)(R9*1), R10
|
|
JMP match_nolit_end_encodeBlockAsm10B
|
|
|
|
matchlen_match4_match_nolit_encodeBlockAsm10B:
|
|
CMPL DI, $0x04
|
|
JB matchlen_match2_match_nolit_encodeBlockAsm10B
|
|
MOVL (R8)(R10*1), R9
|
|
CMPL (SI)(R10*1), R9
|
|
JNE matchlen_match2_match_nolit_encodeBlockAsm10B
|
|
LEAL -4(DI), DI
|
|
LEAL 4(R10), R10
|
|
|
|
matchlen_match2_match_nolit_encodeBlockAsm10B:
|
|
CMPL DI, $0x01
|
|
JE matchlen_match1_match_nolit_encodeBlockAsm10B
|
|
JB match_nolit_end_encodeBlockAsm10B
|
|
MOVW (R8)(R10*1), R9
|
|
CMPW (SI)(R10*1), R9
|
|
JNE matchlen_match1_match_nolit_encodeBlockAsm10B
|
|
LEAL 2(R10), R10
|
|
SUBL $0x02, DI
|
|
JZ match_nolit_end_encodeBlockAsm10B
|
|
|
|
matchlen_match1_match_nolit_encodeBlockAsm10B:
|
|
MOVB (R8)(R10*1), R9
|
|
CMPB (SI)(R10*1), R9
|
|
JNE match_nolit_end_encodeBlockAsm10B
|
|
LEAL 1(R10), R10
|
|
|
|
match_nolit_end_encodeBlockAsm10B:
|
|
ADDL R10, DX
|
|
MOVL 16(SP), SI
|
|
ADDL $0x04, R10
|
|
MOVL DX, 12(SP)
|
|
|
|
// emitCopy
|
|
CMPL R10, $0x40
|
|
JBE two_byte_offset_short_match_nolit_encodeBlockAsm10B
|
|
CMPL SI, $0x00000800
|
|
JAE long_offset_short_match_nolit_encodeBlockAsm10B
|
|
MOVL $0x00000001, DI
|
|
LEAL 16(DI), DI
|
|
MOVB SI, 1(CX)
|
|
SHRL $0x08, SI
|
|
SHLL $0x05, SI
|
|
ORL SI, DI
|
|
MOVB DI, (CX)
|
|
ADDQ $0x02, CX
|
|
SUBL $0x08, R10
|
|
|
|
// emitRepeat
|
|
LEAL -4(R10), R10
|
|
JMP cant_repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short_2b
|
|
MOVL R10, DI
|
|
LEAL -4(R10), R10
|
|
CMPL DI, $0x08
|
|
JBE repeat_two_match_nolit_encodeBlockAsm10B_emit_copy_short_2b
|
|
CMPL DI, $0x0c
|
|
JAE cant_repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short_2b
|
|
CMPL SI, $0x00000800
|
|
JB repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short_2b
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short_2b:
|
|
CMPL R10, $0x00000104
|
|
JB repeat_three_match_nolit_encodeBlockAsm10B_emit_copy_short_2b
|
|
LEAL -256(R10), R10
|
|
MOVW $0x0019, (CX)
|
|
MOVW R10, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm10B
|
|
|
|
repeat_three_match_nolit_encodeBlockAsm10B_emit_copy_short_2b:
|
|
LEAL -4(R10), R10
|
|
MOVW $0x0015, (CX)
|
|
MOVB R10, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm10B
|
|
|
|
repeat_two_match_nolit_encodeBlockAsm10B_emit_copy_short_2b:
|
|
SHLL $0x02, R10
|
|
ORL $0x01, R10
|
|
MOVW R10, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm10B
|
|
|
|
repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short_2b:
|
|
XORQ DI, DI
|
|
LEAL 1(DI)(R10*4), R10
|
|
MOVB SI, 1(CX)
|
|
SARL $0x08, SI
|
|
SHLL $0x05, SI
|
|
ORL SI, R10
|
|
MOVB R10, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm10B
|
|
|
|
long_offset_short_match_nolit_encodeBlockAsm10B:
|
|
MOVB $0xee, (CX)
|
|
MOVW SI, 1(CX)
|
|
LEAL -60(R10), R10
|
|
ADDQ $0x03, CX
|
|
|
|
// emitRepeat
|
|
MOVL R10, DI
|
|
LEAL -4(R10), R10
|
|
CMPL DI, $0x08
|
|
JBE repeat_two_match_nolit_encodeBlockAsm10B_emit_copy_short
|
|
CMPL DI, $0x0c
|
|
JAE cant_repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short
|
|
CMPL SI, $0x00000800
|
|
JB repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short:
|
|
CMPL R10, $0x00000104
|
|
JB repeat_three_match_nolit_encodeBlockAsm10B_emit_copy_short
|
|
LEAL -256(R10), R10
|
|
MOVW $0x0019, (CX)
|
|
MOVW R10, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm10B
|
|
|
|
repeat_three_match_nolit_encodeBlockAsm10B_emit_copy_short:
|
|
LEAL -4(R10), R10
|
|
MOVW $0x0015, (CX)
|
|
MOVB R10, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm10B
|
|
|
|
repeat_two_match_nolit_encodeBlockAsm10B_emit_copy_short:
|
|
SHLL $0x02, R10
|
|
ORL $0x01, R10
|
|
MOVW R10, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm10B
|
|
|
|
repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short:
|
|
XORQ DI, DI
|
|
LEAL 1(DI)(R10*4), R10
|
|
MOVB SI, 1(CX)
|
|
SARL $0x08, SI
|
|
SHLL $0x05, SI
|
|
ORL SI, R10
|
|
MOVB R10, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm10B
|
|
|
|
two_byte_offset_short_match_nolit_encodeBlockAsm10B:
|
|
MOVL R10, DI
|
|
SHLL $0x02, DI
|
|
CMPL R10, $0x0c
|
|
JAE emit_copy_three_match_nolit_encodeBlockAsm10B
|
|
CMPL SI, $0x00000800
|
|
JAE emit_copy_three_match_nolit_encodeBlockAsm10B
|
|
LEAL -15(DI), DI
|
|
MOVB SI, 1(CX)
|
|
SHRL $0x08, SI
|
|
SHLL $0x05, SI
|
|
ORL SI, DI
|
|
MOVB DI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm10B
|
|
|
|
emit_copy_three_match_nolit_encodeBlockAsm10B:
|
|
LEAL -2(DI), DI
|
|
MOVB DI, (CX)
|
|
MOVW SI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
|
|
match_nolit_emitcopy_end_encodeBlockAsm10B:
|
|
CMPL DX, 8(SP)
|
|
JAE emit_remainder_encodeBlockAsm10B
|
|
MOVQ -2(BX)(DX*1), DI
|
|
CMPQ CX, (SP)
|
|
JB match_nolit_dst_ok_encodeBlockAsm10B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeBlockAsm10B:
|
|
MOVQ $0x9e3779b1, R9
|
|
MOVQ DI, R8
|
|
SHRQ $0x10, DI
|
|
MOVQ DI, SI
|
|
SHLQ $0x20, R8
|
|
IMULQ R9, R8
|
|
SHRQ $0x36, R8
|
|
SHLQ $0x20, SI
|
|
IMULQ R9, SI
|
|
SHRQ $0x36, SI
|
|
LEAL -2(DX), R9
|
|
LEAQ (AX)(SI*4), R10
|
|
MOVL (R10), SI
|
|
MOVL R9, (AX)(R8*4)
|
|
MOVL DX, (R10)
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ match_nolit_loop_encodeBlockAsm10B
|
|
INCL DX
|
|
JMP search_loop_encodeBlockAsm10B
|
|
|
|
emit_remainder_encodeBlockAsm10B:
|
|
MOVQ src_len+32(FP), AX
|
|
SUBL 12(SP), AX
|
|
LEAQ 3(CX)(AX*1), AX
|
|
CMPQ AX, (SP)
|
|
JB emit_remainder_ok_encodeBlockAsm10B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeBlockAsm10B:
|
|
MOVQ src_len+32(FP), AX
|
|
MOVL 12(SP), DX
|
|
CMPL DX, AX
|
|
JEQ emit_literal_done_emit_remainder_encodeBlockAsm10B
|
|
MOVL AX, SI
|
|
MOVL AX, 12(SP)
|
|
LEAQ (BX)(DX*1), AX
|
|
SUBL DX, SI
|
|
LEAL -1(SI), DX
|
|
CMPL DX, $0x3c
|
|
JB one_byte_emit_remainder_encodeBlockAsm10B
|
|
CMPL DX, $0x00000100
|
|
JB two_bytes_emit_remainder_encodeBlockAsm10B
|
|
JB three_bytes_emit_remainder_encodeBlockAsm10B
|
|
|
|
three_bytes_emit_remainder_encodeBlockAsm10B:
|
|
MOVB $0xf4, (CX)
|
|
MOVW DX, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_emit_remainder_encodeBlockAsm10B
|
|
|
|
two_bytes_emit_remainder_encodeBlockAsm10B:
|
|
MOVB $0xf0, (CX)
|
|
MOVB DL, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL DX, $0x40
|
|
JB memmove_emit_remainder_encodeBlockAsm10B
|
|
JMP memmove_long_emit_remainder_encodeBlockAsm10B
|
|
|
|
one_byte_emit_remainder_encodeBlockAsm10B:
|
|
SHLB $0x02, DL
|
|
MOVB DL, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_emit_remainder_encodeBlockAsm10B:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveShort
|
|
CMPQ BX, $0x03
|
|
JB emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_1or2
|
|
JE emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_3
|
|
CMPQ BX, $0x08
|
|
JB emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_4through7
|
|
CMPQ BX, $0x10
|
|
JBE emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_8through16
|
|
CMPQ BX, $0x20
|
|
JBE emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_1or2:
|
|
MOVB (AX), SI
|
|
MOVB -1(AX)(BX*1), AL
|
|
MOVB SI, (CX)
|
|
MOVB AL, -1(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_3:
|
|
MOVW (AX), SI
|
|
MOVB 2(AX), AL
|
|
MOVW SI, (CX)
|
|
MOVB AL, 2(CX)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_4through7:
|
|
MOVL (AX), SI
|
|
MOVL -4(AX)(BX*1), AX
|
|
MOVL SI, (CX)
|
|
MOVL AX, -4(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_8through16:
|
|
MOVQ (AX), SI
|
|
MOVQ -8(AX)(BX*1), AX
|
|
MOVQ SI, (CX)
|
|
MOVQ AX, -8(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_17through32:
|
|
MOVOU (AX), X0
|
|
MOVOU -16(AX)(BX*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_33through64:
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
|
|
memmove_end_copy_emit_remainder_encodeBlockAsm10B:
|
|
MOVQ DX, CX
|
|
JMP emit_literal_done_emit_remainder_encodeBlockAsm10B
|
|
|
|
memmove_long_emit_remainder_encodeBlockAsm10B:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveLong
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVQ BX, DI
|
|
SHRQ $0x05, DI
|
|
MOVQ CX, SI
|
|
ANDL $0x0000001f, SI
|
|
MOVQ $0x00000040, R8
|
|
SUBQ SI, R8
|
|
DECQ DI
|
|
JA emit_lit_memmove_long_emit_remainder_encodeBlockAsm10Blarge_forward_sse_loop_32
|
|
LEAQ -32(AX)(R8*1), SI
|
|
LEAQ -32(CX)(R8*1), R9
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeBlockAsm10Blarge_big_loop_back:
|
|
MOVOU (SI), X4
|
|
MOVOU 16(SI), X5
|
|
MOVOA X4, (R9)
|
|
MOVOA X5, 16(R9)
|
|
ADDQ $0x20, R9
|
|
ADDQ $0x20, SI
|
|
ADDQ $0x20, R8
|
|
DECQ DI
|
|
JNA emit_lit_memmove_long_emit_remainder_encodeBlockAsm10Blarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeBlockAsm10Blarge_forward_sse_loop_32:
|
|
MOVOU -32(AX)(R8*1), X4
|
|
MOVOU -16(AX)(R8*1), X5
|
|
MOVOA X4, -32(CX)(R8*1)
|
|
MOVOA X5, -16(CX)(R8*1)
|
|
ADDQ $0x20, R8
|
|
CMPQ BX, R8
|
|
JAE emit_lit_memmove_long_emit_remainder_encodeBlockAsm10Blarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
MOVQ DX, CX
|
|
|
|
emit_literal_done_emit_remainder_encodeBlockAsm10B:
|
|
MOVQ dst_base+0(FP), AX
|
|
SUBQ AX, CX
|
|
MOVQ CX, ret+56(FP)
|
|
RET
|
|
|
|
// func encodeBlockAsm8B(dst []byte, src []byte, tmp *[1024]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeBlockAsm8B(SB), $24-64
|
|
MOVQ tmp+48(FP), AX
|
|
MOVQ dst_base+0(FP), CX
|
|
MOVQ $0x00000008, DX
|
|
MOVQ AX, BX
|
|
PXOR X0, X0
|
|
|
|
zero_loop_encodeBlockAsm8B:
|
|
MOVOU X0, (BX)
|
|
MOVOU X0, 16(BX)
|
|
MOVOU X0, 32(BX)
|
|
MOVOU X0, 48(BX)
|
|
MOVOU X0, 64(BX)
|
|
MOVOU X0, 80(BX)
|
|
MOVOU X0, 96(BX)
|
|
MOVOU X0, 112(BX)
|
|
ADDQ $0x80, BX
|
|
DECQ DX
|
|
JNZ zero_loop_encodeBlockAsm8B
|
|
MOVL $0x00000000, 12(SP)
|
|
MOVQ src_len+32(FP), DX
|
|
LEAQ -9(DX), BX
|
|
LEAQ -8(DX), SI
|
|
MOVL SI, 8(SP)
|
|
SHRQ $0x05, DX
|
|
SUBL DX, BX
|
|
LEAQ (CX)(BX*1), BX
|
|
MOVQ BX, (SP)
|
|
MOVL $0x00000001, DX
|
|
MOVL DX, 16(SP)
|
|
MOVQ src_base+24(FP), BX
|
|
|
|
search_loop_encodeBlockAsm8B:
|
|
MOVL DX, SI
|
|
SUBL 12(SP), SI
|
|
SHRL $0x04, SI
|
|
LEAL 4(DX)(SI*1), SI
|
|
CMPL SI, 8(SP)
|
|
JAE emit_remainder_encodeBlockAsm8B
|
|
MOVQ (BX)(DX*1), DI
|
|
MOVL SI, 20(SP)
|
|
MOVQ $0x9e3779b1, R9
|
|
MOVQ DI, R10
|
|
MOVQ DI, R11
|
|
SHRQ $0x08, R11
|
|
SHLQ $0x20, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x38, R10
|
|
SHLQ $0x20, R11
|
|
IMULQ R9, R11
|
|
SHRQ $0x38, R11
|
|
MOVL (AX)(R10*4), SI
|
|
MOVL (AX)(R11*4), R8
|
|
MOVL DX, (AX)(R10*4)
|
|
LEAL 1(DX), R10
|
|
MOVL R10, (AX)(R11*4)
|
|
MOVQ DI, R10
|
|
SHRQ $0x10, R10
|
|
SHLQ $0x20, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x38, R10
|
|
MOVL DX, R9
|
|
SUBL 16(SP), R9
|
|
MOVL 1(BX)(R9*1), R11
|
|
MOVQ DI, R9
|
|
SHRQ $0x08, R9
|
|
CMPL R9, R11
|
|
JNE no_repeat_found_encodeBlockAsm8B
|
|
LEAL 1(DX), DI
|
|
MOVL 12(SP), R8
|
|
MOVL DI, SI
|
|
SUBL 16(SP), SI
|
|
JZ repeat_extend_back_end_encodeBlockAsm8B
|
|
|
|
repeat_extend_back_loop_encodeBlockAsm8B:
|
|
CMPL DI, R8
|
|
JBE repeat_extend_back_end_encodeBlockAsm8B
|
|
MOVB -1(BX)(SI*1), R9
|
|
MOVB -1(BX)(DI*1), R10
|
|
CMPB R9, R10
|
|
JNE repeat_extend_back_end_encodeBlockAsm8B
|
|
LEAL -1(DI), DI
|
|
DECL SI
|
|
JNZ repeat_extend_back_loop_encodeBlockAsm8B
|
|
|
|
repeat_extend_back_end_encodeBlockAsm8B:
|
|
MOVL DI, SI
|
|
SUBL 12(SP), SI
|
|
LEAQ 3(CX)(SI*1), SI
|
|
CMPQ SI, (SP)
|
|
JB repeat_dst_size_check_encodeBlockAsm8B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
repeat_dst_size_check_encodeBlockAsm8B:
|
|
MOVL 12(SP), SI
|
|
CMPL SI, DI
|
|
JEQ emit_literal_done_repeat_emit_encodeBlockAsm8B
|
|
MOVL DI, R9
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(SI*1), R10
|
|
SUBL SI, R9
|
|
LEAL -1(R9), SI
|
|
CMPL SI, $0x3c
|
|
JB one_byte_repeat_emit_encodeBlockAsm8B
|
|
CMPL SI, $0x00000100
|
|
JB two_bytes_repeat_emit_encodeBlockAsm8B
|
|
JB three_bytes_repeat_emit_encodeBlockAsm8B
|
|
|
|
three_bytes_repeat_emit_encodeBlockAsm8B:
|
|
MOVB $0xf4, (CX)
|
|
MOVW SI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_repeat_emit_encodeBlockAsm8B
|
|
|
|
two_bytes_repeat_emit_encodeBlockAsm8B:
|
|
MOVB $0xf0, (CX)
|
|
MOVB SI, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL SI, $0x40
|
|
JB memmove_repeat_emit_encodeBlockAsm8B
|
|
JMP memmove_long_repeat_emit_encodeBlockAsm8B
|
|
|
|
one_byte_repeat_emit_encodeBlockAsm8B:
|
|
SHLB $0x02, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_repeat_emit_encodeBlockAsm8B:
|
|
LEAQ (CX)(R9*1), SI
|
|
|
|
// genMemMoveShort
|
|
CMPQ R9, $0x08
|
|
JBE emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_8
|
|
CMPQ R9, $0x10
|
|
JBE emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_8through16
|
|
CMPQ R9, $0x20
|
|
JBE emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_17through32
|
|
JMP emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_8:
|
|
MOVQ (R10), R11
|
|
MOVQ R11, (CX)
|
|
JMP memmove_end_copy_repeat_emit_encodeBlockAsm8B
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_8through16:
|
|
MOVQ (R10), R11
|
|
MOVQ -8(R10)(R9*1), R10
|
|
MOVQ R11, (CX)
|
|
MOVQ R10, -8(CX)(R9*1)
|
|
JMP memmove_end_copy_repeat_emit_encodeBlockAsm8B
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_17through32:
|
|
MOVOU (R10), X0
|
|
MOVOU -16(R10)(R9*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(R9*1)
|
|
JMP memmove_end_copy_repeat_emit_encodeBlockAsm8B
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_33through64:
|
|
MOVOU (R10), X0
|
|
MOVOU 16(R10), X1
|
|
MOVOU -32(R10)(R9*1), X2
|
|
MOVOU -16(R10)(R9*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
|
|
memmove_end_copy_repeat_emit_encodeBlockAsm8B:
|
|
MOVQ SI, CX
|
|
JMP emit_literal_done_repeat_emit_encodeBlockAsm8B
|
|
|
|
memmove_long_repeat_emit_encodeBlockAsm8B:
|
|
LEAQ (CX)(R9*1), SI
|
|
|
|
// genMemMoveLong
|
|
MOVOU (R10), X0
|
|
MOVOU 16(R10), X1
|
|
MOVOU -32(R10)(R9*1), X2
|
|
MOVOU -16(R10)(R9*1), X3
|
|
MOVQ R9, R12
|
|
SHRQ $0x05, R12
|
|
MOVQ CX, R11
|
|
ANDL $0x0000001f, R11
|
|
MOVQ $0x00000040, R13
|
|
SUBQ R11, R13
|
|
DECQ R12
|
|
JA emit_lit_memmove_long_repeat_emit_encodeBlockAsm8Blarge_forward_sse_loop_32
|
|
LEAQ -32(R10)(R13*1), R11
|
|
LEAQ -32(CX)(R13*1), R14
|
|
|
|
emit_lit_memmove_long_repeat_emit_encodeBlockAsm8Blarge_big_loop_back:
|
|
MOVOU (R11), X4
|
|
MOVOU 16(R11), X5
|
|
MOVOA X4, (R14)
|
|
MOVOA X5, 16(R14)
|
|
ADDQ $0x20, R14
|
|
ADDQ $0x20, R11
|
|
ADDQ $0x20, R13
|
|
DECQ R12
|
|
JNA emit_lit_memmove_long_repeat_emit_encodeBlockAsm8Blarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_repeat_emit_encodeBlockAsm8Blarge_forward_sse_loop_32:
|
|
MOVOU -32(R10)(R13*1), X4
|
|
MOVOU -16(R10)(R13*1), X5
|
|
MOVOA X4, -32(CX)(R13*1)
|
|
MOVOA X5, -16(CX)(R13*1)
|
|
ADDQ $0x20, R13
|
|
CMPQ R9, R13
|
|
JAE emit_lit_memmove_long_repeat_emit_encodeBlockAsm8Blarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
MOVQ SI, CX
|
|
|
|
emit_literal_done_repeat_emit_encodeBlockAsm8B:
|
|
ADDL $0x05, DX
|
|
MOVL DX, SI
|
|
SUBL 16(SP), SI
|
|
MOVQ src_len+32(FP), R9
|
|
SUBL DX, R9
|
|
LEAQ (BX)(DX*1), R10
|
|
LEAQ (BX)(SI*1), SI
|
|
|
|
// matchLen
|
|
XORL R12, R12
|
|
|
|
matchlen_loopback_16_repeat_extend_encodeBlockAsm8B:
|
|
CMPL R9, $0x10
|
|
JB matchlen_match8_repeat_extend_encodeBlockAsm8B
|
|
MOVQ (R10)(R12*1), R11
|
|
MOVQ 8(R10)(R12*1), R13
|
|
XORQ (SI)(R12*1), R11
|
|
JNZ matchlen_bsf_8_repeat_extend_encodeBlockAsm8B
|
|
XORQ 8(SI)(R12*1), R13
|
|
JNZ matchlen_bsf_16repeat_extend_encodeBlockAsm8B
|
|
LEAL -16(R9), R9
|
|
LEAL 16(R12), R12
|
|
JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm8B
|
|
|
|
matchlen_bsf_16repeat_extend_encodeBlockAsm8B:
|
|
TZCNTQ R13, R13
|
|
SARQ $0x03, R13
|
|
LEAL 8(R12)(R13*1), R12
|
|
JMP repeat_extend_forward_end_encodeBlockAsm8B
|
|
|
|
matchlen_match8_repeat_extend_encodeBlockAsm8B:
|
|
CMPL R9, $0x08
|
|
JB matchlen_match4_repeat_extend_encodeBlockAsm8B
|
|
MOVQ (R10)(R12*1), R11
|
|
XORQ (SI)(R12*1), R11
|
|
JNZ matchlen_bsf_8_repeat_extend_encodeBlockAsm8B
|
|
LEAL -8(R9), R9
|
|
LEAL 8(R12), R12
|
|
JMP matchlen_match4_repeat_extend_encodeBlockAsm8B
|
|
|
|
matchlen_bsf_8_repeat_extend_encodeBlockAsm8B:
|
|
TZCNTQ R11, R11
|
|
SARQ $0x03, R11
|
|
LEAL (R12)(R11*1), R12
|
|
JMP repeat_extend_forward_end_encodeBlockAsm8B
|
|
|
|
matchlen_match4_repeat_extend_encodeBlockAsm8B:
|
|
CMPL R9, $0x04
|
|
JB matchlen_match2_repeat_extend_encodeBlockAsm8B
|
|
MOVL (R10)(R12*1), R11
|
|
CMPL (SI)(R12*1), R11
|
|
JNE matchlen_match2_repeat_extend_encodeBlockAsm8B
|
|
LEAL -4(R9), R9
|
|
LEAL 4(R12), R12
|
|
|
|
matchlen_match2_repeat_extend_encodeBlockAsm8B:
|
|
CMPL R9, $0x01
|
|
JE matchlen_match1_repeat_extend_encodeBlockAsm8B
|
|
JB repeat_extend_forward_end_encodeBlockAsm8B
|
|
MOVW (R10)(R12*1), R11
|
|
CMPW (SI)(R12*1), R11
|
|
JNE matchlen_match1_repeat_extend_encodeBlockAsm8B
|
|
LEAL 2(R12), R12
|
|
SUBL $0x02, R9
|
|
JZ repeat_extend_forward_end_encodeBlockAsm8B
|
|
|
|
matchlen_match1_repeat_extend_encodeBlockAsm8B:
|
|
MOVB (R10)(R12*1), R11
|
|
CMPB (SI)(R12*1), R11
|
|
JNE repeat_extend_forward_end_encodeBlockAsm8B
|
|
LEAL 1(R12), R12
|
|
|
|
repeat_extend_forward_end_encodeBlockAsm8B:
|
|
ADDL R12, DX
|
|
MOVL DX, SI
|
|
SUBL DI, SI
|
|
MOVL 16(SP), DI
|
|
TESTL R8, R8
|
|
JZ repeat_as_copy_encodeBlockAsm8B
|
|
|
|
// emitRepeat
|
|
MOVL SI, DI
|
|
LEAL -4(SI), SI
|
|
CMPL DI, $0x08
|
|
JBE repeat_two_match_repeat_encodeBlockAsm8B
|
|
CMPL DI, $0x0c
|
|
JAE cant_repeat_two_offset_match_repeat_encodeBlockAsm8B
|
|
|
|
cant_repeat_two_offset_match_repeat_encodeBlockAsm8B:
|
|
CMPL SI, $0x00000104
|
|
JB repeat_three_match_repeat_encodeBlockAsm8B
|
|
LEAL -256(SI), SI
|
|
MOVW $0x0019, (CX)
|
|
MOVW SI, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP repeat_end_emit_encodeBlockAsm8B
|
|
|
|
repeat_three_match_repeat_encodeBlockAsm8B:
|
|
LEAL -4(SI), SI
|
|
MOVW $0x0015, (CX)
|
|
MOVB SI, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP repeat_end_emit_encodeBlockAsm8B
|
|
|
|
repeat_two_match_repeat_encodeBlockAsm8B:
|
|
SHLL $0x02, SI
|
|
ORL $0x01, SI
|
|
MOVW SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm8B
|
|
XORQ R8, R8
|
|
LEAL 1(R8)(SI*4), SI
|
|
MOVB DI, 1(CX)
|
|
SARL $0x08, DI
|
|
SHLL $0x05, DI
|
|
ORL DI, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm8B
|
|
|
|
repeat_as_copy_encodeBlockAsm8B:
|
|
// emitCopy
|
|
CMPL SI, $0x40
|
|
JBE two_byte_offset_short_repeat_as_copy_encodeBlockAsm8B
|
|
CMPL DI, $0x00000800
|
|
JAE long_offset_short_repeat_as_copy_encodeBlockAsm8B
|
|
MOVL $0x00000001, R8
|
|
LEAL 16(R8), R8
|
|
MOVB DI, 1(CX)
|
|
SHRL $0x08, DI
|
|
SHLL $0x05, DI
|
|
ORL DI, R8
|
|
MOVB R8, (CX)
|
|
ADDQ $0x02, CX
|
|
SUBL $0x08, SI
|
|
|
|
// emitRepeat
|
|
LEAL -4(SI), SI
|
|
JMP cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b
|
|
MOVL SI, DI
|
|
LEAL -4(SI), SI
|
|
CMPL DI, $0x08
|
|
JBE repeat_two_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b
|
|
CMPL DI, $0x0c
|
|
JAE cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b
|
|
|
|
cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b:
|
|
CMPL SI, $0x00000104
|
|
JB repeat_three_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b
|
|
LEAL -256(SI), SI
|
|
MOVW $0x0019, (CX)
|
|
MOVW SI, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP repeat_end_emit_encodeBlockAsm8B
|
|
|
|
repeat_three_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b:
|
|
LEAL -4(SI), SI
|
|
MOVW $0x0015, (CX)
|
|
MOVB SI, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP repeat_end_emit_encodeBlockAsm8B
|
|
|
|
repeat_two_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b:
|
|
SHLL $0x02, SI
|
|
ORL $0x01, SI
|
|
MOVW SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm8B
|
|
XORQ R8, R8
|
|
LEAL 1(R8)(SI*4), SI
|
|
MOVB DI, 1(CX)
|
|
SARL $0x08, DI
|
|
SHLL $0x05, DI
|
|
ORL DI, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm8B
|
|
|
|
long_offset_short_repeat_as_copy_encodeBlockAsm8B:
|
|
MOVB $0xee, (CX)
|
|
MOVW DI, 1(CX)
|
|
LEAL -60(SI), SI
|
|
ADDQ $0x03, CX
|
|
|
|
// emitRepeat
|
|
MOVL SI, DI
|
|
LEAL -4(SI), SI
|
|
CMPL DI, $0x08
|
|
JBE repeat_two_repeat_as_copy_encodeBlockAsm8B_emit_copy_short
|
|
CMPL DI, $0x0c
|
|
JAE cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm8B_emit_copy_short
|
|
|
|
cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm8B_emit_copy_short:
|
|
CMPL SI, $0x00000104
|
|
JB repeat_three_repeat_as_copy_encodeBlockAsm8B_emit_copy_short
|
|
LEAL -256(SI), SI
|
|
MOVW $0x0019, (CX)
|
|
MOVW SI, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP repeat_end_emit_encodeBlockAsm8B
|
|
|
|
repeat_three_repeat_as_copy_encodeBlockAsm8B_emit_copy_short:
|
|
LEAL -4(SI), SI
|
|
MOVW $0x0015, (CX)
|
|
MOVB SI, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP repeat_end_emit_encodeBlockAsm8B
|
|
|
|
repeat_two_repeat_as_copy_encodeBlockAsm8B_emit_copy_short:
|
|
SHLL $0x02, SI
|
|
ORL $0x01, SI
|
|
MOVW SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm8B
|
|
XORQ R8, R8
|
|
LEAL 1(R8)(SI*4), SI
|
|
MOVB DI, 1(CX)
|
|
SARL $0x08, DI
|
|
SHLL $0x05, DI
|
|
ORL DI, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm8B
|
|
|
|
two_byte_offset_short_repeat_as_copy_encodeBlockAsm8B:
|
|
MOVL SI, R8
|
|
SHLL $0x02, R8
|
|
CMPL SI, $0x0c
|
|
JAE emit_copy_three_repeat_as_copy_encodeBlockAsm8B
|
|
LEAL -15(R8), R8
|
|
MOVB DI, 1(CX)
|
|
SHRL $0x08, DI
|
|
SHLL $0x05, DI
|
|
ORL DI, R8
|
|
MOVB R8, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeBlockAsm8B
|
|
|
|
emit_copy_three_repeat_as_copy_encodeBlockAsm8B:
|
|
LEAL -2(R8), R8
|
|
MOVB R8, (CX)
|
|
MOVW DI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
|
|
repeat_end_emit_encodeBlockAsm8B:
|
|
MOVL DX, 12(SP)
|
|
JMP search_loop_encodeBlockAsm8B
|
|
|
|
no_repeat_found_encodeBlockAsm8B:
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ candidate_match_encodeBlockAsm8B
|
|
SHRQ $0x08, DI
|
|
MOVL (AX)(R10*4), SI
|
|
LEAL 2(DX), R9
|
|
CMPL (BX)(R8*1), DI
|
|
JEQ candidate2_match_encodeBlockAsm8B
|
|
MOVL R9, (AX)(R10*4)
|
|
SHRQ $0x08, DI
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ candidate3_match_encodeBlockAsm8B
|
|
MOVL 20(SP), DX
|
|
JMP search_loop_encodeBlockAsm8B
|
|
|
|
candidate3_match_encodeBlockAsm8B:
|
|
ADDL $0x02, DX
|
|
JMP candidate_match_encodeBlockAsm8B
|
|
|
|
candidate2_match_encodeBlockAsm8B:
|
|
MOVL R9, (AX)(R10*4)
|
|
INCL DX
|
|
MOVL R8, SI
|
|
|
|
candidate_match_encodeBlockAsm8B:
|
|
MOVL 12(SP), DI
|
|
TESTL SI, SI
|
|
JZ match_extend_back_end_encodeBlockAsm8B
|
|
|
|
match_extend_back_loop_encodeBlockAsm8B:
|
|
CMPL DX, DI
|
|
JBE match_extend_back_end_encodeBlockAsm8B
|
|
MOVB -1(BX)(SI*1), R8
|
|
MOVB -1(BX)(DX*1), R9
|
|
CMPB R8, R9
|
|
JNE match_extend_back_end_encodeBlockAsm8B
|
|
LEAL -1(DX), DX
|
|
DECL SI
|
|
JZ match_extend_back_end_encodeBlockAsm8B
|
|
JMP match_extend_back_loop_encodeBlockAsm8B
|
|
|
|
match_extend_back_end_encodeBlockAsm8B:
|
|
MOVL DX, DI
|
|
SUBL 12(SP), DI
|
|
LEAQ 3(CX)(DI*1), DI
|
|
CMPQ DI, (SP)
|
|
JB match_dst_size_check_encodeBlockAsm8B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeBlockAsm8B:
|
|
MOVL DX, DI
|
|
MOVL 12(SP), R8
|
|
CMPL R8, DI
|
|
JEQ emit_literal_done_match_emit_encodeBlockAsm8B
|
|
MOVL DI, R9
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(R8*1), DI
|
|
SUBL R8, R9
|
|
LEAL -1(R9), R8
|
|
CMPL R8, $0x3c
|
|
JB one_byte_match_emit_encodeBlockAsm8B
|
|
CMPL R8, $0x00000100
|
|
JB two_bytes_match_emit_encodeBlockAsm8B
|
|
JB three_bytes_match_emit_encodeBlockAsm8B
|
|
|
|
three_bytes_match_emit_encodeBlockAsm8B:
|
|
MOVB $0xf4, (CX)
|
|
MOVW R8, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_match_emit_encodeBlockAsm8B
|
|
|
|
two_bytes_match_emit_encodeBlockAsm8B:
|
|
MOVB $0xf0, (CX)
|
|
MOVB R8, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL R8, $0x40
|
|
JB memmove_match_emit_encodeBlockAsm8B
|
|
JMP memmove_long_match_emit_encodeBlockAsm8B
|
|
|
|
one_byte_match_emit_encodeBlockAsm8B:
|
|
SHLB $0x02, R8
|
|
MOVB R8, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_match_emit_encodeBlockAsm8B:
|
|
LEAQ (CX)(R9*1), R8
|
|
|
|
// genMemMoveShort
|
|
CMPQ R9, $0x08
|
|
JBE emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_8
|
|
CMPQ R9, $0x10
|
|
JBE emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_8through16
|
|
CMPQ R9, $0x20
|
|
JBE emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_8:
|
|
MOVQ (DI), R10
|
|
MOVQ R10, (CX)
|
|
JMP memmove_end_copy_match_emit_encodeBlockAsm8B
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_8through16:
|
|
MOVQ (DI), R10
|
|
MOVQ -8(DI)(R9*1), DI
|
|
MOVQ R10, (CX)
|
|
MOVQ DI, -8(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeBlockAsm8B
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_17through32:
|
|
MOVOU (DI), X0
|
|
MOVOU -16(DI)(R9*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeBlockAsm8B
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_33through64:
|
|
MOVOU (DI), X0
|
|
MOVOU 16(DI), X1
|
|
MOVOU -32(DI)(R9*1), X2
|
|
MOVOU -16(DI)(R9*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
|
|
memmove_end_copy_match_emit_encodeBlockAsm8B:
|
|
MOVQ R8, CX
|
|
JMP emit_literal_done_match_emit_encodeBlockAsm8B
|
|
|
|
memmove_long_match_emit_encodeBlockAsm8B:
|
|
LEAQ (CX)(R9*1), R8
|
|
|
|
// genMemMoveLong
|
|
MOVOU (DI), X0
|
|
MOVOU 16(DI), X1
|
|
MOVOU -32(DI)(R9*1), X2
|
|
MOVOU -16(DI)(R9*1), X3
|
|
MOVQ R9, R11
|
|
SHRQ $0x05, R11
|
|
MOVQ CX, R10
|
|
ANDL $0x0000001f, R10
|
|
MOVQ $0x00000040, R12
|
|
SUBQ R10, R12
|
|
DECQ R11
|
|
JA emit_lit_memmove_long_match_emit_encodeBlockAsm8Blarge_forward_sse_loop_32
|
|
LEAQ -32(DI)(R12*1), R10
|
|
LEAQ -32(CX)(R12*1), R13
|
|
|
|
emit_lit_memmove_long_match_emit_encodeBlockAsm8Blarge_big_loop_back:
|
|
MOVOU (R10), X4
|
|
MOVOU 16(R10), X5
|
|
MOVOA X4, (R13)
|
|
MOVOA X5, 16(R13)
|
|
ADDQ $0x20, R13
|
|
ADDQ $0x20, R10
|
|
ADDQ $0x20, R12
|
|
DECQ R11
|
|
JNA emit_lit_memmove_long_match_emit_encodeBlockAsm8Blarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_match_emit_encodeBlockAsm8Blarge_forward_sse_loop_32:
|
|
MOVOU -32(DI)(R12*1), X4
|
|
MOVOU -16(DI)(R12*1), X5
|
|
MOVOA X4, -32(CX)(R12*1)
|
|
MOVOA X5, -16(CX)(R12*1)
|
|
ADDQ $0x20, R12
|
|
CMPQ R9, R12
|
|
JAE emit_lit_memmove_long_match_emit_encodeBlockAsm8Blarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
MOVQ R8, CX
|
|
|
|
emit_literal_done_match_emit_encodeBlockAsm8B:
|
|
match_nolit_loop_encodeBlockAsm8B:
|
|
MOVL DX, DI
|
|
SUBL SI, DI
|
|
MOVL DI, 16(SP)
|
|
ADDL $0x04, DX
|
|
ADDL $0x04, SI
|
|
MOVQ src_len+32(FP), DI
|
|
SUBL DX, DI
|
|
LEAQ (BX)(DX*1), R8
|
|
LEAQ (BX)(SI*1), SI
|
|
|
|
// matchLen
|
|
XORL R10, R10
|
|
|
|
matchlen_loopback_16_match_nolit_encodeBlockAsm8B:
|
|
CMPL DI, $0x10
|
|
JB matchlen_match8_match_nolit_encodeBlockAsm8B
|
|
MOVQ (R8)(R10*1), R9
|
|
MOVQ 8(R8)(R10*1), R11
|
|
XORQ (SI)(R10*1), R9
|
|
JNZ matchlen_bsf_8_match_nolit_encodeBlockAsm8B
|
|
XORQ 8(SI)(R10*1), R11
|
|
JNZ matchlen_bsf_16match_nolit_encodeBlockAsm8B
|
|
LEAL -16(DI), DI
|
|
LEAL 16(R10), R10
|
|
JMP matchlen_loopback_16_match_nolit_encodeBlockAsm8B
|
|
|
|
matchlen_bsf_16match_nolit_encodeBlockAsm8B:
|
|
TZCNTQ R11, R11
|
|
SARQ $0x03, R11
|
|
LEAL 8(R10)(R11*1), R10
|
|
JMP match_nolit_end_encodeBlockAsm8B
|
|
|
|
matchlen_match8_match_nolit_encodeBlockAsm8B:
|
|
CMPL DI, $0x08
|
|
JB matchlen_match4_match_nolit_encodeBlockAsm8B
|
|
MOVQ (R8)(R10*1), R9
|
|
XORQ (SI)(R10*1), R9
|
|
JNZ matchlen_bsf_8_match_nolit_encodeBlockAsm8B
|
|
LEAL -8(DI), DI
|
|
LEAL 8(R10), R10
|
|
JMP matchlen_match4_match_nolit_encodeBlockAsm8B
|
|
|
|
matchlen_bsf_8_match_nolit_encodeBlockAsm8B:
|
|
TZCNTQ R9, R9
|
|
SARQ $0x03, R9
|
|
LEAL (R10)(R9*1), R10
|
|
JMP match_nolit_end_encodeBlockAsm8B
|
|
|
|
matchlen_match4_match_nolit_encodeBlockAsm8B:
|
|
CMPL DI, $0x04
|
|
JB matchlen_match2_match_nolit_encodeBlockAsm8B
|
|
MOVL (R8)(R10*1), R9
|
|
CMPL (SI)(R10*1), R9
|
|
JNE matchlen_match2_match_nolit_encodeBlockAsm8B
|
|
LEAL -4(DI), DI
|
|
LEAL 4(R10), R10
|
|
|
|
matchlen_match2_match_nolit_encodeBlockAsm8B:
|
|
CMPL DI, $0x01
|
|
JE matchlen_match1_match_nolit_encodeBlockAsm8B
|
|
JB match_nolit_end_encodeBlockAsm8B
|
|
MOVW (R8)(R10*1), R9
|
|
CMPW (SI)(R10*1), R9
|
|
JNE matchlen_match1_match_nolit_encodeBlockAsm8B
|
|
LEAL 2(R10), R10
|
|
SUBL $0x02, DI
|
|
JZ match_nolit_end_encodeBlockAsm8B
|
|
|
|
matchlen_match1_match_nolit_encodeBlockAsm8B:
|
|
MOVB (R8)(R10*1), R9
|
|
CMPB (SI)(R10*1), R9
|
|
JNE match_nolit_end_encodeBlockAsm8B
|
|
LEAL 1(R10), R10
|
|
|
|
match_nolit_end_encodeBlockAsm8B:
|
|
ADDL R10, DX
|
|
MOVL 16(SP), SI
|
|
ADDL $0x04, R10
|
|
MOVL DX, 12(SP)
|
|
|
|
// emitCopy
|
|
CMPL R10, $0x40
|
|
JBE two_byte_offset_short_match_nolit_encodeBlockAsm8B
|
|
CMPL SI, $0x00000800
|
|
JAE long_offset_short_match_nolit_encodeBlockAsm8B
|
|
MOVL $0x00000001, DI
|
|
LEAL 16(DI), DI
|
|
MOVB SI, 1(CX)
|
|
SHRL $0x08, SI
|
|
SHLL $0x05, SI
|
|
ORL SI, DI
|
|
MOVB DI, (CX)
|
|
ADDQ $0x02, CX
|
|
SUBL $0x08, R10
|
|
|
|
// emitRepeat
|
|
LEAL -4(R10), R10
|
|
JMP cant_repeat_two_offset_match_nolit_encodeBlockAsm8B_emit_copy_short_2b
|
|
MOVL R10, SI
|
|
LEAL -4(R10), R10
|
|
CMPL SI, $0x08
|
|
JBE repeat_two_match_nolit_encodeBlockAsm8B_emit_copy_short_2b
|
|
CMPL SI, $0x0c
|
|
JAE cant_repeat_two_offset_match_nolit_encodeBlockAsm8B_emit_copy_short_2b
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBlockAsm8B_emit_copy_short_2b:
|
|
CMPL R10, $0x00000104
|
|
JB repeat_three_match_nolit_encodeBlockAsm8B_emit_copy_short_2b
|
|
LEAL -256(R10), R10
|
|
MOVW $0x0019, (CX)
|
|
MOVW R10, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm8B
|
|
|
|
repeat_three_match_nolit_encodeBlockAsm8B_emit_copy_short_2b:
|
|
LEAL -4(R10), R10
|
|
MOVW $0x0015, (CX)
|
|
MOVB R10, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm8B
|
|
|
|
repeat_two_match_nolit_encodeBlockAsm8B_emit_copy_short_2b:
|
|
SHLL $0x02, R10
|
|
ORL $0x01, R10
|
|
MOVW R10, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm8B
|
|
XORQ DI, DI
|
|
LEAL 1(DI)(R10*4), R10
|
|
MOVB SI, 1(CX)
|
|
SARL $0x08, SI
|
|
SHLL $0x05, SI
|
|
ORL SI, R10
|
|
MOVB R10, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm8B
|
|
|
|
long_offset_short_match_nolit_encodeBlockAsm8B:
|
|
MOVB $0xee, (CX)
|
|
MOVW SI, 1(CX)
|
|
LEAL -60(R10), R10
|
|
ADDQ $0x03, CX
|
|
|
|
// emitRepeat
|
|
MOVL R10, SI
|
|
LEAL -4(R10), R10
|
|
CMPL SI, $0x08
|
|
JBE repeat_two_match_nolit_encodeBlockAsm8B_emit_copy_short
|
|
CMPL SI, $0x0c
|
|
JAE cant_repeat_two_offset_match_nolit_encodeBlockAsm8B_emit_copy_short
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBlockAsm8B_emit_copy_short:
|
|
CMPL R10, $0x00000104
|
|
JB repeat_three_match_nolit_encodeBlockAsm8B_emit_copy_short
|
|
LEAL -256(R10), R10
|
|
MOVW $0x0019, (CX)
|
|
MOVW R10, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm8B
|
|
|
|
repeat_three_match_nolit_encodeBlockAsm8B_emit_copy_short:
|
|
LEAL -4(R10), R10
|
|
MOVW $0x0015, (CX)
|
|
MOVB R10, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm8B
|
|
|
|
repeat_two_match_nolit_encodeBlockAsm8B_emit_copy_short:
|
|
SHLL $0x02, R10
|
|
ORL $0x01, R10
|
|
MOVW R10, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm8B
|
|
XORQ DI, DI
|
|
LEAL 1(DI)(R10*4), R10
|
|
MOVB SI, 1(CX)
|
|
SARL $0x08, SI
|
|
SHLL $0x05, SI
|
|
ORL SI, R10
|
|
MOVB R10, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm8B
|
|
|
|
two_byte_offset_short_match_nolit_encodeBlockAsm8B:
|
|
MOVL R10, DI
|
|
SHLL $0x02, DI
|
|
CMPL R10, $0x0c
|
|
JAE emit_copy_three_match_nolit_encodeBlockAsm8B
|
|
LEAL -15(DI), DI
|
|
MOVB SI, 1(CX)
|
|
SHRL $0x08, SI
|
|
SHLL $0x05, SI
|
|
ORL SI, DI
|
|
MOVB DI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm8B
|
|
|
|
emit_copy_three_match_nolit_encodeBlockAsm8B:
|
|
LEAL -2(DI), DI
|
|
MOVB DI, (CX)
|
|
MOVW SI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
|
|
match_nolit_emitcopy_end_encodeBlockAsm8B:
|
|
CMPL DX, 8(SP)
|
|
JAE emit_remainder_encodeBlockAsm8B
|
|
MOVQ -2(BX)(DX*1), DI
|
|
CMPQ CX, (SP)
|
|
JB match_nolit_dst_ok_encodeBlockAsm8B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeBlockAsm8B:
|
|
MOVQ $0x9e3779b1, R9
|
|
MOVQ DI, R8
|
|
SHRQ $0x10, DI
|
|
MOVQ DI, SI
|
|
SHLQ $0x20, R8
|
|
IMULQ R9, R8
|
|
SHRQ $0x38, R8
|
|
SHLQ $0x20, SI
|
|
IMULQ R9, SI
|
|
SHRQ $0x38, SI
|
|
LEAL -2(DX), R9
|
|
LEAQ (AX)(SI*4), R10
|
|
MOVL (R10), SI
|
|
MOVL R9, (AX)(R8*4)
|
|
MOVL DX, (R10)
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ match_nolit_loop_encodeBlockAsm8B
|
|
INCL DX
|
|
JMP search_loop_encodeBlockAsm8B
|
|
|
|
emit_remainder_encodeBlockAsm8B:
|
|
MOVQ src_len+32(FP), AX
|
|
SUBL 12(SP), AX
|
|
LEAQ 3(CX)(AX*1), AX
|
|
CMPQ AX, (SP)
|
|
JB emit_remainder_ok_encodeBlockAsm8B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeBlockAsm8B:
|
|
MOVQ src_len+32(FP), AX
|
|
MOVL 12(SP), DX
|
|
CMPL DX, AX
|
|
JEQ emit_literal_done_emit_remainder_encodeBlockAsm8B
|
|
MOVL AX, SI
|
|
MOVL AX, 12(SP)
|
|
LEAQ (BX)(DX*1), AX
|
|
SUBL DX, SI
|
|
LEAL -1(SI), DX
|
|
CMPL DX, $0x3c
|
|
JB one_byte_emit_remainder_encodeBlockAsm8B
|
|
CMPL DX, $0x00000100
|
|
JB two_bytes_emit_remainder_encodeBlockAsm8B
|
|
JB three_bytes_emit_remainder_encodeBlockAsm8B
|
|
|
|
three_bytes_emit_remainder_encodeBlockAsm8B:
|
|
MOVB $0xf4, (CX)
|
|
MOVW DX, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_emit_remainder_encodeBlockAsm8B
|
|
|
|
two_bytes_emit_remainder_encodeBlockAsm8B:
|
|
MOVB $0xf0, (CX)
|
|
MOVB DL, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL DX, $0x40
|
|
JB memmove_emit_remainder_encodeBlockAsm8B
|
|
JMP memmove_long_emit_remainder_encodeBlockAsm8B
|
|
|
|
one_byte_emit_remainder_encodeBlockAsm8B:
|
|
SHLB $0x02, DL
|
|
MOVB DL, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_emit_remainder_encodeBlockAsm8B:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveShort
|
|
CMPQ BX, $0x03
|
|
JB emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_1or2
|
|
JE emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_3
|
|
CMPQ BX, $0x08
|
|
JB emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_4through7
|
|
CMPQ BX, $0x10
|
|
JBE emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_8through16
|
|
CMPQ BX, $0x20
|
|
JBE emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_1or2:
|
|
MOVB (AX), SI
|
|
MOVB -1(AX)(BX*1), AL
|
|
MOVB SI, (CX)
|
|
MOVB AL, -1(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_3:
|
|
MOVW (AX), SI
|
|
MOVB 2(AX), AL
|
|
MOVW SI, (CX)
|
|
MOVB AL, 2(CX)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_4through7:
|
|
MOVL (AX), SI
|
|
MOVL -4(AX)(BX*1), AX
|
|
MOVL SI, (CX)
|
|
MOVL AX, -4(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_8through16:
|
|
MOVQ (AX), SI
|
|
MOVQ -8(AX)(BX*1), AX
|
|
MOVQ SI, (CX)
|
|
MOVQ AX, -8(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_17through32:
|
|
MOVOU (AX), X0
|
|
MOVOU -16(AX)(BX*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_33through64:
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
|
|
memmove_end_copy_emit_remainder_encodeBlockAsm8B:
|
|
MOVQ DX, CX
|
|
JMP emit_literal_done_emit_remainder_encodeBlockAsm8B
|
|
|
|
memmove_long_emit_remainder_encodeBlockAsm8B:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveLong
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVQ BX, DI
|
|
SHRQ $0x05, DI
|
|
MOVQ CX, SI
|
|
ANDL $0x0000001f, SI
|
|
MOVQ $0x00000040, R8
|
|
SUBQ SI, R8
|
|
DECQ DI
|
|
JA emit_lit_memmove_long_emit_remainder_encodeBlockAsm8Blarge_forward_sse_loop_32
|
|
LEAQ -32(AX)(R8*1), SI
|
|
LEAQ -32(CX)(R8*1), R9
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeBlockAsm8Blarge_big_loop_back:
|
|
MOVOU (SI), X4
|
|
MOVOU 16(SI), X5
|
|
MOVOA X4, (R9)
|
|
MOVOA X5, 16(R9)
|
|
ADDQ $0x20, R9
|
|
ADDQ $0x20, SI
|
|
ADDQ $0x20, R8
|
|
DECQ DI
|
|
JNA emit_lit_memmove_long_emit_remainder_encodeBlockAsm8Blarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeBlockAsm8Blarge_forward_sse_loop_32:
|
|
MOVOU -32(AX)(R8*1), X4
|
|
MOVOU -16(AX)(R8*1), X5
|
|
MOVOA X4, -32(CX)(R8*1)
|
|
MOVOA X5, -16(CX)(R8*1)
|
|
ADDQ $0x20, R8
|
|
CMPQ BX, R8
|
|
JAE emit_lit_memmove_long_emit_remainder_encodeBlockAsm8Blarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
MOVQ DX, CX
|
|
|
|
emit_literal_done_emit_remainder_encodeBlockAsm8B:
|
|
MOVQ dst_base+0(FP), AX
|
|
SUBQ AX, CX
|
|
MOVQ CX, ret+56(FP)
|
|
RET
|
|
|
|
// func encodeBetterBlockAsm(dst []byte, src []byte, tmp *[589824]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeBetterBlockAsm(SB), $24-64
|
|
MOVQ tmp+48(FP), AX
|
|
MOVQ dst_base+0(FP), CX
|
|
MOVQ $0x00001200, DX
|
|
MOVQ AX, BX
|
|
PXOR X0, X0
|
|
|
|
zero_loop_encodeBetterBlockAsm:
|
|
MOVOU X0, (BX)
|
|
MOVOU X0, 16(BX)
|
|
MOVOU X0, 32(BX)
|
|
MOVOU X0, 48(BX)
|
|
MOVOU X0, 64(BX)
|
|
MOVOU X0, 80(BX)
|
|
MOVOU X0, 96(BX)
|
|
MOVOU X0, 112(BX)
|
|
ADDQ $0x80, BX
|
|
DECQ DX
|
|
JNZ zero_loop_encodeBetterBlockAsm
|
|
MOVL $0x00000000, 12(SP)
|
|
MOVQ src_len+32(FP), DX
|
|
LEAQ -6(DX), BX
|
|
LEAQ -8(DX), SI
|
|
MOVL SI, 8(SP)
|
|
SHRQ $0x05, DX
|
|
SUBL DX, BX
|
|
LEAQ (CX)(BX*1), BX
|
|
MOVQ BX, (SP)
|
|
MOVL $0x00000001, DX
|
|
MOVL $0x00000000, 16(SP)
|
|
MOVQ src_base+24(FP), BX
|
|
|
|
search_loop_encodeBetterBlockAsm:
|
|
MOVL DX, SI
|
|
SUBL 12(SP), SI
|
|
SHRL $0x07, SI
|
|
CMPL SI, $0x63
|
|
JBE check_maxskip_ok_encodeBetterBlockAsm
|
|
LEAL 100(DX), SI
|
|
JMP check_maxskip_cont_encodeBetterBlockAsm
|
|
|
|
check_maxskip_ok_encodeBetterBlockAsm:
|
|
LEAL 1(DX)(SI*1), SI
|
|
|
|
check_maxskip_cont_encodeBetterBlockAsm:
|
|
CMPL SI, 8(SP)
|
|
JAE emit_remainder_encodeBetterBlockAsm
|
|
MOVQ (BX)(DX*1), DI
|
|
MOVL SI, 20(SP)
|
|
MOVQ $0x00cf1bbcdcbfa563, R9
|
|
MOVQ $0x9e3779b1, SI
|
|
MOVQ DI, R10
|
|
MOVQ DI, R11
|
|
SHLQ $0x08, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x2f, R10
|
|
SHLQ $0x20, R11
|
|
IMULQ SI, R11
|
|
SHRQ $0x32, R11
|
|
MOVL (AX)(R10*4), SI
|
|
MOVL 524288(AX)(R11*4), R8
|
|
MOVL DX, (AX)(R10*4)
|
|
MOVL DX, 524288(AX)(R11*4)
|
|
MOVQ (BX)(SI*1), R10
|
|
MOVQ (BX)(R8*1), R11
|
|
CMPQ R10, DI
|
|
JEQ candidate_match_encodeBetterBlockAsm
|
|
CMPQ R11, DI
|
|
JNE no_short_found_encodeBetterBlockAsm
|
|
MOVL R8, SI
|
|
JMP candidate_match_encodeBetterBlockAsm
|
|
|
|
no_short_found_encodeBetterBlockAsm:
|
|
CMPL R10, DI
|
|
JEQ candidate_match_encodeBetterBlockAsm
|
|
CMPL R11, DI
|
|
JEQ candidateS_match_encodeBetterBlockAsm
|
|
MOVL 20(SP), DX
|
|
JMP search_loop_encodeBetterBlockAsm
|
|
|
|
candidateS_match_encodeBetterBlockAsm:
|
|
SHRQ $0x08, DI
|
|
MOVQ DI, R10
|
|
SHLQ $0x08, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x2f, R10
|
|
MOVL (AX)(R10*4), SI
|
|
INCL DX
|
|
MOVL DX, (AX)(R10*4)
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ candidate_match_encodeBetterBlockAsm
|
|
DECL DX
|
|
MOVL R8, SI
|
|
|
|
candidate_match_encodeBetterBlockAsm:
|
|
MOVL 12(SP), DI
|
|
TESTL SI, SI
|
|
JZ match_extend_back_end_encodeBetterBlockAsm
|
|
|
|
match_extend_back_loop_encodeBetterBlockAsm:
|
|
CMPL DX, DI
|
|
JBE match_extend_back_end_encodeBetterBlockAsm
|
|
MOVB -1(BX)(SI*1), R8
|
|
MOVB -1(BX)(DX*1), R9
|
|
CMPB R8, R9
|
|
JNE match_extend_back_end_encodeBetterBlockAsm
|
|
LEAL -1(DX), DX
|
|
DECL SI
|
|
JZ match_extend_back_end_encodeBetterBlockAsm
|
|
JMP match_extend_back_loop_encodeBetterBlockAsm
|
|
|
|
match_extend_back_end_encodeBetterBlockAsm:
|
|
MOVL DX, DI
|
|
SUBL 12(SP), DI
|
|
LEAQ 5(CX)(DI*1), DI
|
|
CMPQ DI, (SP)
|
|
JB match_dst_size_check_encodeBetterBlockAsm
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeBetterBlockAsm:
|
|
MOVL DX, DI
|
|
ADDL $0x04, DX
|
|
ADDL $0x04, SI
|
|
MOVQ src_len+32(FP), R8
|
|
SUBL DX, R8
|
|
LEAQ (BX)(DX*1), R9
|
|
LEAQ (BX)(SI*1), R10
|
|
|
|
// matchLen
|
|
XORL R12, R12
|
|
|
|
matchlen_loopback_16_match_nolit_encodeBetterBlockAsm:
|
|
CMPL R8, $0x10
|
|
JB matchlen_match8_match_nolit_encodeBetterBlockAsm
|
|
MOVQ (R9)(R12*1), R11
|
|
MOVQ 8(R9)(R12*1), R13
|
|
XORQ (R10)(R12*1), R11
|
|
JNZ matchlen_bsf_8_match_nolit_encodeBetterBlockAsm
|
|
XORQ 8(R10)(R12*1), R13
|
|
JNZ matchlen_bsf_16match_nolit_encodeBetterBlockAsm
|
|
LEAL -16(R8), R8
|
|
LEAL 16(R12), R12
|
|
JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm
|
|
|
|
matchlen_bsf_16match_nolit_encodeBetterBlockAsm:
|
|
TZCNTQ R13, R13
|
|
SARQ $0x03, R13
|
|
LEAL 8(R12)(R13*1), R12
|
|
JMP match_nolit_end_encodeBetterBlockAsm
|
|
|
|
matchlen_match8_match_nolit_encodeBetterBlockAsm:
|
|
CMPL R8, $0x08
|
|
JB matchlen_match4_match_nolit_encodeBetterBlockAsm
|
|
MOVQ (R9)(R12*1), R11
|
|
XORQ (R10)(R12*1), R11
|
|
JNZ matchlen_bsf_8_match_nolit_encodeBetterBlockAsm
|
|
LEAL -8(R8), R8
|
|
LEAL 8(R12), R12
|
|
JMP matchlen_match4_match_nolit_encodeBetterBlockAsm
|
|
|
|
matchlen_bsf_8_match_nolit_encodeBetterBlockAsm:
|
|
TZCNTQ R11, R11
|
|
SARQ $0x03, R11
|
|
LEAL (R12)(R11*1), R12
|
|
JMP match_nolit_end_encodeBetterBlockAsm
|
|
|
|
matchlen_match4_match_nolit_encodeBetterBlockAsm:
|
|
CMPL R8, $0x04
|
|
JB matchlen_match2_match_nolit_encodeBetterBlockAsm
|
|
MOVL (R9)(R12*1), R11
|
|
CMPL (R10)(R12*1), R11
|
|
JNE matchlen_match2_match_nolit_encodeBetterBlockAsm
|
|
LEAL -4(R8), R8
|
|
LEAL 4(R12), R12
|
|
|
|
matchlen_match2_match_nolit_encodeBetterBlockAsm:
|
|
CMPL R8, $0x01
|
|
JE matchlen_match1_match_nolit_encodeBetterBlockAsm
|
|
JB match_nolit_end_encodeBetterBlockAsm
|
|
MOVW (R9)(R12*1), R11
|
|
CMPW (R10)(R12*1), R11
|
|
JNE matchlen_match1_match_nolit_encodeBetterBlockAsm
|
|
LEAL 2(R12), R12
|
|
SUBL $0x02, R8
|
|
JZ match_nolit_end_encodeBetterBlockAsm
|
|
|
|
matchlen_match1_match_nolit_encodeBetterBlockAsm:
|
|
MOVB (R9)(R12*1), R11
|
|
CMPB (R10)(R12*1), R11
|
|
JNE match_nolit_end_encodeBetterBlockAsm
|
|
LEAL 1(R12), R12
|
|
|
|
match_nolit_end_encodeBetterBlockAsm:
|
|
MOVL DX, R8
|
|
SUBL SI, R8
|
|
|
|
// Check if repeat
|
|
CMPL 16(SP), R8
|
|
JEQ match_is_repeat_encodeBetterBlockAsm
|
|
CMPL R12, $0x01
|
|
JA match_length_ok_encodeBetterBlockAsm
|
|
CMPL R8, $0x0000ffff
|
|
JBE match_length_ok_encodeBetterBlockAsm
|
|
MOVL 20(SP), DX
|
|
INCL DX
|
|
JMP search_loop_encodeBetterBlockAsm
|
|
|
|
match_length_ok_encodeBetterBlockAsm:
|
|
MOVL R8, 16(SP)
|
|
MOVL 12(SP), SI
|
|
CMPL SI, DI
|
|
JEQ emit_literal_done_match_emit_encodeBetterBlockAsm
|
|
MOVL DI, R9
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(SI*1), R10
|
|
SUBL SI, R9
|
|
LEAL -1(R9), SI
|
|
CMPL SI, $0x3c
|
|
JB one_byte_match_emit_encodeBetterBlockAsm
|
|
CMPL SI, $0x00000100
|
|
JB two_bytes_match_emit_encodeBetterBlockAsm
|
|
CMPL SI, $0x00010000
|
|
JB three_bytes_match_emit_encodeBetterBlockAsm
|
|
CMPL SI, $0x01000000
|
|
JB four_bytes_match_emit_encodeBetterBlockAsm
|
|
MOVB $0xfc, (CX)
|
|
MOVL SI, 1(CX)
|
|
ADDQ $0x05, CX
|
|
JMP memmove_long_match_emit_encodeBetterBlockAsm
|
|
|
|
four_bytes_match_emit_encodeBetterBlockAsm:
|
|
MOVL SI, R11
|
|
SHRL $0x10, R11
|
|
MOVB $0xf8, (CX)
|
|
MOVW SI, 1(CX)
|
|
MOVB R11, 3(CX)
|
|
ADDQ $0x04, CX
|
|
JMP memmove_long_match_emit_encodeBetterBlockAsm
|
|
|
|
three_bytes_match_emit_encodeBetterBlockAsm:
|
|
MOVB $0xf4, (CX)
|
|
MOVW SI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_match_emit_encodeBetterBlockAsm
|
|
|
|
two_bytes_match_emit_encodeBetterBlockAsm:
|
|
MOVB $0xf0, (CX)
|
|
MOVB SI, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL SI, $0x40
|
|
JB memmove_match_emit_encodeBetterBlockAsm
|
|
JMP memmove_long_match_emit_encodeBetterBlockAsm
|
|
|
|
one_byte_match_emit_encodeBetterBlockAsm:
|
|
SHLB $0x02, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_match_emit_encodeBetterBlockAsm:
|
|
LEAQ (CX)(R9*1), SI
|
|
|
|
// genMemMoveShort
|
|
CMPQ R9, $0x04
|
|
JBE emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_4
|
|
CMPQ R9, $0x08
|
|
JB emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_4through7
|
|
CMPQ R9, $0x10
|
|
JBE emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_8through16
|
|
CMPQ R9, $0x20
|
|
JBE emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_4:
|
|
MOVL (R10), R11
|
|
MOVL R11, (CX)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_4through7:
|
|
MOVL (R10), R11
|
|
MOVL -4(R10)(R9*1), R10
|
|
MOVL R11, (CX)
|
|
MOVL R10, -4(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_8through16:
|
|
MOVQ (R10), R11
|
|
MOVQ -8(R10)(R9*1), R10
|
|
MOVQ R11, (CX)
|
|
MOVQ R10, -8(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_17through32:
|
|
MOVOU (R10), X0
|
|
MOVOU -16(R10)(R9*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_33through64:
|
|
MOVOU (R10), X0
|
|
MOVOU 16(R10), X1
|
|
MOVOU -32(R10)(R9*1), X2
|
|
MOVOU -16(R10)(R9*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
|
|
memmove_end_copy_match_emit_encodeBetterBlockAsm:
|
|
MOVQ SI, CX
|
|
JMP emit_literal_done_match_emit_encodeBetterBlockAsm
|
|
|
|
memmove_long_match_emit_encodeBetterBlockAsm:
|
|
LEAQ (CX)(R9*1), SI
|
|
|
|
// genMemMoveLong
|
|
MOVOU (R10), X0
|
|
MOVOU 16(R10), X1
|
|
MOVOU -32(R10)(R9*1), X2
|
|
MOVOU -16(R10)(R9*1), X3
|
|
MOVQ R9, R13
|
|
SHRQ $0x05, R13
|
|
MOVQ CX, R11
|
|
ANDL $0x0000001f, R11
|
|
MOVQ $0x00000040, R14
|
|
SUBQ R11, R14
|
|
DECQ R13
|
|
JA emit_lit_memmove_long_match_emit_encodeBetterBlockAsmlarge_forward_sse_loop_32
|
|
LEAQ -32(R10)(R14*1), R11
|
|
LEAQ -32(CX)(R14*1), R15
|
|
|
|
emit_lit_memmove_long_match_emit_encodeBetterBlockAsmlarge_big_loop_back:
|
|
MOVOU (R11), X4
|
|
MOVOU 16(R11), X5
|
|
MOVOA X4, (R15)
|
|
MOVOA X5, 16(R15)
|
|
ADDQ $0x20, R15
|
|
ADDQ $0x20, R11
|
|
ADDQ $0x20, R14
|
|
DECQ R13
|
|
JNA emit_lit_memmove_long_match_emit_encodeBetterBlockAsmlarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_match_emit_encodeBetterBlockAsmlarge_forward_sse_loop_32:
|
|
MOVOU -32(R10)(R14*1), X4
|
|
MOVOU -16(R10)(R14*1), X5
|
|
MOVOA X4, -32(CX)(R14*1)
|
|
MOVOA X5, -16(CX)(R14*1)
|
|
ADDQ $0x20, R14
|
|
CMPQ R9, R14
|
|
JAE emit_lit_memmove_long_match_emit_encodeBetterBlockAsmlarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
MOVQ SI, CX
|
|
|
|
emit_literal_done_match_emit_encodeBetterBlockAsm:
|
|
ADDL R12, DX
|
|
ADDL $0x04, R12
|
|
MOVL DX, 12(SP)
|
|
|
|
// emitCopy
|
|
CMPL R8, $0x00010000
|
|
JB two_byte_offset_match_nolit_encodeBetterBlockAsm
|
|
CMPL R12, $0x40
|
|
JBE four_bytes_remain_match_nolit_encodeBetterBlockAsm
|
|
MOVB $0xff, (CX)
|
|
MOVL R8, 1(CX)
|
|
LEAL -64(R12), R12
|
|
ADDQ $0x05, CX
|
|
CMPL R12, $0x04
|
|
JB four_bytes_remain_match_nolit_encodeBetterBlockAsm
|
|
|
|
// emitRepeat
|
|
emit_repeat_again_match_nolit_encodeBetterBlockAsm_emit_copy:
|
|
MOVL R12, SI
|
|
LEAL -4(R12), R12
|
|
CMPL SI, $0x08
|
|
JBE repeat_two_match_nolit_encodeBetterBlockAsm_emit_copy
|
|
CMPL SI, $0x0c
|
|
JAE cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy
|
|
CMPL R8, $0x00000800
|
|
JB repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy:
|
|
CMPL R12, $0x00000104
|
|
JB repeat_three_match_nolit_encodeBetterBlockAsm_emit_copy
|
|
CMPL R12, $0x00010100
|
|
JB repeat_four_match_nolit_encodeBetterBlockAsm_emit_copy
|
|
CMPL R12, $0x0100ffff
|
|
JB repeat_five_match_nolit_encodeBetterBlockAsm_emit_copy
|
|
LEAL -16842747(R12), R12
|
|
MOVL $0xfffb001d, (CX)
|
|
MOVB $0xff, 4(CX)
|
|
ADDQ $0x05, CX
|
|
JMP emit_repeat_again_match_nolit_encodeBetterBlockAsm_emit_copy
|
|
|
|
repeat_five_match_nolit_encodeBetterBlockAsm_emit_copy:
|
|
LEAL -65536(R12), R12
|
|
MOVL R12, R8
|
|
MOVW $0x001d, (CX)
|
|
MOVW R12, 2(CX)
|
|
SARL $0x10, R8
|
|
MOVB R8, 4(CX)
|
|
ADDQ $0x05, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
repeat_four_match_nolit_encodeBetterBlockAsm_emit_copy:
|
|
LEAL -256(R12), R12
|
|
MOVW $0x0019, (CX)
|
|
MOVW R12, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
repeat_three_match_nolit_encodeBetterBlockAsm_emit_copy:
|
|
LEAL -4(R12), R12
|
|
MOVW $0x0015, (CX)
|
|
MOVB R12, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
repeat_two_match_nolit_encodeBetterBlockAsm_emit_copy:
|
|
SHLL $0x02, R12
|
|
ORL $0x01, R12
|
|
MOVW R12, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy:
|
|
XORQ SI, SI
|
|
LEAL 1(SI)(R12*4), R12
|
|
MOVB R8, 1(CX)
|
|
SARL $0x08, R8
|
|
SHLL $0x05, R8
|
|
ORL R8, R12
|
|
MOVB R12, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
four_bytes_remain_match_nolit_encodeBetterBlockAsm:
|
|
TESTL R12, R12
|
|
JZ match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
XORL SI, SI
|
|
LEAL -1(SI)(R12*4), R12
|
|
MOVB R12, (CX)
|
|
MOVL R8, 1(CX)
|
|
ADDQ $0x05, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
two_byte_offset_match_nolit_encodeBetterBlockAsm:
|
|
CMPL R12, $0x40
|
|
JBE two_byte_offset_short_match_nolit_encodeBetterBlockAsm
|
|
CMPL R8, $0x00000800
|
|
JAE long_offset_short_match_nolit_encodeBetterBlockAsm
|
|
MOVL $0x00000001, SI
|
|
LEAL 16(SI), SI
|
|
MOVB R8, 1(CX)
|
|
MOVL R8, R9
|
|
SHRL $0x08, R9
|
|
SHLL $0x05, R9
|
|
ORL R9, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x02, CX
|
|
SUBL $0x08, R12
|
|
|
|
// emitRepeat
|
|
LEAL -4(R12), R12
|
|
JMP cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b
|
|
|
|
emit_repeat_again_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b:
|
|
MOVL R12, SI
|
|
LEAL -4(R12), R12
|
|
CMPL SI, $0x08
|
|
JBE repeat_two_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b
|
|
CMPL SI, $0x0c
|
|
JAE cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b
|
|
CMPL R8, $0x00000800
|
|
JB repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b:
|
|
CMPL R12, $0x00000104
|
|
JB repeat_three_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b
|
|
CMPL R12, $0x00010100
|
|
JB repeat_four_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b
|
|
CMPL R12, $0x0100ffff
|
|
JB repeat_five_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b
|
|
LEAL -16842747(R12), R12
|
|
MOVL $0xfffb001d, (CX)
|
|
MOVB $0xff, 4(CX)
|
|
ADDQ $0x05, CX
|
|
JMP emit_repeat_again_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b
|
|
|
|
repeat_five_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b:
|
|
LEAL -65536(R12), R12
|
|
MOVL R12, R8
|
|
MOVW $0x001d, (CX)
|
|
MOVW R12, 2(CX)
|
|
SARL $0x10, R8
|
|
MOVB R8, 4(CX)
|
|
ADDQ $0x05, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
repeat_four_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b:
|
|
LEAL -256(R12), R12
|
|
MOVW $0x0019, (CX)
|
|
MOVW R12, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
repeat_three_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b:
|
|
LEAL -4(R12), R12
|
|
MOVW $0x0015, (CX)
|
|
MOVB R12, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
repeat_two_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b:
|
|
SHLL $0x02, R12
|
|
ORL $0x01, R12
|
|
MOVW R12, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b:
|
|
XORQ SI, SI
|
|
LEAL 1(SI)(R12*4), R12
|
|
MOVB R8, 1(CX)
|
|
SARL $0x08, R8
|
|
SHLL $0x05, R8
|
|
ORL R8, R12
|
|
MOVB R12, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
long_offset_short_match_nolit_encodeBetterBlockAsm:
|
|
MOVB $0xee, (CX)
|
|
MOVW R8, 1(CX)
|
|
LEAL -60(R12), R12
|
|
ADDQ $0x03, CX
|
|
|
|
// emitRepeat
|
|
emit_repeat_again_match_nolit_encodeBetterBlockAsm_emit_copy_short:
|
|
MOVL R12, SI
|
|
LEAL -4(R12), R12
|
|
CMPL SI, $0x08
|
|
JBE repeat_two_match_nolit_encodeBetterBlockAsm_emit_copy_short
|
|
CMPL SI, $0x0c
|
|
JAE cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short
|
|
CMPL R8, $0x00000800
|
|
JB repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short:
|
|
CMPL R12, $0x00000104
|
|
JB repeat_three_match_nolit_encodeBetterBlockAsm_emit_copy_short
|
|
CMPL R12, $0x00010100
|
|
JB repeat_four_match_nolit_encodeBetterBlockAsm_emit_copy_short
|
|
CMPL R12, $0x0100ffff
|
|
JB repeat_five_match_nolit_encodeBetterBlockAsm_emit_copy_short
|
|
LEAL -16842747(R12), R12
|
|
MOVL $0xfffb001d, (CX)
|
|
MOVB $0xff, 4(CX)
|
|
ADDQ $0x05, CX
|
|
JMP emit_repeat_again_match_nolit_encodeBetterBlockAsm_emit_copy_short
|
|
|
|
repeat_five_match_nolit_encodeBetterBlockAsm_emit_copy_short:
|
|
LEAL -65536(R12), R12
|
|
MOVL R12, R8
|
|
MOVW $0x001d, (CX)
|
|
MOVW R12, 2(CX)
|
|
SARL $0x10, R8
|
|
MOVB R8, 4(CX)
|
|
ADDQ $0x05, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
repeat_four_match_nolit_encodeBetterBlockAsm_emit_copy_short:
|
|
LEAL -256(R12), R12
|
|
MOVW $0x0019, (CX)
|
|
MOVW R12, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
repeat_three_match_nolit_encodeBetterBlockAsm_emit_copy_short:
|
|
LEAL -4(R12), R12
|
|
MOVW $0x0015, (CX)
|
|
MOVB R12, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
repeat_two_match_nolit_encodeBetterBlockAsm_emit_copy_short:
|
|
SHLL $0x02, R12
|
|
ORL $0x01, R12
|
|
MOVW R12, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short:
|
|
XORQ SI, SI
|
|
LEAL 1(SI)(R12*4), R12
|
|
MOVB R8, 1(CX)
|
|
SARL $0x08, R8
|
|
SHLL $0x05, R8
|
|
ORL R8, R12
|
|
MOVB R12, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
two_byte_offset_short_match_nolit_encodeBetterBlockAsm:
|
|
MOVL R12, SI
|
|
SHLL $0x02, SI
|
|
CMPL R12, $0x0c
|
|
JAE emit_copy_three_match_nolit_encodeBetterBlockAsm
|
|
CMPL R8, $0x00000800
|
|
JAE emit_copy_three_match_nolit_encodeBetterBlockAsm
|
|
LEAL -15(SI), SI
|
|
MOVB R8, 1(CX)
|
|
SHRL $0x08, R8
|
|
SHLL $0x05, R8
|
|
ORL R8, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
emit_copy_three_match_nolit_encodeBetterBlockAsm:
|
|
LEAL -2(SI), SI
|
|
MOVB SI, (CX)
|
|
MOVW R8, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
match_is_repeat_encodeBetterBlockAsm:
|
|
MOVL 12(SP), SI
|
|
CMPL SI, DI
|
|
JEQ emit_literal_done_match_emit_repeat_encodeBetterBlockAsm
|
|
MOVL DI, R9
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(SI*1), R10
|
|
SUBL SI, R9
|
|
LEAL -1(R9), SI
|
|
CMPL SI, $0x3c
|
|
JB one_byte_match_emit_repeat_encodeBetterBlockAsm
|
|
CMPL SI, $0x00000100
|
|
JB two_bytes_match_emit_repeat_encodeBetterBlockAsm
|
|
CMPL SI, $0x00010000
|
|
JB three_bytes_match_emit_repeat_encodeBetterBlockAsm
|
|
CMPL SI, $0x01000000
|
|
JB four_bytes_match_emit_repeat_encodeBetterBlockAsm
|
|
MOVB $0xfc, (CX)
|
|
MOVL SI, 1(CX)
|
|
ADDQ $0x05, CX
|
|
JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm
|
|
|
|
four_bytes_match_emit_repeat_encodeBetterBlockAsm:
|
|
MOVL SI, R11
|
|
SHRL $0x10, R11
|
|
MOVB $0xf8, (CX)
|
|
MOVW SI, 1(CX)
|
|
MOVB R11, 3(CX)
|
|
ADDQ $0x04, CX
|
|
JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm
|
|
|
|
three_bytes_match_emit_repeat_encodeBetterBlockAsm:
|
|
MOVB $0xf4, (CX)
|
|
MOVW SI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm
|
|
|
|
two_bytes_match_emit_repeat_encodeBetterBlockAsm:
|
|
MOVB $0xf0, (CX)
|
|
MOVB SI, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL SI, $0x40
|
|
JB memmove_match_emit_repeat_encodeBetterBlockAsm
|
|
JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm
|
|
|
|
one_byte_match_emit_repeat_encodeBetterBlockAsm:
|
|
SHLB $0x02, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_match_emit_repeat_encodeBetterBlockAsm:
|
|
LEAQ (CX)(R9*1), SI
|
|
|
|
// genMemMoveShort
|
|
CMPQ R9, $0x04
|
|
JBE emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_4
|
|
CMPQ R9, $0x08
|
|
JB emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_4through7
|
|
CMPQ R9, $0x10
|
|
JBE emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_8through16
|
|
CMPQ R9, $0x20
|
|
JBE emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_4:
|
|
MOVL (R10), R11
|
|
MOVL R11, (CX)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_4through7:
|
|
MOVL (R10), R11
|
|
MOVL -4(R10)(R9*1), R10
|
|
MOVL R11, (CX)
|
|
MOVL R10, -4(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_8through16:
|
|
MOVQ (R10), R11
|
|
MOVQ -8(R10)(R9*1), R10
|
|
MOVQ R11, (CX)
|
|
MOVQ R10, -8(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_17through32:
|
|
MOVOU (R10), X0
|
|
MOVOU -16(R10)(R9*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_33through64:
|
|
MOVOU (R10), X0
|
|
MOVOU 16(R10), X1
|
|
MOVOU -32(R10)(R9*1), X2
|
|
MOVOU -16(R10)(R9*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
|
|
memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm:
|
|
MOVQ SI, CX
|
|
JMP emit_literal_done_match_emit_repeat_encodeBetterBlockAsm
|
|
|
|
memmove_long_match_emit_repeat_encodeBetterBlockAsm:
|
|
LEAQ (CX)(R9*1), SI
|
|
|
|
// genMemMoveLong
|
|
MOVOU (R10), X0
|
|
MOVOU 16(R10), X1
|
|
MOVOU -32(R10)(R9*1), X2
|
|
MOVOU -16(R10)(R9*1), X3
|
|
MOVQ R9, R13
|
|
SHRQ $0x05, R13
|
|
MOVQ CX, R11
|
|
ANDL $0x0000001f, R11
|
|
MOVQ $0x00000040, R14
|
|
SUBQ R11, R14
|
|
DECQ R13
|
|
JA emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsmlarge_forward_sse_loop_32
|
|
LEAQ -32(R10)(R14*1), R11
|
|
LEAQ -32(CX)(R14*1), R15
|
|
|
|
emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsmlarge_big_loop_back:
|
|
MOVOU (R11), X4
|
|
MOVOU 16(R11), X5
|
|
MOVOA X4, (R15)
|
|
MOVOA X5, 16(R15)
|
|
ADDQ $0x20, R15
|
|
ADDQ $0x20, R11
|
|
ADDQ $0x20, R14
|
|
DECQ R13
|
|
JNA emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsmlarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsmlarge_forward_sse_loop_32:
|
|
MOVOU -32(R10)(R14*1), X4
|
|
MOVOU -16(R10)(R14*1), X5
|
|
MOVOA X4, -32(CX)(R14*1)
|
|
MOVOA X5, -16(CX)(R14*1)
|
|
ADDQ $0x20, R14
|
|
CMPQ R9, R14
|
|
JAE emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsmlarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
MOVQ SI, CX
|
|
|
|
emit_literal_done_match_emit_repeat_encodeBetterBlockAsm:
|
|
ADDL R12, DX
|
|
ADDL $0x04, R12
|
|
MOVL DX, 12(SP)
|
|
|
|
// emitRepeat
|
|
emit_repeat_again_match_nolit_repeat_encodeBetterBlockAsm:
|
|
MOVL R12, SI
|
|
LEAL -4(R12), R12
|
|
CMPL SI, $0x08
|
|
JBE repeat_two_match_nolit_repeat_encodeBetterBlockAsm
|
|
CMPL SI, $0x0c
|
|
JAE cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm
|
|
CMPL R8, $0x00000800
|
|
JB repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm
|
|
|
|
cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm:
|
|
CMPL R12, $0x00000104
|
|
JB repeat_three_match_nolit_repeat_encodeBetterBlockAsm
|
|
CMPL R12, $0x00010100
|
|
JB repeat_four_match_nolit_repeat_encodeBetterBlockAsm
|
|
CMPL R12, $0x0100ffff
|
|
JB repeat_five_match_nolit_repeat_encodeBetterBlockAsm
|
|
LEAL -16842747(R12), R12
|
|
MOVL $0xfffb001d, (CX)
|
|
MOVB $0xff, 4(CX)
|
|
ADDQ $0x05, CX
|
|
JMP emit_repeat_again_match_nolit_repeat_encodeBetterBlockAsm
|
|
|
|
repeat_five_match_nolit_repeat_encodeBetterBlockAsm:
|
|
LEAL -65536(R12), R12
|
|
MOVL R12, R8
|
|
MOVW $0x001d, (CX)
|
|
MOVW R12, 2(CX)
|
|
SARL $0x10, R8
|
|
MOVB R8, 4(CX)
|
|
ADDQ $0x05, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
repeat_four_match_nolit_repeat_encodeBetterBlockAsm:
|
|
LEAL -256(R12), R12
|
|
MOVW $0x0019, (CX)
|
|
MOVW R12, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
repeat_three_match_nolit_repeat_encodeBetterBlockAsm:
|
|
LEAL -4(R12), R12
|
|
MOVW $0x0015, (CX)
|
|
MOVB R12, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
repeat_two_match_nolit_repeat_encodeBetterBlockAsm:
|
|
SHLL $0x02, R12
|
|
ORL $0x01, R12
|
|
MOVW R12, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm:
|
|
XORQ SI, SI
|
|
LEAL 1(SI)(R12*4), R12
|
|
MOVB R8, 1(CX)
|
|
SARL $0x08, R8
|
|
SHLL $0x05, R8
|
|
ORL R8, R12
|
|
MOVB R12, (CX)
|
|
ADDQ $0x02, CX
|
|
|
|
match_nolit_emitcopy_end_encodeBetterBlockAsm:
|
|
CMPL DX, 8(SP)
|
|
JAE emit_remainder_encodeBetterBlockAsm
|
|
CMPQ CX, (SP)
|
|
JB match_nolit_dst_ok_encodeBetterBlockAsm
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeBetterBlockAsm:
|
|
MOVQ $0x00cf1bbcdcbfa563, SI
|
|
MOVQ $0x9e3779b1, R8
|
|
LEAQ 1(DI), DI
|
|
LEAQ -2(DX), R9
|
|
MOVQ (BX)(DI*1), R10
|
|
MOVQ 1(BX)(DI*1), R11
|
|
MOVQ (BX)(R9*1), R12
|
|
MOVQ 1(BX)(R9*1), R13
|
|
SHLQ $0x08, R10
|
|
IMULQ SI, R10
|
|
SHRQ $0x2f, R10
|
|
SHLQ $0x20, R11
|
|
IMULQ R8, R11
|
|
SHRQ $0x32, R11
|
|
SHLQ $0x08, R12
|
|
IMULQ SI, R12
|
|
SHRQ $0x2f, R12
|
|
SHLQ $0x20, R13
|
|
IMULQ R8, R13
|
|
SHRQ $0x32, R13
|
|
LEAQ 1(DI), R8
|
|
LEAQ 1(R9), R14
|
|
MOVL DI, (AX)(R10*4)
|
|
MOVL R9, (AX)(R12*4)
|
|
MOVL R8, 524288(AX)(R11*4)
|
|
MOVL R14, 524288(AX)(R13*4)
|
|
LEAQ 1(R9)(DI*1), R8
|
|
SHRQ $0x01, R8
|
|
ADDQ $0x01, DI
|
|
SUBQ $0x01, R9
|
|
|
|
index_loop_encodeBetterBlockAsm:
|
|
CMPQ R8, R9
|
|
JAE search_loop_encodeBetterBlockAsm
|
|
MOVQ (BX)(DI*1), R10
|
|
MOVQ (BX)(R8*1), R11
|
|
SHLQ $0x08, R10
|
|
IMULQ SI, R10
|
|
SHRQ $0x2f, R10
|
|
SHLQ $0x08, R11
|
|
IMULQ SI, R11
|
|
SHRQ $0x2f, R11
|
|
MOVL DI, (AX)(R10*4)
|
|
MOVL R8, (AX)(R11*4)
|
|
ADDQ $0x02, DI
|
|
ADDQ $0x02, R8
|
|
JMP index_loop_encodeBetterBlockAsm
|
|
|
|
emit_remainder_encodeBetterBlockAsm:
|
|
MOVQ src_len+32(FP), AX
|
|
SUBL 12(SP), AX
|
|
LEAQ 5(CX)(AX*1), AX
|
|
CMPQ AX, (SP)
|
|
JB emit_remainder_ok_encodeBetterBlockAsm
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeBetterBlockAsm:
|
|
MOVQ src_len+32(FP), AX
|
|
MOVL 12(SP), DX
|
|
CMPL DX, AX
|
|
JEQ emit_literal_done_emit_remainder_encodeBetterBlockAsm
|
|
MOVL AX, SI
|
|
MOVL AX, 12(SP)
|
|
LEAQ (BX)(DX*1), AX
|
|
SUBL DX, SI
|
|
LEAL -1(SI), DX
|
|
CMPL DX, $0x3c
|
|
JB one_byte_emit_remainder_encodeBetterBlockAsm
|
|
CMPL DX, $0x00000100
|
|
JB two_bytes_emit_remainder_encodeBetterBlockAsm
|
|
CMPL DX, $0x00010000
|
|
JB three_bytes_emit_remainder_encodeBetterBlockAsm
|
|
CMPL DX, $0x01000000
|
|
JB four_bytes_emit_remainder_encodeBetterBlockAsm
|
|
MOVB $0xfc, (CX)
|
|
MOVL DX, 1(CX)
|
|
ADDQ $0x05, CX
|
|
JMP memmove_long_emit_remainder_encodeBetterBlockAsm
|
|
|
|
four_bytes_emit_remainder_encodeBetterBlockAsm:
|
|
MOVL DX, BX
|
|
SHRL $0x10, BX
|
|
MOVB $0xf8, (CX)
|
|
MOVW DX, 1(CX)
|
|
MOVB BL, 3(CX)
|
|
ADDQ $0x04, CX
|
|
JMP memmove_long_emit_remainder_encodeBetterBlockAsm
|
|
|
|
three_bytes_emit_remainder_encodeBetterBlockAsm:
|
|
MOVB $0xf4, (CX)
|
|
MOVW DX, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_emit_remainder_encodeBetterBlockAsm
|
|
|
|
two_bytes_emit_remainder_encodeBetterBlockAsm:
|
|
MOVB $0xf0, (CX)
|
|
MOVB DL, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL DX, $0x40
|
|
JB memmove_emit_remainder_encodeBetterBlockAsm
|
|
JMP memmove_long_emit_remainder_encodeBetterBlockAsm
|
|
|
|
one_byte_emit_remainder_encodeBetterBlockAsm:
|
|
SHLB $0x02, DL
|
|
MOVB DL, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_emit_remainder_encodeBetterBlockAsm:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveShort
|
|
CMPQ BX, $0x03
|
|
JB emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_1or2
|
|
JE emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_3
|
|
CMPQ BX, $0x08
|
|
JB emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_4through7
|
|
CMPQ BX, $0x10
|
|
JBE emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_8through16
|
|
CMPQ BX, $0x20
|
|
JBE emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_1or2:
|
|
MOVB (AX), SI
|
|
MOVB -1(AX)(BX*1), AL
|
|
MOVB SI, (CX)
|
|
MOVB AL, -1(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_3:
|
|
MOVW (AX), SI
|
|
MOVB 2(AX), AL
|
|
MOVW SI, (CX)
|
|
MOVB AL, 2(CX)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_4through7:
|
|
MOVL (AX), SI
|
|
MOVL -4(AX)(BX*1), AX
|
|
MOVL SI, (CX)
|
|
MOVL AX, -4(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_8through16:
|
|
MOVQ (AX), SI
|
|
MOVQ -8(AX)(BX*1), AX
|
|
MOVQ SI, (CX)
|
|
MOVQ AX, -8(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_17through32:
|
|
MOVOU (AX), X0
|
|
MOVOU -16(AX)(BX*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_33through64:
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
|
|
memmove_end_copy_emit_remainder_encodeBetterBlockAsm:
|
|
MOVQ DX, CX
|
|
JMP emit_literal_done_emit_remainder_encodeBetterBlockAsm
|
|
|
|
memmove_long_emit_remainder_encodeBetterBlockAsm:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveLong
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVQ BX, DI
|
|
SHRQ $0x05, DI
|
|
MOVQ CX, SI
|
|
ANDL $0x0000001f, SI
|
|
MOVQ $0x00000040, R8
|
|
SUBQ SI, R8
|
|
DECQ DI
|
|
JA emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsmlarge_forward_sse_loop_32
|
|
LEAQ -32(AX)(R8*1), SI
|
|
LEAQ -32(CX)(R8*1), R9
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsmlarge_big_loop_back:
|
|
MOVOU (SI), X4
|
|
MOVOU 16(SI), X5
|
|
MOVOA X4, (R9)
|
|
MOVOA X5, 16(R9)
|
|
ADDQ $0x20, R9
|
|
ADDQ $0x20, SI
|
|
ADDQ $0x20, R8
|
|
DECQ DI
|
|
JNA emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsmlarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsmlarge_forward_sse_loop_32:
|
|
MOVOU -32(AX)(R8*1), X4
|
|
MOVOU -16(AX)(R8*1), X5
|
|
MOVOA X4, -32(CX)(R8*1)
|
|
MOVOA X5, -16(CX)(R8*1)
|
|
ADDQ $0x20, R8
|
|
CMPQ BX, R8
|
|
JAE emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsmlarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
MOVQ DX, CX
|
|
|
|
emit_literal_done_emit_remainder_encodeBetterBlockAsm:
|
|
MOVQ dst_base+0(FP), AX
|
|
SUBQ AX, CX
|
|
MOVQ CX, ret+56(FP)
|
|
RET
|
|
|
|
// func encodeBetterBlockAsm4MB(dst []byte, src []byte, tmp *[589824]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeBetterBlockAsm4MB(SB), $24-64
|
|
MOVQ tmp+48(FP), AX
|
|
MOVQ dst_base+0(FP), CX
|
|
MOVQ $0x00001200, DX
|
|
MOVQ AX, BX
|
|
PXOR X0, X0
|
|
|
|
zero_loop_encodeBetterBlockAsm4MB:
|
|
MOVOU X0, (BX)
|
|
MOVOU X0, 16(BX)
|
|
MOVOU X0, 32(BX)
|
|
MOVOU X0, 48(BX)
|
|
MOVOU X0, 64(BX)
|
|
MOVOU X0, 80(BX)
|
|
MOVOU X0, 96(BX)
|
|
MOVOU X0, 112(BX)
|
|
ADDQ $0x80, BX
|
|
DECQ DX
|
|
JNZ zero_loop_encodeBetterBlockAsm4MB
|
|
MOVL $0x00000000, 12(SP)
|
|
MOVQ src_len+32(FP), DX
|
|
LEAQ -6(DX), BX
|
|
LEAQ -8(DX), SI
|
|
MOVL SI, 8(SP)
|
|
SHRQ $0x05, DX
|
|
SUBL DX, BX
|
|
LEAQ (CX)(BX*1), BX
|
|
MOVQ BX, (SP)
|
|
MOVL $0x00000001, DX
|
|
MOVL $0x00000000, 16(SP)
|
|
MOVQ src_base+24(FP), BX
|
|
|
|
search_loop_encodeBetterBlockAsm4MB:
|
|
MOVL DX, SI
|
|
SUBL 12(SP), SI
|
|
SHRL $0x07, SI
|
|
CMPL SI, $0x63
|
|
JBE check_maxskip_ok_encodeBetterBlockAsm4MB
|
|
LEAL 100(DX), SI
|
|
JMP check_maxskip_cont_encodeBetterBlockAsm4MB
|
|
|
|
check_maxskip_ok_encodeBetterBlockAsm4MB:
|
|
LEAL 1(DX)(SI*1), SI
|
|
|
|
check_maxskip_cont_encodeBetterBlockAsm4MB:
|
|
CMPL SI, 8(SP)
|
|
JAE emit_remainder_encodeBetterBlockAsm4MB
|
|
MOVQ (BX)(DX*1), DI
|
|
MOVL SI, 20(SP)
|
|
MOVQ $0x00cf1bbcdcbfa563, R9
|
|
MOVQ $0x9e3779b1, SI
|
|
MOVQ DI, R10
|
|
MOVQ DI, R11
|
|
SHLQ $0x08, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x2f, R10
|
|
SHLQ $0x20, R11
|
|
IMULQ SI, R11
|
|
SHRQ $0x32, R11
|
|
MOVL (AX)(R10*4), SI
|
|
MOVL 524288(AX)(R11*4), R8
|
|
MOVL DX, (AX)(R10*4)
|
|
MOVL DX, 524288(AX)(R11*4)
|
|
MOVQ (BX)(SI*1), R10
|
|
MOVQ (BX)(R8*1), R11
|
|
CMPQ R10, DI
|
|
JEQ candidate_match_encodeBetterBlockAsm4MB
|
|
CMPQ R11, DI
|
|
JNE no_short_found_encodeBetterBlockAsm4MB
|
|
MOVL R8, SI
|
|
JMP candidate_match_encodeBetterBlockAsm4MB
|
|
|
|
no_short_found_encodeBetterBlockAsm4MB:
|
|
CMPL R10, DI
|
|
JEQ candidate_match_encodeBetterBlockAsm4MB
|
|
CMPL R11, DI
|
|
JEQ candidateS_match_encodeBetterBlockAsm4MB
|
|
MOVL 20(SP), DX
|
|
JMP search_loop_encodeBetterBlockAsm4MB
|
|
|
|
candidateS_match_encodeBetterBlockAsm4MB:
|
|
SHRQ $0x08, DI
|
|
MOVQ DI, R10
|
|
SHLQ $0x08, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x2f, R10
|
|
MOVL (AX)(R10*4), SI
|
|
INCL DX
|
|
MOVL DX, (AX)(R10*4)
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ candidate_match_encodeBetterBlockAsm4MB
|
|
DECL DX
|
|
MOVL R8, SI
|
|
|
|
candidate_match_encodeBetterBlockAsm4MB:
|
|
MOVL 12(SP), DI
|
|
TESTL SI, SI
|
|
JZ match_extend_back_end_encodeBetterBlockAsm4MB
|
|
|
|
match_extend_back_loop_encodeBetterBlockAsm4MB:
|
|
CMPL DX, DI
|
|
JBE match_extend_back_end_encodeBetterBlockAsm4MB
|
|
MOVB -1(BX)(SI*1), R8
|
|
MOVB -1(BX)(DX*1), R9
|
|
CMPB R8, R9
|
|
JNE match_extend_back_end_encodeBetterBlockAsm4MB
|
|
LEAL -1(DX), DX
|
|
DECL SI
|
|
JZ match_extend_back_end_encodeBetterBlockAsm4MB
|
|
JMP match_extend_back_loop_encodeBetterBlockAsm4MB
|
|
|
|
match_extend_back_end_encodeBetterBlockAsm4MB:
|
|
MOVL DX, DI
|
|
SUBL 12(SP), DI
|
|
LEAQ 4(CX)(DI*1), DI
|
|
CMPQ DI, (SP)
|
|
JB match_dst_size_check_encodeBetterBlockAsm4MB
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeBetterBlockAsm4MB:
|
|
MOVL DX, DI
|
|
ADDL $0x04, DX
|
|
ADDL $0x04, SI
|
|
MOVQ src_len+32(FP), R8
|
|
SUBL DX, R8
|
|
LEAQ (BX)(DX*1), R9
|
|
LEAQ (BX)(SI*1), R10
|
|
|
|
// matchLen
|
|
XORL R12, R12
|
|
|
|
matchlen_loopback_16_match_nolit_encodeBetterBlockAsm4MB:
|
|
CMPL R8, $0x10
|
|
JB matchlen_match8_match_nolit_encodeBetterBlockAsm4MB
|
|
MOVQ (R9)(R12*1), R11
|
|
MOVQ 8(R9)(R12*1), R13
|
|
XORQ (R10)(R12*1), R11
|
|
JNZ matchlen_bsf_8_match_nolit_encodeBetterBlockAsm4MB
|
|
XORQ 8(R10)(R12*1), R13
|
|
JNZ matchlen_bsf_16match_nolit_encodeBetterBlockAsm4MB
|
|
LEAL -16(R8), R8
|
|
LEAL 16(R12), R12
|
|
JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm4MB
|
|
|
|
matchlen_bsf_16match_nolit_encodeBetterBlockAsm4MB:
|
|
TZCNTQ R13, R13
|
|
SARQ $0x03, R13
|
|
LEAL 8(R12)(R13*1), R12
|
|
JMP match_nolit_end_encodeBetterBlockAsm4MB
|
|
|
|
matchlen_match8_match_nolit_encodeBetterBlockAsm4MB:
|
|
CMPL R8, $0x08
|
|
JB matchlen_match4_match_nolit_encodeBetterBlockAsm4MB
|
|
MOVQ (R9)(R12*1), R11
|
|
XORQ (R10)(R12*1), R11
|
|
JNZ matchlen_bsf_8_match_nolit_encodeBetterBlockAsm4MB
|
|
LEAL -8(R8), R8
|
|
LEAL 8(R12), R12
|
|
JMP matchlen_match4_match_nolit_encodeBetterBlockAsm4MB
|
|
|
|
matchlen_bsf_8_match_nolit_encodeBetterBlockAsm4MB:
|
|
TZCNTQ R11, R11
|
|
SARQ $0x03, R11
|
|
LEAL (R12)(R11*1), R12
|
|
JMP match_nolit_end_encodeBetterBlockAsm4MB
|
|
|
|
matchlen_match4_match_nolit_encodeBetterBlockAsm4MB:
|
|
CMPL R8, $0x04
|
|
JB matchlen_match2_match_nolit_encodeBetterBlockAsm4MB
|
|
MOVL (R9)(R12*1), R11
|
|
CMPL (R10)(R12*1), R11
|
|
JNE matchlen_match2_match_nolit_encodeBetterBlockAsm4MB
|
|
LEAL -4(R8), R8
|
|
LEAL 4(R12), R12
|
|
|
|
matchlen_match2_match_nolit_encodeBetterBlockAsm4MB:
|
|
CMPL R8, $0x01
|
|
JE matchlen_match1_match_nolit_encodeBetterBlockAsm4MB
|
|
JB match_nolit_end_encodeBetterBlockAsm4MB
|
|
MOVW (R9)(R12*1), R11
|
|
CMPW (R10)(R12*1), R11
|
|
JNE matchlen_match1_match_nolit_encodeBetterBlockAsm4MB
|
|
LEAL 2(R12), R12
|
|
SUBL $0x02, R8
|
|
JZ match_nolit_end_encodeBetterBlockAsm4MB
|
|
|
|
matchlen_match1_match_nolit_encodeBetterBlockAsm4MB:
|
|
MOVB (R9)(R12*1), R11
|
|
CMPB (R10)(R12*1), R11
|
|
JNE match_nolit_end_encodeBetterBlockAsm4MB
|
|
LEAL 1(R12), R12
|
|
|
|
match_nolit_end_encodeBetterBlockAsm4MB:
|
|
MOVL DX, R8
|
|
SUBL SI, R8
|
|
|
|
// Check if repeat
|
|
CMPL 16(SP), R8
|
|
JEQ match_is_repeat_encodeBetterBlockAsm4MB
|
|
CMPL R12, $0x01
|
|
JA match_length_ok_encodeBetterBlockAsm4MB
|
|
CMPL R8, $0x0000ffff
|
|
JBE match_length_ok_encodeBetterBlockAsm4MB
|
|
MOVL 20(SP), DX
|
|
INCL DX
|
|
JMP search_loop_encodeBetterBlockAsm4MB
|
|
|
|
match_length_ok_encodeBetterBlockAsm4MB:
|
|
MOVL R8, 16(SP)
|
|
MOVL 12(SP), SI
|
|
CMPL SI, DI
|
|
JEQ emit_literal_done_match_emit_encodeBetterBlockAsm4MB
|
|
MOVL DI, R9
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(SI*1), R10
|
|
SUBL SI, R9
|
|
LEAL -1(R9), SI
|
|
CMPL SI, $0x3c
|
|
JB one_byte_match_emit_encodeBetterBlockAsm4MB
|
|
CMPL SI, $0x00000100
|
|
JB two_bytes_match_emit_encodeBetterBlockAsm4MB
|
|
CMPL SI, $0x00010000
|
|
JB three_bytes_match_emit_encodeBetterBlockAsm4MB
|
|
MOVL SI, R11
|
|
SHRL $0x10, R11
|
|
MOVB $0xf8, (CX)
|
|
MOVW SI, 1(CX)
|
|
MOVB R11, 3(CX)
|
|
ADDQ $0x04, CX
|
|
JMP memmove_long_match_emit_encodeBetterBlockAsm4MB
|
|
|
|
three_bytes_match_emit_encodeBetterBlockAsm4MB:
|
|
MOVB $0xf4, (CX)
|
|
MOVW SI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_match_emit_encodeBetterBlockAsm4MB
|
|
|
|
two_bytes_match_emit_encodeBetterBlockAsm4MB:
|
|
MOVB $0xf0, (CX)
|
|
MOVB SI, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL SI, $0x40
|
|
JB memmove_match_emit_encodeBetterBlockAsm4MB
|
|
JMP memmove_long_match_emit_encodeBetterBlockAsm4MB
|
|
|
|
one_byte_match_emit_encodeBetterBlockAsm4MB:
|
|
SHLB $0x02, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_match_emit_encodeBetterBlockAsm4MB:
|
|
LEAQ (CX)(R9*1), SI
|
|
|
|
// genMemMoveShort
|
|
CMPQ R9, $0x04
|
|
JBE emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_4
|
|
CMPQ R9, $0x08
|
|
JB emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_4through7
|
|
CMPQ R9, $0x10
|
|
JBE emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_8through16
|
|
CMPQ R9, $0x20
|
|
JBE emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_4:
|
|
MOVL (R10), R11
|
|
MOVL R11, (CX)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm4MB
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_4through7:
|
|
MOVL (R10), R11
|
|
MOVL -4(R10)(R9*1), R10
|
|
MOVL R11, (CX)
|
|
MOVL R10, -4(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm4MB
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_8through16:
|
|
MOVQ (R10), R11
|
|
MOVQ -8(R10)(R9*1), R10
|
|
MOVQ R11, (CX)
|
|
MOVQ R10, -8(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm4MB
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_17through32:
|
|
MOVOU (R10), X0
|
|
MOVOU -16(R10)(R9*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm4MB
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_33through64:
|
|
MOVOU (R10), X0
|
|
MOVOU 16(R10), X1
|
|
MOVOU -32(R10)(R9*1), X2
|
|
MOVOU -16(R10)(R9*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
|
|
memmove_end_copy_match_emit_encodeBetterBlockAsm4MB:
|
|
MOVQ SI, CX
|
|
JMP emit_literal_done_match_emit_encodeBetterBlockAsm4MB
|
|
|
|
memmove_long_match_emit_encodeBetterBlockAsm4MB:
|
|
LEAQ (CX)(R9*1), SI
|
|
|
|
// genMemMoveLong
|
|
MOVOU (R10), X0
|
|
MOVOU 16(R10), X1
|
|
MOVOU -32(R10)(R9*1), X2
|
|
MOVOU -16(R10)(R9*1), X3
|
|
MOVQ R9, R13
|
|
SHRQ $0x05, R13
|
|
MOVQ CX, R11
|
|
ANDL $0x0000001f, R11
|
|
MOVQ $0x00000040, R14
|
|
SUBQ R11, R14
|
|
DECQ R13
|
|
JA emit_lit_memmove_long_match_emit_encodeBetterBlockAsm4MBlarge_forward_sse_loop_32
|
|
LEAQ -32(R10)(R14*1), R11
|
|
LEAQ -32(CX)(R14*1), R15
|
|
|
|
emit_lit_memmove_long_match_emit_encodeBetterBlockAsm4MBlarge_big_loop_back:
|
|
MOVOU (R11), X4
|
|
MOVOU 16(R11), X5
|
|
MOVOA X4, (R15)
|
|
MOVOA X5, 16(R15)
|
|
ADDQ $0x20, R15
|
|
ADDQ $0x20, R11
|
|
ADDQ $0x20, R14
|
|
DECQ R13
|
|
JNA emit_lit_memmove_long_match_emit_encodeBetterBlockAsm4MBlarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_match_emit_encodeBetterBlockAsm4MBlarge_forward_sse_loop_32:
|
|
MOVOU -32(R10)(R14*1), X4
|
|
MOVOU -16(R10)(R14*1), X5
|
|
MOVOA X4, -32(CX)(R14*1)
|
|
MOVOA X5, -16(CX)(R14*1)
|
|
ADDQ $0x20, R14
|
|
CMPQ R9, R14
|
|
JAE emit_lit_memmove_long_match_emit_encodeBetterBlockAsm4MBlarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
MOVQ SI, CX
|
|
|
|
emit_literal_done_match_emit_encodeBetterBlockAsm4MB:
|
|
ADDL R12, DX
|
|
ADDL $0x04, R12
|
|
MOVL DX, 12(SP)
|
|
|
|
// emitCopy
|
|
CMPL R8, $0x00010000
|
|
JB two_byte_offset_match_nolit_encodeBetterBlockAsm4MB
|
|
CMPL R12, $0x40
|
|
JBE four_bytes_remain_match_nolit_encodeBetterBlockAsm4MB
|
|
MOVB $0xff, (CX)
|
|
MOVL R8, 1(CX)
|
|
LEAL -64(R12), R12
|
|
ADDQ $0x05, CX
|
|
CMPL R12, $0x04
|
|
JB four_bytes_remain_match_nolit_encodeBetterBlockAsm4MB
|
|
|
|
// emitRepeat
|
|
MOVL R12, SI
|
|
LEAL -4(R12), R12
|
|
CMPL SI, $0x08
|
|
JBE repeat_two_match_nolit_encodeBetterBlockAsm4MB_emit_copy
|
|
CMPL SI, $0x0c
|
|
JAE cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy
|
|
CMPL R8, $0x00000800
|
|
JB repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy:
|
|
CMPL R12, $0x00000104
|
|
JB repeat_three_match_nolit_encodeBetterBlockAsm4MB_emit_copy
|
|
CMPL R12, $0x00010100
|
|
JB repeat_four_match_nolit_encodeBetterBlockAsm4MB_emit_copy
|
|
LEAL -65536(R12), R12
|
|
MOVL R12, R8
|
|
MOVW $0x001d, (CX)
|
|
MOVW R12, 2(CX)
|
|
SARL $0x10, R8
|
|
MOVB R8, 4(CX)
|
|
ADDQ $0x05, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
repeat_four_match_nolit_encodeBetterBlockAsm4MB_emit_copy:
|
|
LEAL -256(R12), R12
|
|
MOVW $0x0019, (CX)
|
|
MOVW R12, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
repeat_three_match_nolit_encodeBetterBlockAsm4MB_emit_copy:
|
|
LEAL -4(R12), R12
|
|
MOVW $0x0015, (CX)
|
|
MOVB R12, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
repeat_two_match_nolit_encodeBetterBlockAsm4MB_emit_copy:
|
|
SHLL $0x02, R12
|
|
ORL $0x01, R12
|
|
MOVW R12, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy:
|
|
XORQ SI, SI
|
|
LEAL 1(SI)(R12*4), R12
|
|
MOVB R8, 1(CX)
|
|
SARL $0x08, R8
|
|
SHLL $0x05, R8
|
|
ORL R8, R12
|
|
MOVB R12, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
four_bytes_remain_match_nolit_encodeBetterBlockAsm4MB:
|
|
TESTL R12, R12
|
|
JZ match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
XORL SI, SI
|
|
LEAL -1(SI)(R12*4), R12
|
|
MOVB R12, (CX)
|
|
MOVL R8, 1(CX)
|
|
ADDQ $0x05, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
two_byte_offset_match_nolit_encodeBetterBlockAsm4MB:
|
|
CMPL R12, $0x40
|
|
JBE two_byte_offset_short_match_nolit_encodeBetterBlockAsm4MB
|
|
CMPL R8, $0x00000800
|
|
JAE long_offset_short_match_nolit_encodeBetterBlockAsm4MB
|
|
MOVL $0x00000001, SI
|
|
LEAL 16(SI), SI
|
|
MOVB R8, 1(CX)
|
|
SHRL $0x08, R8
|
|
SHLL $0x05, R8
|
|
ORL R8, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x02, CX
|
|
SUBL $0x08, R12
|
|
|
|
// emitRepeat
|
|
LEAL -4(R12), R12
|
|
JMP cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b
|
|
MOVL R12, SI
|
|
LEAL -4(R12), R12
|
|
CMPL SI, $0x08
|
|
JBE repeat_two_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b
|
|
CMPL SI, $0x0c
|
|
JAE cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b
|
|
CMPL R8, $0x00000800
|
|
JB repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b:
|
|
CMPL R12, $0x00000104
|
|
JB repeat_three_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b
|
|
CMPL R12, $0x00010100
|
|
JB repeat_four_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b
|
|
LEAL -65536(R12), R12
|
|
MOVL R12, R8
|
|
MOVW $0x001d, (CX)
|
|
MOVW R12, 2(CX)
|
|
SARL $0x10, R8
|
|
MOVB R8, 4(CX)
|
|
ADDQ $0x05, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
repeat_four_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b:
|
|
LEAL -256(R12), R12
|
|
MOVW $0x0019, (CX)
|
|
MOVW R12, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
repeat_three_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b:
|
|
LEAL -4(R12), R12
|
|
MOVW $0x0015, (CX)
|
|
MOVB R12, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
repeat_two_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b:
|
|
SHLL $0x02, R12
|
|
ORL $0x01, R12
|
|
MOVW R12, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b:
|
|
XORQ SI, SI
|
|
LEAL 1(SI)(R12*4), R12
|
|
MOVB R8, 1(CX)
|
|
SARL $0x08, R8
|
|
SHLL $0x05, R8
|
|
ORL R8, R12
|
|
MOVB R12, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
long_offset_short_match_nolit_encodeBetterBlockAsm4MB:
|
|
MOVB $0xee, (CX)
|
|
MOVW R8, 1(CX)
|
|
LEAL -60(R12), R12
|
|
ADDQ $0x03, CX
|
|
|
|
// emitRepeat
|
|
MOVL R12, SI
|
|
LEAL -4(R12), R12
|
|
CMPL SI, $0x08
|
|
JBE repeat_two_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short
|
|
CMPL SI, $0x0c
|
|
JAE cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short
|
|
CMPL R8, $0x00000800
|
|
JB repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short:
|
|
CMPL R12, $0x00000104
|
|
JB repeat_three_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short
|
|
CMPL R12, $0x00010100
|
|
JB repeat_four_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short
|
|
LEAL -65536(R12), R12
|
|
MOVL R12, R8
|
|
MOVW $0x001d, (CX)
|
|
MOVW R12, 2(CX)
|
|
SARL $0x10, R8
|
|
MOVB R8, 4(CX)
|
|
ADDQ $0x05, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
repeat_four_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short:
|
|
LEAL -256(R12), R12
|
|
MOVW $0x0019, (CX)
|
|
MOVW R12, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
repeat_three_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short:
|
|
LEAL -4(R12), R12
|
|
MOVW $0x0015, (CX)
|
|
MOVB R12, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
repeat_two_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short:
|
|
SHLL $0x02, R12
|
|
ORL $0x01, R12
|
|
MOVW R12, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short:
|
|
XORQ SI, SI
|
|
LEAL 1(SI)(R12*4), R12
|
|
MOVB R8, 1(CX)
|
|
SARL $0x08, R8
|
|
SHLL $0x05, R8
|
|
ORL R8, R12
|
|
MOVB R12, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
two_byte_offset_short_match_nolit_encodeBetterBlockAsm4MB:
|
|
MOVL R12, SI
|
|
SHLL $0x02, SI
|
|
CMPL R12, $0x0c
|
|
JAE emit_copy_three_match_nolit_encodeBetterBlockAsm4MB
|
|
CMPL R8, $0x00000800
|
|
JAE emit_copy_three_match_nolit_encodeBetterBlockAsm4MB
|
|
LEAL -15(SI), SI
|
|
MOVB R8, 1(CX)
|
|
SHRL $0x08, R8
|
|
SHLL $0x05, R8
|
|
ORL R8, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
emit_copy_three_match_nolit_encodeBetterBlockAsm4MB:
|
|
LEAL -2(SI), SI
|
|
MOVB SI, (CX)
|
|
MOVW R8, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
match_is_repeat_encodeBetterBlockAsm4MB:
|
|
MOVL 12(SP), SI
|
|
CMPL SI, DI
|
|
JEQ emit_literal_done_match_emit_repeat_encodeBetterBlockAsm4MB
|
|
MOVL DI, R9
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(SI*1), R10
|
|
SUBL SI, R9
|
|
LEAL -1(R9), SI
|
|
CMPL SI, $0x3c
|
|
JB one_byte_match_emit_repeat_encodeBetterBlockAsm4MB
|
|
CMPL SI, $0x00000100
|
|
JB two_bytes_match_emit_repeat_encodeBetterBlockAsm4MB
|
|
CMPL SI, $0x00010000
|
|
JB three_bytes_match_emit_repeat_encodeBetterBlockAsm4MB
|
|
MOVL SI, R11
|
|
SHRL $0x10, R11
|
|
MOVB $0xf8, (CX)
|
|
MOVW SI, 1(CX)
|
|
MOVB R11, 3(CX)
|
|
ADDQ $0x04, CX
|
|
JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm4MB
|
|
|
|
three_bytes_match_emit_repeat_encodeBetterBlockAsm4MB:
|
|
MOVB $0xf4, (CX)
|
|
MOVW SI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm4MB
|
|
|
|
two_bytes_match_emit_repeat_encodeBetterBlockAsm4MB:
|
|
MOVB $0xf0, (CX)
|
|
MOVB SI, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL SI, $0x40
|
|
JB memmove_match_emit_repeat_encodeBetterBlockAsm4MB
|
|
JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm4MB
|
|
|
|
one_byte_match_emit_repeat_encodeBetterBlockAsm4MB:
|
|
SHLB $0x02, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_match_emit_repeat_encodeBetterBlockAsm4MB:
|
|
LEAQ (CX)(R9*1), SI
|
|
|
|
// genMemMoveShort
|
|
CMPQ R9, $0x04
|
|
JBE emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_4
|
|
CMPQ R9, $0x08
|
|
JB emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_4through7
|
|
CMPQ R9, $0x10
|
|
JBE emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_8through16
|
|
CMPQ R9, $0x20
|
|
JBE emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_4:
|
|
MOVL (R10), R11
|
|
MOVL R11, (CX)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm4MB
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_4through7:
|
|
MOVL (R10), R11
|
|
MOVL -4(R10)(R9*1), R10
|
|
MOVL R11, (CX)
|
|
MOVL R10, -4(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm4MB
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_8through16:
|
|
MOVQ (R10), R11
|
|
MOVQ -8(R10)(R9*1), R10
|
|
MOVQ R11, (CX)
|
|
MOVQ R10, -8(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm4MB
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_17through32:
|
|
MOVOU (R10), X0
|
|
MOVOU -16(R10)(R9*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm4MB
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_33through64:
|
|
MOVOU (R10), X0
|
|
MOVOU 16(R10), X1
|
|
MOVOU -32(R10)(R9*1), X2
|
|
MOVOU -16(R10)(R9*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
|
|
memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm4MB:
|
|
MOVQ SI, CX
|
|
JMP emit_literal_done_match_emit_repeat_encodeBetterBlockAsm4MB
|
|
|
|
memmove_long_match_emit_repeat_encodeBetterBlockAsm4MB:
|
|
LEAQ (CX)(R9*1), SI
|
|
|
|
// genMemMoveLong
|
|
MOVOU (R10), X0
|
|
MOVOU 16(R10), X1
|
|
MOVOU -32(R10)(R9*1), X2
|
|
MOVOU -16(R10)(R9*1), X3
|
|
MOVQ R9, R13
|
|
SHRQ $0x05, R13
|
|
MOVQ CX, R11
|
|
ANDL $0x0000001f, R11
|
|
MOVQ $0x00000040, R14
|
|
SUBQ R11, R14
|
|
DECQ R13
|
|
JA emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm4MBlarge_forward_sse_loop_32
|
|
LEAQ -32(R10)(R14*1), R11
|
|
LEAQ -32(CX)(R14*1), R15
|
|
|
|
emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm4MBlarge_big_loop_back:
|
|
MOVOU (R11), X4
|
|
MOVOU 16(R11), X5
|
|
MOVOA X4, (R15)
|
|
MOVOA X5, 16(R15)
|
|
ADDQ $0x20, R15
|
|
ADDQ $0x20, R11
|
|
ADDQ $0x20, R14
|
|
DECQ R13
|
|
JNA emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm4MBlarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm4MBlarge_forward_sse_loop_32:
|
|
MOVOU -32(R10)(R14*1), X4
|
|
MOVOU -16(R10)(R14*1), X5
|
|
MOVOA X4, -32(CX)(R14*1)
|
|
MOVOA X5, -16(CX)(R14*1)
|
|
ADDQ $0x20, R14
|
|
CMPQ R9, R14
|
|
JAE emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm4MBlarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
MOVQ SI, CX
|
|
|
|
emit_literal_done_match_emit_repeat_encodeBetterBlockAsm4MB:
|
|
ADDL R12, DX
|
|
ADDL $0x04, R12
|
|
MOVL DX, 12(SP)
|
|
|
|
// emitRepeat
|
|
MOVL R12, SI
|
|
LEAL -4(R12), R12
|
|
CMPL SI, $0x08
|
|
JBE repeat_two_match_nolit_repeat_encodeBetterBlockAsm4MB
|
|
CMPL SI, $0x0c
|
|
JAE cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm4MB
|
|
CMPL R8, $0x00000800
|
|
JB repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm4MB
|
|
|
|
cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm4MB:
|
|
CMPL R12, $0x00000104
|
|
JB repeat_three_match_nolit_repeat_encodeBetterBlockAsm4MB
|
|
CMPL R12, $0x00010100
|
|
JB repeat_four_match_nolit_repeat_encodeBetterBlockAsm4MB
|
|
LEAL -65536(R12), R12
|
|
MOVL R12, R8
|
|
MOVW $0x001d, (CX)
|
|
MOVW R12, 2(CX)
|
|
SARL $0x10, R8
|
|
MOVB R8, 4(CX)
|
|
ADDQ $0x05, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
repeat_four_match_nolit_repeat_encodeBetterBlockAsm4MB:
|
|
LEAL -256(R12), R12
|
|
MOVW $0x0019, (CX)
|
|
MOVW R12, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
repeat_three_match_nolit_repeat_encodeBetterBlockAsm4MB:
|
|
LEAL -4(R12), R12
|
|
MOVW $0x0015, (CX)
|
|
MOVB R12, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
repeat_two_match_nolit_repeat_encodeBetterBlockAsm4MB:
|
|
SHLL $0x02, R12
|
|
ORL $0x01, R12
|
|
MOVW R12, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm4MB:
|
|
XORQ SI, SI
|
|
LEAL 1(SI)(R12*4), R12
|
|
MOVB R8, 1(CX)
|
|
SARL $0x08, R8
|
|
SHLL $0x05, R8
|
|
ORL R8, R12
|
|
MOVB R12, (CX)
|
|
ADDQ $0x02, CX
|
|
|
|
match_nolit_emitcopy_end_encodeBetterBlockAsm4MB:
|
|
CMPL DX, 8(SP)
|
|
JAE emit_remainder_encodeBetterBlockAsm4MB
|
|
CMPQ CX, (SP)
|
|
JB match_nolit_dst_ok_encodeBetterBlockAsm4MB
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeBetterBlockAsm4MB:
|
|
MOVQ $0x00cf1bbcdcbfa563, SI
|
|
MOVQ $0x9e3779b1, R8
|
|
LEAQ 1(DI), DI
|
|
LEAQ -2(DX), R9
|
|
MOVQ (BX)(DI*1), R10
|
|
MOVQ 1(BX)(DI*1), R11
|
|
MOVQ (BX)(R9*1), R12
|
|
MOVQ 1(BX)(R9*1), R13
|
|
SHLQ $0x08, R10
|
|
IMULQ SI, R10
|
|
SHRQ $0x2f, R10
|
|
SHLQ $0x20, R11
|
|
IMULQ R8, R11
|
|
SHRQ $0x32, R11
|
|
SHLQ $0x08, R12
|
|
IMULQ SI, R12
|
|
SHRQ $0x2f, R12
|
|
SHLQ $0x20, R13
|
|
IMULQ R8, R13
|
|
SHRQ $0x32, R13
|
|
LEAQ 1(DI), R8
|
|
LEAQ 1(R9), R14
|
|
MOVL DI, (AX)(R10*4)
|
|
MOVL R9, (AX)(R12*4)
|
|
MOVL R8, 524288(AX)(R11*4)
|
|
MOVL R14, 524288(AX)(R13*4)
|
|
LEAQ 1(R9)(DI*1), R8
|
|
SHRQ $0x01, R8
|
|
ADDQ $0x01, DI
|
|
SUBQ $0x01, R9
|
|
|
|
index_loop_encodeBetterBlockAsm4MB:
|
|
CMPQ R8, R9
|
|
JAE search_loop_encodeBetterBlockAsm4MB
|
|
MOVQ (BX)(DI*1), R10
|
|
MOVQ (BX)(R8*1), R11
|
|
SHLQ $0x08, R10
|
|
IMULQ SI, R10
|
|
SHRQ $0x2f, R10
|
|
SHLQ $0x08, R11
|
|
IMULQ SI, R11
|
|
SHRQ $0x2f, R11
|
|
MOVL DI, (AX)(R10*4)
|
|
MOVL R8, (AX)(R11*4)
|
|
ADDQ $0x02, DI
|
|
ADDQ $0x02, R8
|
|
JMP index_loop_encodeBetterBlockAsm4MB
|
|
|
|
emit_remainder_encodeBetterBlockAsm4MB:
|
|
MOVQ src_len+32(FP), AX
|
|
SUBL 12(SP), AX
|
|
LEAQ 4(CX)(AX*1), AX
|
|
CMPQ AX, (SP)
|
|
JB emit_remainder_ok_encodeBetterBlockAsm4MB
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeBetterBlockAsm4MB:
|
|
MOVQ src_len+32(FP), AX
|
|
MOVL 12(SP), DX
|
|
CMPL DX, AX
|
|
JEQ emit_literal_done_emit_remainder_encodeBetterBlockAsm4MB
|
|
MOVL AX, SI
|
|
MOVL AX, 12(SP)
|
|
LEAQ (BX)(DX*1), AX
|
|
SUBL DX, SI
|
|
LEAL -1(SI), DX
|
|
CMPL DX, $0x3c
|
|
JB one_byte_emit_remainder_encodeBetterBlockAsm4MB
|
|
CMPL DX, $0x00000100
|
|
JB two_bytes_emit_remainder_encodeBetterBlockAsm4MB
|
|
CMPL DX, $0x00010000
|
|
JB three_bytes_emit_remainder_encodeBetterBlockAsm4MB
|
|
MOVL DX, BX
|
|
SHRL $0x10, BX
|
|
MOVB $0xf8, (CX)
|
|
MOVW DX, 1(CX)
|
|
MOVB BL, 3(CX)
|
|
ADDQ $0x04, CX
|
|
JMP memmove_long_emit_remainder_encodeBetterBlockAsm4MB
|
|
|
|
three_bytes_emit_remainder_encodeBetterBlockAsm4MB:
|
|
MOVB $0xf4, (CX)
|
|
MOVW DX, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_emit_remainder_encodeBetterBlockAsm4MB
|
|
|
|
two_bytes_emit_remainder_encodeBetterBlockAsm4MB:
|
|
MOVB $0xf0, (CX)
|
|
MOVB DL, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL DX, $0x40
|
|
JB memmove_emit_remainder_encodeBetterBlockAsm4MB
|
|
JMP memmove_long_emit_remainder_encodeBetterBlockAsm4MB
|
|
|
|
one_byte_emit_remainder_encodeBetterBlockAsm4MB:
|
|
SHLB $0x02, DL
|
|
MOVB DL, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_emit_remainder_encodeBetterBlockAsm4MB:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveShort
|
|
CMPQ BX, $0x03
|
|
JB emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_1or2
|
|
JE emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_3
|
|
CMPQ BX, $0x08
|
|
JB emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_4through7
|
|
CMPQ BX, $0x10
|
|
JBE emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_8through16
|
|
CMPQ BX, $0x20
|
|
JBE emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_1or2:
|
|
MOVB (AX), SI
|
|
MOVB -1(AX)(BX*1), AL
|
|
MOVB SI, (CX)
|
|
MOVB AL, -1(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm4MB
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_3:
|
|
MOVW (AX), SI
|
|
MOVB 2(AX), AL
|
|
MOVW SI, (CX)
|
|
MOVB AL, 2(CX)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm4MB
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_4through7:
|
|
MOVL (AX), SI
|
|
MOVL -4(AX)(BX*1), AX
|
|
MOVL SI, (CX)
|
|
MOVL AX, -4(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm4MB
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_8through16:
|
|
MOVQ (AX), SI
|
|
MOVQ -8(AX)(BX*1), AX
|
|
MOVQ SI, (CX)
|
|
MOVQ AX, -8(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm4MB
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_17through32:
|
|
MOVOU (AX), X0
|
|
MOVOU -16(AX)(BX*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm4MB
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_33through64:
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
|
|
memmove_end_copy_emit_remainder_encodeBetterBlockAsm4MB:
|
|
MOVQ DX, CX
|
|
JMP emit_literal_done_emit_remainder_encodeBetterBlockAsm4MB
|
|
|
|
memmove_long_emit_remainder_encodeBetterBlockAsm4MB:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveLong
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVQ BX, DI
|
|
SHRQ $0x05, DI
|
|
MOVQ CX, SI
|
|
ANDL $0x0000001f, SI
|
|
MOVQ $0x00000040, R8
|
|
SUBQ SI, R8
|
|
DECQ DI
|
|
JA emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm4MBlarge_forward_sse_loop_32
|
|
LEAQ -32(AX)(R8*1), SI
|
|
LEAQ -32(CX)(R8*1), R9
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm4MBlarge_big_loop_back:
|
|
MOVOU (SI), X4
|
|
MOVOU 16(SI), X5
|
|
MOVOA X4, (R9)
|
|
MOVOA X5, 16(R9)
|
|
ADDQ $0x20, R9
|
|
ADDQ $0x20, SI
|
|
ADDQ $0x20, R8
|
|
DECQ DI
|
|
JNA emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm4MBlarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm4MBlarge_forward_sse_loop_32:
|
|
MOVOU -32(AX)(R8*1), X4
|
|
MOVOU -16(AX)(R8*1), X5
|
|
MOVOA X4, -32(CX)(R8*1)
|
|
MOVOA X5, -16(CX)(R8*1)
|
|
ADDQ $0x20, R8
|
|
CMPQ BX, R8
|
|
JAE emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm4MBlarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
MOVQ DX, CX
|
|
|
|
emit_literal_done_emit_remainder_encodeBetterBlockAsm4MB:
|
|
MOVQ dst_base+0(FP), AX
|
|
SUBQ AX, CX
|
|
MOVQ CX, ret+56(FP)
|
|
RET
|
|
|
|
// func encodeBetterBlockAsm12B(dst []byte, src []byte, tmp *[81920]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeBetterBlockAsm12B(SB), $24-64
|
|
MOVQ tmp+48(FP), AX
|
|
MOVQ dst_base+0(FP), CX
|
|
MOVQ $0x00000280, DX
|
|
MOVQ AX, BX
|
|
PXOR X0, X0
|
|
|
|
zero_loop_encodeBetterBlockAsm12B:
|
|
MOVOU X0, (BX)
|
|
MOVOU X0, 16(BX)
|
|
MOVOU X0, 32(BX)
|
|
MOVOU X0, 48(BX)
|
|
MOVOU X0, 64(BX)
|
|
MOVOU X0, 80(BX)
|
|
MOVOU X0, 96(BX)
|
|
MOVOU X0, 112(BX)
|
|
ADDQ $0x80, BX
|
|
DECQ DX
|
|
JNZ zero_loop_encodeBetterBlockAsm12B
|
|
MOVL $0x00000000, 12(SP)
|
|
MOVQ src_len+32(FP), DX
|
|
LEAQ -6(DX), BX
|
|
LEAQ -8(DX), SI
|
|
MOVL SI, 8(SP)
|
|
SHRQ $0x05, DX
|
|
SUBL DX, BX
|
|
LEAQ (CX)(BX*1), BX
|
|
MOVQ BX, (SP)
|
|
MOVL $0x00000001, DX
|
|
MOVL $0x00000000, 16(SP)
|
|
MOVQ src_base+24(FP), BX
|
|
|
|
search_loop_encodeBetterBlockAsm12B:
|
|
MOVL DX, SI
|
|
SUBL 12(SP), SI
|
|
SHRL $0x06, SI
|
|
LEAL 1(DX)(SI*1), SI
|
|
CMPL SI, 8(SP)
|
|
JAE emit_remainder_encodeBetterBlockAsm12B
|
|
MOVQ (BX)(DX*1), DI
|
|
MOVL SI, 20(SP)
|
|
MOVQ $0x0000cf1bbcdcbf9b, R9
|
|
MOVQ $0x9e3779b1, SI
|
|
MOVQ DI, R10
|
|
MOVQ DI, R11
|
|
SHLQ $0x10, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x32, R10
|
|
SHLQ $0x20, R11
|
|
IMULQ SI, R11
|
|
SHRQ $0x34, R11
|
|
MOVL (AX)(R10*4), SI
|
|
MOVL 65536(AX)(R11*4), R8
|
|
MOVL DX, (AX)(R10*4)
|
|
MOVL DX, 65536(AX)(R11*4)
|
|
MOVQ (BX)(SI*1), R10
|
|
MOVQ (BX)(R8*1), R11
|
|
CMPQ R10, DI
|
|
JEQ candidate_match_encodeBetterBlockAsm12B
|
|
CMPQ R11, DI
|
|
JNE no_short_found_encodeBetterBlockAsm12B
|
|
MOVL R8, SI
|
|
JMP candidate_match_encodeBetterBlockAsm12B
|
|
|
|
no_short_found_encodeBetterBlockAsm12B:
|
|
CMPL R10, DI
|
|
JEQ candidate_match_encodeBetterBlockAsm12B
|
|
CMPL R11, DI
|
|
JEQ candidateS_match_encodeBetterBlockAsm12B
|
|
MOVL 20(SP), DX
|
|
JMP search_loop_encodeBetterBlockAsm12B
|
|
|
|
candidateS_match_encodeBetterBlockAsm12B:
|
|
SHRQ $0x08, DI
|
|
MOVQ DI, R10
|
|
SHLQ $0x10, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x32, R10
|
|
MOVL (AX)(R10*4), SI
|
|
INCL DX
|
|
MOVL DX, (AX)(R10*4)
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ candidate_match_encodeBetterBlockAsm12B
|
|
DECL DX
|
|
MOVL R8, SI
|
|
|
|
candidate_match_encodeBetterBlockAsm12B:
|
|
MOVL 12(SP), DI
|
|
TESTL SI, SI
|
|
JZ match_extend_back_end_encodeBetterBlockAsm12B
|
|
|
|
match_extend_back_loop_encodeBetterBlockAsm12B:
|
|
CMPL DX, DI
|
|
JBE match_extend_back_end_encodeBetterBlockAsm12B
|
|
MOVB -1(BX)(SI*1), R8
|
|
MOVB -1(BX)(DX*1), R9
|
|
CMPB R8, R9
|
|
JNE match_extend_back_end_encodeBetterBlockAsm12B
|
|
LEAL -1(DX), DX
|
|
DECL SI
|
|
JZ match_extend_back_end_encodeBetterBlockAsm12B
|
|
JMP match_extend_back_loop_encodeBetterBlockAsm12B
|
|
|
|
match_extend_back_end_encodeBetterBlockAsm12B:
|
|
MOVL DX, DI
|
|
SUBL 12(SP), DI
|
|
LEAQ 3(CX)(DI*1), DI
|
|
CMPQ DI, (SP)
|
|
JB match_dst_size_check_encodeBetterBlockAsm12B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeBetterBlockAsm12B:
|
|
MOVL DX, DI
|
|
ADDL $0x04, DX
|
|
ADDL $0x04, SI
|
|
MOVQ src_len+32(FP), R8
|
|
SUBL DX, R8
|
|
LEAQ (BX)(DX*1), R9
|
|
LEAQ (BX)(SI*1), R10
|
|
|
|
// matchLen
|
|
XORL R12, R12
|
|
|
|
matchlen_loopback_16_match_nolit_encodeBetterBlockAsm12B:
|
|
CMPL R8, $0x10
|
|
JB matchlen_match8_match_nolit_encodeBetterBlockAsm12B
|
|
MOVQ (R9)(R12*1), R11
|
|
MOVQ 8(R9)(R12*1), R13
|
|
XORQ (R10)(R12*1), R11
|
|
JNZ matchlen_bsf_8_match_nolit_encodeBetterBlockAsm12B
|
|
XORQ 8(R10)(R12*1), R13
|
|
JNZ matchlen_bsf_16match_nolit_encodeBetterBlockAsm12B
|
|
LEAL -16(R8), R8
|
|
LEAL 16(R12), R12
|
|
JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm12B
|
|
|
|
matchlen_bsf_16match_nolit_encodeBetterBlockAsm12B:
|
|
TZCNTQ R13, R13
|
|
SARQ $0x03, R13
|
|
LEAL 8(R12)(R13*1), R12
|
|
JMP match_nolit_end_encodeBetterBlockAsm12B
|
|
|
|
matchlen_match8_match_nolit_encodeBetterBlockAsm12B:
|
|
CMPL R8, $0x08
|
|
JB matchlen_match4_match_nolit_encodeBetterBlockAsm12B
|
|
MOVQ (R9)(R12*1), R11
|
|
XORQ (R10)(R12*1), R11
|
|
JNZ matchlen_bsf_8_match_nolit_encodeBetterBlockAsm12B
|
|
LEAL -8(R8), R8
|
|
LEAL 8(R12), R12
|
|
JMP matchlen_match4_match_nolit_encodeBetterBlockAsm12B
|
|
|
|
matchlen_bsf_8_match_nolit_encodeBetterBlockAsm12B:
|
|
TZCNTQ R11, R11
|
|
SARQ $0x03, R11
|
|
LEAL (R12)(R11*1), R12
|
|
JMP match_nolit_end_encodeBetterBlockAsm12B
|
|
|
|
matchlen_match4_match_nolit_encodeBetterBlockAsm12B:
|
|
CMPL R8, $0x04
|
|
JB matchlen_match2_match_nolit_encodeBetterBlockAsm12B
|
|
MOVL (R9)(R12*1), R11
|
|
CMPL (R10)(R12*1), R11
|
|
JNE matchlen_match2_match_nolit_encodeBetterBlockAsm12B
|
|
LEAL -4(R8), R8
|
|
LEAL 4(R12), R12
|
|
|
|
matchlen_match2_match_nolit_encodeBetterBlockAsm12B:
|
|
CMPL R8, $0x01
|
|
JE matchlen_match1_match_nolit_encodeBetterBlockAsm12B
|
|
JB match_nolit_end_encodeBetterBlockAsm12B
|
|
MOVW (R9)(R12*1), R11
|
|
CMPW (R10)(R12*1), R11
|
|
JNE matchlen_match1_match_nolit_encodeBetterBlockAsm12B
|
|
LEAL 2(R12), R12
|
|
SUBL $0x02, R8
|
|
JZ match_nolit_end_encodeBetterBlockAsm12B
|
|
|
|
matchlen_match1_match_nolit_encodeBetterBlockAsm12B:
|
|
MOVB (R9)(R12*1), R11
|
|
CMPB (R10)(R12*1), R11
|
|
JNE match_nolit_end_encodeBetterBlockAsm12B
|
|
LEAL 1(R12), R12
|
|
|
|
match_nolit_end_encodeBetterBlockAsm12B:
|
|
MOVL DX, R8
|
|
SUBL SI, R8
|
|
|
|
// Check if repeat
|
|
CMPL 16(SP), R8
|
|
JEQ match_is_repeat_encodeBetterBlockAsm12B
|
|
MOVL R8, 16(SP)
|
|
MOVL 12(SP), SI
|
|
CMPL SI, DI
|
|
JEQ emit_literal_done_match_emit_encodeBetterBlockAsm12B
|
|
MOVL DI, R9
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(SI*1), R10
|
|
SUBL SI, R9
|
|
LEAL -1(R9), SI
|
|
CMPL SI, $0x3c
|
|
JB one_byte_match_emit_encodeBetterBlockAsm12B
|
|
CMPL SI, $0x00000100
|
|
JB two_bytes_match_emit_encodeBetterBlockAsm12B
|
|
JB three_bytes_match_emit_encodeBetterBlockAsm12B
|
|
|
|
three_bytes_match_emit_encodeBetterBlockAsm12B:
|
|
MOVB $0xf4, (CX)
|
|
MOVW SI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_match_emit_encodeBetterBlockAsm12B
|
|
|
|
two_bytes_match_emit_encodeBetterBlockAsm12B:
|
|
MOVB $0xf0, (CX)
|
|
MOVB SI, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL SI, $0x40
|
|
JB memmove_match_emit_encodeBetterBlockAsm12B
|
|
JMP memmove_long_match_emit_encodeBetterBlockAsm12B
|
|
|
|
one_byte_match_emit_encodeBetterBlockAsm12B:
|
|
SHLB $0x02, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_match_emit_encodeBetterBlockAsm12B:
|
|
LEAQ (CX)(R9*1), SI
|
|
|
|
// genMemMoveShort
|
|
CMPQ R9, $0x04
|
|
JBE emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_4
|
|
CMPQ R9, $0x08
|
|
JB emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_4through7
|
|
CMPQ R9, $0x10
|
|
JBE emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_8through16
|
|
CMPQ R9, $0x20
|
|
JBE emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_4:
|
|
MOVL (R10), R11
|
|
MOVL R11, (CX)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_4through7:
|
|
MOVL (R10), R11
|
|
MOVL -4(R10)(R9*1), R10
|
|
MOVL R11, (CX)
|
|
MOVL R10, -4(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_8through16:
|
|
MOVQ (R10), R11
|
|
MOVQ -8(R10)(R9*1), R10
|
|
MOVQ R11, (CX)
|
|
MOVQ R10, -8(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_17through32:
|
|
MOVOU (R10), X0
|
|
MOVOU -16(R10)(R9*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_33through64:
|
|
MOVOU (R10), X0
|
|
MOVOU 16(R10), X1
|
|
MOVOU -32(R10)(R9*1), X2
|
|
MOVOU -16(R10)(R9*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
|
|
memmove_end_copy_match_emit_encodeBetterBlockAsm12B:
|
|
MOVQ SI, CX
|
|
JMP emit_literal_done_match_emit_encodeBetterBlockAsm12B
|
|
|
|
memmove_long_match_emit_encodeBetterBlockAsm12B:
|
|
LEAQ (CX)(R9*1), SI
|
|
|
|
// genMemMoveLong
|
|
MOVOU (R10), X0
|
|
MOVOU 16(R10), X1
|
|
MOVOU -32(R10)(R9*1), X2
|
|
MOVOU -16(R10)(R9*1), X3
|
|
MOVQ R9, R13
|
|
SHRQ $0x05, R13
|
|
MOVQ CX, R11
|
|
ANDL $0x0000001f, R11
|
|
MOVQ $0x00000040, R14
|
|
SUBQ R11, R14
|
|
DECQ R13
|
|
JA emit_lit_memmove_long_match_emit_encodeBetterBlockAsm12Blarge_forward_sse_loop_32
|
|
LEAQ -32(R10)(R14*1), R11
|
|
LEAQ -32(CX)(R14*1), R15
|
|
|
|
emit_lit_memmove_long_match_emit_encodeBetterBlockAsm12Blarge_big_loop_back:
|
|
MOVOU (R11), X4
|
|
MOVOU 16(R11), X5
|
|
MOVOA X4, (R15)
|
|
MOVOA X5, 16(R15)
|
|
ADDQ $0x20, R15
|
|
ADDQ $0x20, R11
|
|
ADDQ $0x20, R14
|
|
DECQ R13
|
|
JNA emit_lit_memmove_long_match_emit_encodeBetterBlockAsm12Blarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_match_emit_encodeBetterBlockAsm12Blarge_forward_sse_loop_32:
|
|
MOVOU -32(R10)(R14*1), X4
|
|
MOVOU -16(R10)(R14*1), X5
|
|
MOVOA X4, -32(CX)(R14*1)
|
|
MOVOA X5, -16(CX)(R14*1)
|
|
ADDQ $0x20, R14
|
|
CMPQ R9, R14
|
|
JAE emit_lit_memmove_long_match_emit_encodeBetterBlockAsm12Blarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
MOVQ SI, CX
|
|
|
|
emit_literal_done_match_emit_encodeBetterBlockAsm12B:
|
|
ADDL R12, DX
|
|
ADDL $0x04, R12
|
|
MOVL DX, 12(SP)
|
|
|
|
// emitCopy
|
|
CMPL R12, $0x40
|
|
JBE two_byte_offset_short_match_nolit_encodeBetterBlockAsm12B
|
|
CMPL R8, $0x00000800
|
|
JAE long_offset_short_match_nolit_encodeBetterBlockAsm12B
|
|
MOVL $0x00000001, SI
|
|
LEAL 16(SI), SI
|
|
MOVB R8, 1(CX)
|
|
SHRL $0x08, R8
|
|
SHLL $0x05, R8
|
|
ORL R8, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x02, CX
|
|
SUBL $0x08, R12
|
|
|
|
// emitRepeat
|
|
LEAL -4(R12), R12
|
|
JMP cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b
|
|
MOVL R12, SI
|
|
LEAL -4(R12), R12
|
|
CMPL SI, $0x08
|
|
JBE repeat_two_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b
|
|
CMPL SI, $0x0c
|
|
JAE cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b
|
|
CMPL R8, $0x00000800
|
|
JB repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b:
|
|
CMPL R12, $0x00000104
|
|
JB repeat_three_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b
|
|
LEAL -256(R12), R12
|
|
MOVW $0x0019, (CX)
|
|
MOVW R12, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B
|
|
|
|
repeat_three_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b:
|
|
LEAL -4(R12), R12
|
|
MOVW $0x0015, (CX)
|
|
MOVB R12, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B
|
|
|
|
repeat_two_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b:
|
|
SHLL $0x02, R12
|
|
ORL $0x01, R12
|
|
MOVW R12, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B
|
|
|
|
repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b:
|
|
XORQ SI, SI
|
|
LEAL 1(SI)(R12*4), R12
|
|
MOVB R8, 1(CX)
|
|
SARL $0x08, R8
|
|
SHLL $0x05, R8
|
|
ORL R8, R12
|
|
MOVB R12, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B
|
|
|
|
long_offset_short_match_nolit_encodeBetterBlockAsm12B:
|
|
MOVB $0xee, (CX)
|
|
MOVW R8, 1(CX)
|
|
LEAL -60(R12), R12
|
|
ADDQ $0x03, CX
|
|
|
|
// emitRepeat
|
|
MOVL R12, SI
|
|
LEAL -4(R12), R12
|
|
CMPL SI, $0x08
|
|
JBE repeat_two_match_nolit_encodeBetterBlockAsm12B_emit_copy_short
|
|
CMPL SI, $0x0c
|
|
JAE cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short
|
|
CMPL R8, $0x00000800
|
|
JB repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short:
|
|
CMPL R12, $0x00000104
|
|
JB repeat_three_match_nolit_encodeBetterBlockAsm12B_emit_copy_short
|
|
LEAL -256(R12), R12
|
|
MOVW $0x0019, (CX)
|
|
MOVW R12, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B
|
|
|
|
repeat_three_match_nolit_encodeBetterBlockAsm12B_emit_copy_short:
|
|
LEAL -4(R12), R12
|
|
MOVW $0x0015, (CX)
|
|
MOVB R12, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B
|
|
|
|
repeat_two_match_nolit_encodeBetterBlockAsm12B_emit_copy_short:
|
|
SHLL $0x02, R12
|
|
ORL $0x01, R12
|
|
MOVW R12, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B
|
|
|
|
repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short:
|
|
XORQ SI, SI
|
|
LEAL 1(SI)(R12*4), R12
|
|
MOVB R8, 1(CX)
|
|
SARL $0x08, R8
|
|
SHLL $0x05, R8
|
|
ORL R8, R12
|
|
MOVB R12, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B
|
|
|
|
two_byte_offset_short_match_nolit_encodeBetterBlockAsm12B:
|
|
MOVL R12, SI
|
|
SHLL $0x02, SI
|
|
CMPL R12, $0x0c
|
|
JAE emit_copy_three_match_nolit_encodeBetterBlockAsm12B
|
|
CMPL R8, $0x00000800
|
|
JAE emit_copy_three_match_nolit_encodeBetterBlockAsm12B
|
|
LEAL -15(SI), SI
|
|
MOVB R8, 1(CX)
|
|
SHRL $0x08, R8
|
|
SHLL $0x05, R8
|
|
ORL R8, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B
|
|
|
|
emit_copy_three_match_nolit_encodeBetterBlockAsm12B:
|
|
LEAL -2(SI), SI
|
|
MOVB SI, (CX)
|
|
MOVW R8, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B
|
|
|
|
match_is_repeat_encodeBetterBlockAsm12B:
|
|
MOVL 12(SP), SI
|
|
CMPL SI, DI
|
|
JEQ emit_literal_done_match_emit_repeat_encodeBetterBlockAsm12B
|
|
MOVL DI, R9
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(SI*1), R10
|
|
SUBL SI, R9
|
|
LEAL -1(R9), SI
|
|
CMPL SI, $0x3c
|
|
JB one_byte_match_emit_repeat_encodeBetterBlockAsm12B
|
|
CMPL SI, $0x00000100
|
|
JB two_bytes_match_emit_repeat_encodeBetterBlockAsm12B
|
|
JB three_bytes_match_emit_repeat_encodeBetterBlockAsm12B
|
|
|
|
three_bytes_match_emit_repeat_encodeBetterBlockAsm12B:
|
|
MOVB $0xf4, (CX)
|
|
MOVW SI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm12B
|
|
|
|
two_bytes_match_emit_repeat_encodeBetterBlockAsm12B:
|
|
MOVB $0xf0, (CX)
|
|
MOVB SI, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL SI, $0x40
|
|
JB memmove_match_emit_repeat_encodeBetterBlockAsm12B
|
|
JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm12B
|
|
|
|
one_byte_match_emit_repeat_encodeBetterBlockAsm12B:
|
|
SHLB $0x02, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_match_emit_repeat_encodeBetterBlockAsm12B:
|
|
LEAQ (CX)(R9*1), SI
|
|
|
|
// genMemMoveShort
|
|
CMPQ R9, $0x04
|
|
JBE emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_4
|
|
CMPQ R9, $0x08
|
|
JB emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_4through7
|
|
CMPQ R9, $0x10
|
|
JBE emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_8through16
|
|
CMPQ R9, $0x20
|
|
JBE emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_4:
|
|
MOVL (R10), R11
|
|
MOVL R11, (CX)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_4through7:
|
|
MOVL (R10), R11
|
|
MOVL -4(R10)(R9*1), R10
|
|
MOVL R11, (CX)
|
|
MOVL R10, -4(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_8through16:
|
|
MOVQ (R10), R11
|
|
MOVQ -8(R10)(R9*1), R10
|
|
MOVQ R11, (CX)
|
|
MOVQ R10, -8(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_17through32:
|
|
MOVOU (R10), X0
|
|
MOVOU -16(R10)(R9*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_33through64:
|
|
MOVOU (R10), X0
|
|
MOVOU 16(R10), X1
|
|
MOVOU -32(R10)(R9*1), X2
|
|
MOVOU -16(R10)(R9*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
|
|
memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm12B:
|
|
MOVQ SI, CX
|
|
JMP emit_literal_done_match_emit_repeat_encodeBetterBlockAsm12B
|
|
|
|
memmove_long_match_emit_repeat_encodeBetterBlockAsm12B:
|
|
LEAQ (CX)(R9*1), SI
|
|
|
|
// genMemMoveLong
|
|
MOVOU (R10), X0
|
|
MOVOU 16(R10), X1
|
|
MOVOU -32(R10)(R9*1), X2
|
|
MOVOU -16(R10)(R9*1), X3
|
|
MOVQ R9, R13
|
|
SHRQ $0x05, R13
|
|
MOVQ CX, R11
|
|
ANDL $0x0000001f, R11
|
|
MOVQ $0x00000040, R14
|
|
SUBQ R11, R14
|
|
DECQ R13
|
|
JA emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm12Blarge_forward_sse_loop_32
|
|
LEAQ -32(R10)(R14*1), R11
|
|
LEAQ -32(CX)(R14*1), R15
|
|
|
|
emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm12Blarge_big_loop_back:
|
|
MOVOU (R11), X4
|
|
MOVOU 16(R11), X5
|
|
MOVOA X4, (R15)
|
|
MOVOA X5, 16(R15)
|
|
ADDQ $0x20, R15
|
|
ADDQ $0x20, R11
|
|
ADDQ $0x20, R14
|
|
DECQ R13
|
|
JNA emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm12Blarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm12Blarge_forward_sse_loop_32:
|
|
MOVOU -32(R10)(R14*1), X4
|
|
MOVOU -16(R10)(R14*1), X5
|
|
MOVOA X4, -32(CX)(R14*1)
|
|
MOVOA X5, -16(CX)(R14*1)
|
|
ADDQ $0x20, R14
|
|
CMPQ R9, R14
|
|
JAE emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm12Blarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
MOVQ SI, CX
|
|
|
|
emit_literal_done_match_emit_repeat_encodeBetterBlockAsm12B:
|
|
ADDL R12, DX
|
|
ADDL $0x04, R12
|
|
MOVL DX, 12(SP)
|
|
|
|
// emitRepeat
|
|
MOVL R12, SI
|
|
LEAL -4(R12), R12
|
|
CMPL SI, $0x08
|
|
JBE repeat_two_match_nolit_repeat_encodeBetterBlockAsm12B
|
|
CMPL SI, $0x0c
|
|
JAE cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm12B
|
|
CMPL R8, $0x00000800
|
|
JB repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm12B
|
|
|
|
cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm12B:
|
|
CMPL R12, $0x00000104
|
|
JB repeat_three_match_nolit_repeat_encodeBetterBlockAsm12B
|
|
LEAL -256(R12), R12
|
|
MOVW $0x0019, (CX)
|
|
MOVW R12, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B
|
|
|
|
repeat_three_match_nolit_repeat_encodeBetterBlockAsm12B:
|
|
LEAL -4(R12), R12
|
|
MOVW $0x0015, (CX)
|
|
MOVB R12, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B
|
|
|
|
repeat_two_match_nolit_repeat_encodeBetterBlockAsm12B:
|
|
SHLL $0x02, R12
|
|
ORL $0x01, R12
|
|
MOVW R12, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B
|
|
|
|
repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm12B:
|
|
XORQ SI, SI
|
|
LEAL 1(SI)(R12*4), R12
|
|
MOVB R8, 1(CX)
|
|
SARL $0x08, R8
|
|
SHLL $0x05, R8
|
|
ORL R8, R12
|
|
MOVB R12, (CX)
|
|
ADDQ $0x02, CX
|
|
|
|
match_nolit_emitcopy_end_encodeBetterBlockAsm12B:
|
|
CMPL DX, 8(SP)
|
|
JAE emit_remainder_encodeBetterBlockAsm12B
|
|
CMPQ CX, (SP)
|
|
JB match_nolit_dst_ok_encodeBetterBlockAsm12B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeBetterBlockAsm12B:
|
|
MOVQ $0x0000cf1bbcdcbf9b, SI
|
|
MOVQ $0x9e3779b1, R8
|
|
LEAQ 1(DI), DI
|
|
LEAQ -2(DX), R9
|
|
MOVQ (BX)(DI*1), R10
|
|
MOVQ 1(BX)(DI*1), R11
|
|
MOVQ (BX)(R9*1), R12
|
|
MOVQ 1(BX)(R9*1), R13
|
|
SHLQ $0x10, R10
|
|
IMULQ SI, R10
|
|
SHRQ $0x32, R10
|
|
SHLQ $0x20, R11
|
|
IMULQ R8, R11
|
|
SHRQ $0x34, R11
|
|
SHLQ $0x10, R12
|
|
IMULQ SI, R12
|
|
SHRQ $0x32, R12
|
|
SHLQ $0x20, R13
|
|
IMULQ R8, R13
|
|
SHRQ $0x34, R13
|
|
LEAQ 1(DI), R8
|
|
LEAQ 1(R9), R14
|
|
MOVL DI, (AX)(R10*4)
|
|
MOVL R9, (AX)(R12*4)
|
|
MOVL R8, 65536(AX)(R11*4)
|
|
MOVL R14, 65536(AX)(R13*4)
|
|
LEAQ 1(R9)(DI*1), R8
|
|
SHRQ $0x01, R8
|
|
ADDQ $0x01, DI
|
|
SUBQ $0x01, R9
|
|
|
|
index_loop_encodeBetterBlockAsm12B:
|
|
CMPQ R8, R9
|
|
JAE search_loop_encodeBetterBlockAsm12B
|
|
MOVQ (BX)(DI*1), R10
|
|
MOVQ (BX)(R8*1), R11
|
|
SHLQ $0x10, R10
|
|
IMULQ SI, R10
|
|
SHRQ $0x32, R10
|
|
SHLQ $0x10, R11
|
|
IMULQ SI, R11
|
|
SHRQ $0x32, R11
|
|
MOVL DI, (AX)(R10*4)
|
|
MOVL R8, (AX)(R11*4)
|
|
ADDQ $0x02, DI
|
|
ADDQ $0x02, R8
|
|
JMP index_loop_encodeBetterBlockAsm12B
|
|
|
|
emit_remainder_encodeBetterBlockAsm12B:
|
|
MOVQ src_len+32(FP), AX
|
|
SUBL 12(SP), AX
|
|
LEAQ 3(CX)(AX*1), AX
|
|
CMPQ AX, (SP)
|
|
JB emit_remainder_ok_encodeBetterBlockAsm12B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeBetterBlockAsm12B:
|
|
MOVQ src_len+32(FP), AX
|
|
MOVL 12(SP), DX
|
|
CMPL DX, AX
|
|
JEQ emit_literal_done_emit_remainder_encodeBetterBlockAsm12B
|
|
MOVL AX, SI
|
|
MOVL AX, 12(SP)
|
|
LEAQ (BX)(DX*1), AX
|
|
SUBL DX, SI
|
|
LEAL -1(SI), DX
|
|
CMPL DX, $0x3c
|
|
JB one_byte_emit_remainder_encodeBetterBlockAsm12B
|
|
CMPL DX, $0x00000100
|
|
JB two_bytes_emit_remainder_encodeBetterBlockAsm12B
|
|
JB three_bytes_emit_remainder_encodeBetterBlockAsm12B
|
|
|
|
three_bytes_emit_remainder_encodeBetterBlockAsm12B:
|
|
MOVB $0xf4, (CX)
|
|
MOVW DX, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_emit_remainder_encodeBetterBlockAsm12B
|
|
|
|
two_bytes_emit_remainder_encodeBetterBlockAsm12B:
|
|
MOVB $0xf0, (CX)
|
|
MOVB DL, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL DX, $0x40
|
|
JB memmove_emit_remainder_encodeBetterBlockAsm12B
|
|
JMP memmove_long_emit_remainder_encodeBetterBlockAsm12B
|
|
|
|
one_byte_emit_remainder_encodeBetterBlockAsm12B:
|
|
SHLB $0x02, DL
|
|
MOVB DL, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_emit_remainder_encodeBetterBlockAsm12B:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveShort
|
|
CMPQ BX, $0x03
|
|
JB emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_1or2
|
|
JE emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_3
|
|
CMPQ BX, $0x08
|
|
JB emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_4through7
|
|
CMPQ BX, $0x10
|
|
JBE emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_8through16
|
|
CMPQ BX, $0x20
|
|
JBE emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_1or2:
|
|
MOVB (AX), SI
|
|
MOVB -1(AX)(BX*1), AL
|
|
MOVB SI, (CX)
|
|
MOVB AL, -1(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_3:
|
|
MOVW (AX), SI
|
|
MOVB 2(AX), AL
|
|
MOVW SI, (CX)
|
|
MOVB AL, 2(CX)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_4through7:
|
|
MOVL (AX), SI
|
|
MOVL -4(AX)(BX*1), AX
|
|
MOVL SI, (CX)
|
|
MOVL AX, -4(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_8through16:
|
|
MOVQ (AX), SI
|
|
MOVQ -8(AX)(BX*1), AX
|
|
MOVQ SI, (CX)
|
|
MOVQ AX, -8(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_17through32:
|
|
MOVOU (AX), X0
|
|
MOVOU -16(AX)(BX*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_33through64:
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
|
|
memmove_end_copy_emit_remainder_encodeBetterBlockAsm12B:
|
|
MOVQ DX, CX
|
|
JMP emit_literal_done_emit_remainder_encodeBetterBlockAsm12B
|
|
|
|
memmove_long_emit_remainder_encodeBetterBlockAsm12B:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveLong
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVQ BX, DI
|
|
SHRQ $0x05, DI
|
|
MOVQ CX, SI
|
|
ANDL $0x0000001f, SI
|
|
MOVQ $0x00000040, R8
|
|
SUBQ SI, R8
|
|
DECQ DI
|
|
JA emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm12Blarge_forward_sse_loop_32
|
|
LEAQ -32(AX)(R8*1), SI
|
|
LEAQ -32(CX)(R8*1), R9
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm12Blarge_big_loop_back:
|
|
MOVOU (SI), X4
|
|
MOVOU 16(SI), X5
|
|
MOVOA X4, (R9)
|
|
MOVOA X5, 16(R9)
|
|
ADDQ $0x20, R9
|
|
ADDQ $0x20, SI
|
|
ADDQ $0x20, R8
|
|
DECQ DI
|
|
JNA emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm12Blarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm12Blarge_forward_sse_loop_32:
|
|
MOVOU -32(AX)(R8*1), X4
|
|
MOVOU -16(AX)(R8*1), X5
|
|
MOVOA X4, -32(CX)(R8*1)
|
|
MOVOA X5, -16(CX)(R8*1)
|
|
ADDQ $0x20, R8
|
|
CMPQ BX, R8
|
|
JAE emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm12Blarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
MOVQ DX, CX
|
|
|
|
emit_literal_done_emit_remainder_encodeBetterBlockAsm12B:
|
|
MOVQ dst_base+0(FP), AX
|
|
SUBQ AX, CX
|
|
MOVQ CX, ret+56(FP)
|
|
RET
|
|
|
|
// func encodeBetterBlockAsm10B(dst []byte, src []byte, tmp *[20480]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeBetterBlockAsm10B(SB), $24-64
|
|
MOVQ tmp+48(FP), AX
|
|
MOVQ dst_base+0(FP), CX
|
|
MOVQ $0x000000a0, DX
|
|
MOVQ AX, BX
|
|
PXOR X0, X0
|
|
|
|
zero_loop_encodeBetterBlockAsm10B:
|
|
MOVOU X0, (BX)
|
|
MOVOU X0, 16(BX)
|
|
MOVOU X0, 32(BX)
|
|
MOVOU X0, 48(BX)
|
|
MOVOU X0, 64(BX)
|
|
MOVOU X0, 80(BX)
|
|
MOVOU X0, 96(BX)
|
|
MOVOU X0, 112(BX)
|
|
ADDQ $0x80, BX
|
|
DECQ DX
|
|
JNZ zero_loop_encodeBetterBlockAsm10B
|
|
MOVL $0x00000000, 12(SP)
|
|
MOVQ src_len+32(FP), DX
|
|
LEAQ -6(DX), BX
|
|
LEAQ -8(DX), SI
|
|
MOVL SI, 8(SP)
|
|
SHRQ $0x05, DX
|
|
SUBL DX, BX
|
|
LEAQ (CX)(BX*1), BX
|
|
MOVQ BX, (SP)
|
|
MOVL $0x00000001, DX
|
|
MOVL $0x00000000, 16(SP)
|
|
MOVQ src_base+24(FP), BX
|
|
|
|
search_loop_encodeBetterBlockAsm10B:
|
|
MOVL DX, SI
|
|
SUBL 12(SP), SI
|
|
SHRL $0x05, SI
|
|
LEAL 1(DX)(SI*1), SI
|
|
CMPL SI, 8(SP)
|
|
JAE emit_remainder_encodeBetterBlockAsm10B
|
|
MOVQ (BX)(DX*1), DI
|
|
MOVL SI, 20(SP)
|
|
MOVQ $0x0000cf1bbcdcbf9b, R9
|
|
MOVQ $0x9e3779b1, SI
|
|
MOVQ DI, R10
|
|
MOVQ DI, R11
|
|
SHLQ $0x10, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x34, R10
|
|
SHLQ $0x20, R11
|
|
IMULQ SI, R11
|
|
SHRQ $0x36, R11
|
|
MOVL (AX)(R10*4), SI
|
|
MOVL 16384(AX)(R11*4), R8
|
|
MOVL DX, (AX)(R10*4)
|
|
MOVL DX, 16384(AX)(R11*4)
|
|
MOVQ (BX)(SI*1), R10
|
|
MOVQ (BX)(R8*1), R11
|
|
CMPQ R10, DI
|
|
JEQ candidate_match_encodeBetterBlockAsm10B
|
|
CMPQ R11, DI
|
|
JNE no_short_found_encodeBetterBlockAsm10B
|
|
MOVL R8, SI
|
|
JMP candidate_match_encodeBetterBlockAsm10B
|
|
|
|
no_short_found_encodeBetterBlockAsm10B:
|
|
CMPL R10, DI
|
|
JEQ candidate_match_encodeBetterBlockAsm10B
|
|
CMPL R11, DI
|
|
JEQ candidateS_match_encodeBetterBlockAsm10B
|
|
MOVL 20(SP), DX
|
|
JMP search_loop_encodeBetterBlockAsm10B
|
|
|
|
candidateS_match_encodeBetterBlockAsm10B:
|
|
SHRQ $0x08, DI
|
|
MOVQ DI, R10
|
|
SHLQ $0x10, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x34, R10
|
|
MOVL (AX)(R10*4), SI
|
|
INCL DX
|
|
MOVL DX, (AX)(R10*4)
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ candidate_match_encodeBetterBlockAsm10B
|
|
DECL DX
|
|
MOVL R8, SI
|
|
|
|
candidate_match_encodeBetterBlockAsm10B:
|
|
MOVL 12(SP), DI
|
|
TESTL SI, SI
|
|
JZ match_extend_back_end_encodeBetterBlockAsm10B
|
|
|
|
match_extend_back_loop_encodeBetterBlockAsm10B:
|
|
CMPL DX, DI
|
|
JBE match_extend_back_end_encodeBetterBlockAsm10B
|
|
MOVB -1(BX)(SI*1), R8
|
|
MOVB -1(BX)(DX*1), R9
|
|
CMPB R8, R9
|
|
JNE match_extend_back_end_encodeBetterBlockAsm10B
|
|
LEAL -1(DX), DX
|
|
DECL SI
|
|
JZ match_extend_back_end_encodeBetterBlockAsm10B
|
|
JMP match_extend_back_loop_encodeBetterBlockAsm10B
|
|
|
|
match_extend_back_end_encodeBetterBlockAsm10B:
|
|
MOVL DX, DI
|
|
SUBL 12(SP), DI
|
|
LEAQ 3(CX)(DI*1), DI
|
|
CMPQ DI, (SP)
|
|
JB match_dst_size_check_encodeBetterBlockAsm10B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeBetterBlockAsm10B:
|
|
MOVL DX, DI
|
|
ADDL $0x04, DX
|
|
ADDL $0x04, SI
|
|
MOVQ src_len+32(FP), R8
|
|
SUBL DX, R8
|
|
LEAQ (BX)(DX*1), R9
|
|
LEAQ (BX)(SI*1), R10
|
|
|
|
// matchLen
|
|
XORL R12, R12
|
|
|
|
matchlen_loopback_16_match_nolit_encodeBetterBlockAsm10B:
|
|
CMPL R8, $0x10
|
|
JB matchlen_match8_match_nolit_encodeBetterBlockAsm10B
|
|
MOVQ (R9)(R12*1), R11
|
|
MOVQ 8(R9)(R12*1), R13
|
|
XORQ (R10)(R12*1), R11
|
|
JNZ matchlen_bsf_8_match_nolit_encodeBetterBlockAsm10B
|
|
XORQ 8(R10)(R12*1), R13
|
|
JNZ matchlen_bsf_16match_nolit_encodeBetterBlockAsm10B
|
|
LEAL -16(R8), R8
|
|
LEAL 16(R12), R12
|
|
JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm10B
|
|
|
|
matchlen_bsf_16match_nolit_encodeBetterBlockAsm10B:
|
|
TZCNTQ R13, R13
|
|
SARQ $0x03, R13
|
|
LEAL 8(R12)(R13*1), R12
|
|
JMP match_nolit_end_encodeBetterBlockAsm10B
|
|
|
|
matchlen_match8_match_nolit_encodeBetterBlockAsm10B:
|
|
CMPL R8, $0x08
|
|
JB matchlen_match4_match_nolit_encodeBetterBlockAsm10B
|
|
MOVQ (R9)(R12*1), R11
|
|
XORQ (R10)(R12*1), R11
|
|
JNZ matchlen_bsf_8_match_nolit_encodeBetterBlockAsm10B
|
|
LEAL -8(R8), R8
|
|
LEAL 8(R12), R12
|
|
JMP matchlen_match4_match_nolit_encodeBetterBlockAsm10B
|
|
|
|
matchlen_bsf_8_match_nolit_encodeBetterBlockAsm10B:
|
|
TZCNTQ R11, R11
|
|
SARQ $0x03, R11
|
|
LEAL (R12)(R11*1), R12
|
|
JMP match_nolit_end_encodeBetterBlockAsm10B
|
|
|
|
matchlen_match4_match_nolit_encodeBetterBlockAsm10B:
|
|
CMPL R8, $0x04
|
|
JB matchlen_match2_match_nolit_encodeBetterBlockAsm10B
|
|
MOVL (R9)(R12*1), R11
|
|
CMPL (R10)(R12*1), R11
|
|
JNE matchlen_match2_match_nolit_encodeBetterBlockAsm10B
|
|
LEAL -4(R8), R8
|
|
LEAL 4(R12), R12
|
|
|
|
matchlen_match2_match_nolit_encodeBetterBlockAsm10B:
|
|
CMPL R8, $0x01
|
|
JE matchlen_match1_match_nolit_encodeBetterBlockAsm10B
|
|
JB match_nolit_end_encodeBetterBlockAsm10B
|
|
MOVW (R9)(R12*1), R11
|
|
CMPW (R10)(R12*1), R11
|
|
JNE matchlen_match1_match_nolit_encodeBetterBlockAsm10B
|
|
LEAL 2(R12), R12
|
|
SUBL $0x02, R8
|
|
JZ match_nolit_end_encodeBetterBlockAsm10B
|
|
|
|
matchlen_match1_match_nolit_encodeBetterBlockAsm10B:
|
|
MOVB (R9)(R12*1), R11
|
|
CMPB (R10)(R12*1), R11
|
|
JNE match_nolit_end_encodeBetterBlockAsm10B
|
|
LEAL 1(R12), R12
|
|
|
|
match_nolit_end_encodeBetterBlockAsm10B:
|
|
MOVL DX, R8
|
|
SUBL SI, R8
|
|
|
|
// Check if repeat
|
|
CMPL 16(SP), R8
|
|
JEQ match_is_repeat_encodeBetterBlockAsm10B
|
|
MOVL R8, 16(SP)
|
|
MOVL 12(SP), SI
|
|
CMPL SI, DI
|
|
JEQ emit_literal_done_match_emit_encodeBetterBlockAsm10B
|
|
MOVL DI, R9
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(SI*1), R10
|
|
SUBL SI, R9
|
|
LEAL -1(R9), SI
|
|
CMPL SI, $0x3c
|
|
JB one_byte_match_emit_encodeBetterBlockAsm10B
|
|
CMPL SI, $0x00000100
|
|
JB two_bytes_match_emit_encodeBetterBlockAsm10B
|
|
JB three_bytes_match_emit_encodeBetterBlockAsm10B
|
|
|
|
three_bytes_match_emit_encodeBetterBlockAsm10B:
|
|
MOVB $0xf4, (CX)
|
|
MOVW SI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_match_emit_encodeBetterBlockAsm10B
|
|
|
|
two_bytes_match_emit_encodeBetterBlockAsm10B:
|
|
MOVB $0xf0, (CX)
|
|
MOVB SI, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL SI, $0x40
|
|
JB memmove_match_emit_encodeBetterBlockAsm10B
|
|
JMP memmove_long_match_emit_encodeBetterBlockAsm10B
|
|
|
|
one_byte_match_emit_encodeBetterBlockAsm10B:
|
|
SHLB $0x02, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_match_emit_encodeBetterBlockAsm10B:
|
|
LEAQ (CX)(R9*1), SI
|
|
|
|
// genMemMoveShort
|
|
CMPQ R9, $0x04
|
|
JBE emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_4
|
|
CMPQ R9, $0x08
|
|
JB emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_4through7
|
|
CMPQ R9, $0x10
|
|
JBE emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_8through16
|
|
CMPQ R9, $0x20
|
|
JBE emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_4:
|
|
MOVL (R10), R11
|
|
MOVL R11, (CX)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_4through7:
|
|
MOVL (R10), R11
|
|
MOVL -4(R10)(R9*1), R10
|
|
MOVL R11, (CX)
|
|
MOVL R10, -4(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_8through16:
|
|
MOVQ (R10), R11
|
|
MOVQ -8(R10)(R9*1), R10
|
|
MOVQ R11, (CX)
|
|
MOVQ R10, -8(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_17through32:
|
|
MOVOU (R10), X0
|
|
MOVOU -16(R10)(R9*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_33through64:
|
|
MOVOU (R10), X0
|
|
MOVOU 16(R10), X1
|
|
MOVOU -32(R10)(R9*1), X2
|
|
MOVOU -16(R10)(R9*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
|
|
memmove_end_copy_match_emit_encodeBetterBlockAsm10B:
|
|
MOVQ SI, CX
|
|
JMP emit_literal_done_match_emit_encodeBetterBlockAsm10B
|
|
|
|
memmove_long_match_emit_encodeBetterBlockAsm10B:
|
|
LEAQ (CX)(R9*1), SI
|
|
|
|
// genMemMoveLong
|
|
MOVOU (R10), X0
|
|
MOVOU 16(R10), X1
|
|
MOVOU -32(R10)(R9*1), X2
|
|
MOVOU -16(R10)(R9*1), X3
|
|
MOVQ R9, R13
|
|
SHRQ $0x05, R13
|
|
MOVQ CX, R11
|
|
ANDL $0x0000001f, R11
|
|
MOVQ $0x00000040, R14
|
|
SUBQ R11, R14
|
|
DECQ R13
|
|
JA emit_lit_memmove_long_match_emit_encodeBetterBlockAsm10Blarge_forward_sse_loop_32
|
|
LEAQ -32(R10)(R14*1), R11
|
|
LEAQ -32(CX)(R14*1), R15
|
|
|
|
emit_lit_memmove_long_match_emit_encodeBetterBlockAsm10Blarge_big_loop_back:
|
|
MOVOU (R11), X4
|
|
MOVOU 16(R11), X5
|
|
MOVOA X4, (R15)
|
|
MOVOA X5, 16(R15)
|
|
ADDQ $0x20, R15
|
|
ADDQ $0x20, R11
|
|
ADDQ $0x20, R14
|
|
DECQ R13
|
|
JNA emit_lit_memmove_long_match_emit_encodeBetterBlockAsm10Blarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_match_emit_encodeBetterBlockAsm10Blarge_forward_sse_loop_32:
|
|
MOVOU -32(R10)(R14*1), X4
|
|
MOVOU -16(R10)(R14*1), X5
|
|
MOVOA X4, -32(CX)(R14*1)
|
|
MOVOA X5, -16(CX)(R14*1)
|
|
ADDQ $0x20, R14
|
|
CMPQ R9, R14
|
|
JAE emit_lit_memmove_long_match_emit_encodeBetterBlockAsm10Blarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
MOVQ SI, CX
|
|
|
|
emit_literal_done_match_emit_encodeBetterBlockAsm10B:
|
|
ADDL R12, DX
|
|
ADDL $0x04, R12
|
|
MOVL DX, 12(SP)
|
|
|
|
// emitCopy
|
|
CMPL R12, $0x40
|
|
JBE two_byte_offset_short_match_nolit_encodeBetterBlockAsm10B
|
|
CMPL R8, $0x00000800
|
|
JAE long_offset_short_match_nolit_encodeBetterBlockAsm10B
|
|
MOVL $0x00000001, SI
|
|
LEAL 16(SI), SI
|
|
MOVB R8, 1(CX)
|
|
SHRL $0x08, R8
|
|
SHLL $0x05, R8
|
|
ORL R8, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x02, CX
|
|
SUBL $0x08, R12
|
|
|
|
// emitRepeat
|
|
LEAL -4(R12), R12
|
|
JMP cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b
|
|
MOVL R12, SI
|
|
LEAL -4(R12), R12
|
|
CMPL SI, $0x08
|
|
JBE repeat_two_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b
|
|
CMPL SI, $0x0c
|
|
JAE cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b
|
|
CMPL R8, $0x00000800
|
|
JB repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b:
|
|
CMPL R12, $0x00000104
|
|
JB repeat_three_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b
|
|
LEAL -256(R12), R12
|
|
MOVW $0x0019, (CX)
|
|
MOVW R12, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B
|
|
|
|
repeat_three_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b:
|
|
LEAL -4(R12), R12
|
|
MOVW $0x0015, (CX)
|
|
MOVB R12, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B
|
|
|
|
repeat_two_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b:
|
|
SHLL $0x02, R12
|
|
ORL $0x01, R12
|
|
MOVW R12, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B
|
|
|
|
repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b:
|
|
XORQ SI, SI
|
|
LEAL 1(SI)(R12*4), R12
|
|
MOVB R8, 1(CX)
|
|
SARL $0x08, R8
|
|
SHLL $0x05, R8
|
|
ORL R8, R12
|
|
MOVB R12, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B
|
|
|
|
long_offset_short_match_nolit_encodeBetterBlockAsm10B:
|
|
MOVB $0xee, (CX)
|
|
MOVW R8, 1(CX)
|
|
LEAL -60(R12), R12
|
|
ADDQ $0x03, CX
|
|
|
|
// emitRepeat
|
|
MOVL R12, SI
|
|
LEAL -4(R12), R12
|
|
CMPL SI, $0x08
|
|
JBE repeat_two_match_nolit_encodeBetterBlockAsm10B_emit_copy_short
|
|
CMPL SI, $0x0c
|
|
JAE cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short
|
|
CMPL R8, $0x00000800
|
|
JB repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short:
|
|
CMPL R12, $0x00000104
|
|
JB repeat_three_match_nolit_encodeBetterBlockAsm10B_emit_copy_short
|
|
LEAL -256(R12), R12
|
|
MOVW $0x0019, (CX)
|
|
MOVW R12, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B
|
|
|
|
repeat_three_match_nolit_encodeBetterBlockAsm10B_emit_copy_short:
|
|
LEAL -4(R12), R12
|
|
MOVW $0x0015, (CX)
|
|
MOVB R12, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B
|
|
|
|
repeat_two_match_nolit_encodeBetterBlockAsm10B_emit_copy_short:
|
|
SHLL $0x02, R12
|
|
ORL $0x01, R12
|
|
MOVW R12, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B
|
|
|
|
repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short:
|
|
XORQ SI, SI
|
|
LEAL 1(SI)(R12*4), R12
|
|
MOVB R8, 1(CX)
|
|
SARL $0x08, R8
|
|
SHLL $0x05, R8
|
|
ORL R8, R12
|
|
MOVB R12, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B
|
|
|
|
two_byte_offset_short_match_nolit_encodeBetterBlockAsm10B:
|
|
MOVL R12, SI
|
|
SHLL $0x02, SI
|
|
CMPL R12, $0x0c
|
|
JAE emit_copy_three_match_nolit_encodeBetterBlockAsm10B
|
|
CMPL R8, $0x00000800
|
|
JAE emit_copy_three_match_nolit_encodeBetterBlockAsm10B
|
|
LEAL -15(SI), SI
|
|
MOVB R8, 1(CX)
|
|
SHRL $0x08, R8
|
|
SHLL $0x05, R8
|
|
ORL R8, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B
|
|
|
|
emit_copy_three_match_nolit_encodeBetterBlockAsm10B:
|
|
LEAL -2(SI), SI
|
|
MOVB SI, (CX)
|
|
MOVW R8, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B
|
|
|
|
match_is_repeat_encodeBetterBlockAsm10B:
|
|
MOVL 12(SP), SI
|
|
CMPL SI, DI
|
|
JEQ emit_literal_done_match_emit_repeat_encodeBetterBlockAsm10B
|
|
MOVL DI, R9
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(SI*1), R10
|
|
SUBL SI, R9
|
|
LEAL -1(R9), SI
|
|
CMPL SI, $0x3c
|
|
JB one_byte_match_emit_repeat_encodeBetterBlockAsm10B
|
|
CMPL SI, $0x00000100
|
|
JB two_bytes_match_emit_repeat_encodeBetterBlockAsm10B
|
|
JB three_bytes_match_emit_repeat_encodeBetterBlockAsm10B
|
|
|
|
three_bytes_match_emit_repeat_encodeBetterBlockAsm10B:
|
|
MOVB $0xf4, (CX)
|
|
MOVW SI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm10B
|
|
|
|
two_bytes_match_emit_repeat_encodeBetterBlockAsm10B:
|
|
MOVB $0xf0, (CX)
|
|
MOVB SI, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL SI, $0x40
|
|
JB memmove_match_emit_repeat_encodeBetterBlockAsm10B
|
|
JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm10B
|
|
|
|
one_byte_match_emit_repeat_encodeBetterBlockAsm10B:
|
|
SHLB $0x02, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_match_emit_repeat_encodeBetterBlockAsm10B:
|
|
LEAQ (CX)(R9*1), SI
|
|
|
|
// genMemMoveShort
|
|
CMPQ R9, $0x04
|
|
JBE emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_4
|
|
CMPQ R9, $0x08
|
|
JB emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_4through7
|
|
CMPQ R9, $0x10
|
|
JBE emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_8through16
|
|
CMPQ R9, $0x20
|
|
JBE emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_4:
|
|
MOVL (R10), R11
|
|
MOVL R11, (CX)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_4through7:
|
|
MOVL (R10), R11
|
|
MOVL -4(R10)(R9*1), R10
|
|
MOVL R11, (CX)
|
|
MOVL R10, -4(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_8through16:
|
|
MOVQ (R10), R11
|
|
MOVQ -8(R10)(R9*1), R10
|
|
MOVQ R11, (CX)
|
|
MOVQ R10, -8(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_17through32:
|
|
MOVOU (R10), X0
|
|
MOVOU -16(R10)(R9*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_33through64:
|
|
MOVOU (R10), X0
|
|
MOVOU 16(R10), X1
|
|
MOVOU -32(R10)(R9*1), X2
|
|
MOVOU -16(R10)(R9*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
|
|
memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm10B:
|
|
MOVQ SI, CX
|
|
JMP emit_literal_done_match_emit_repeat_encodeBetterBlockAsm10B
|
|
|
|
memmove_long_match_emit_repeat_encodeBetterBlockAsm10B:
|
|
LEAQ (CX)(R9*1), SI
|
|
|
|
// genMemMoveLong
|
|
MOVOU (R10), X0
|
|
MOVOU 16(R10), X1
|
|
MOVOU -32(R10)(R9*1), X2
|
|
MOVOU -16(R10)(R9*1), X3
|
|
MOVQ R9, R13
|
|
SHRQ $0x05, R13
|
|
MOVQ CX, R11
|
|
ANDL $0x0000001f, R11
|
|
MOVQ $0x00000040, R14
|
|
SUBQ R11, R14
|
|
DECQ R13
|
|
JA emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm10Blarge_forward_sse_loop_32
|
|
LEAQ -32(R10)(R14*1), R11
|
|
LEAQ -32(CX)(R14*1), R15
|
|
|
|
emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm10Blarge_big_loop_back:
|
|
MOVOU (R11), X4
|
|
MOVOU 16(R11), X5
|
|
MOVOA X4, (R15)
|
|
MOVOA X5, 16(R15)
|
|
ADDQ $0x20, R15
|
|
ADDQ $0x20, R11
|
|
ADDQ $0x20, R14
|
|
DECQ R13
|
|
JNA emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm10Blarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm10Blarge_forward_sse_loop_32:
|
|
MOVOU -32(R10)(R14*1), X4
|
|
MOVOU -16(R10)(R14*1), X5
|
|
MOVOA X4, -32(CX)(R14*1)
|
|
MOVOA X5, -16(CX)(R14*1)
|
|
ADDQ $0x20, R14
|
|
CMPQ R9, R14
|
|
JAE emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm10Blarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
MOVQ SI, CX
|
|
|
|
emit_literal_done_match_emit_repeat_encodeBetterBlockAsm10B:
|
|
ADDL R12, DX
|
|
ADDL $0x04, R12
|
|
MOVL DX, 12(SP)
|
|
|
|
// emitRepeat
|
|
MOVL R12, SI
|
|
LEAL -4(R12), R12
|
|
CMPL SI, $0x08
|
|
JBE repeat_two_match_nolit_repeat_encodeBetterBlockAsm10B
|
|
CMPL SI, $0x0c
|
|
JAE cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm10B
|
|
CMPL R8, $0x00000800
|
|
JB repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm10B
|
|
|
|
cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm10B:
|
|
CMPL R12, $0x00000104
|
|
JB repeat_three_match_nolit_repeat_encodeBetterBlockAsm10B
|
|
LEAL -256(R12), R12
|
|
MOVW $0x0019, (CX)
|
|
MOVW R12, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B
|
|
|
|
repeat_three_match_nolit_repeat_encodeBetterBlockAsm10B:
|
|
LEAL -4(R12), R12
|
|
MOVW $0x0015, (CX)
|
|
MOVB R12, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B
|
|
|
|
repeat_two_match_nolit_repeat_encodeBetterBlockAsm10B:
|
|
SHLL $0x02, R12
|
|
ORL $0x01, R12
|
|
MOVW R12, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B
|
|
|
|
repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm10B:
|
|
XORQ SI, SI
|
|
LEAL 1(SI)(R12*4), R12
|
|
MOVB R8, 1(CX)
|
|
SARL $0x08, R8
|
|
SHLL $0x05, R8
|
|
ORL R8, R12
|
|
MOVB R12, (CX)
|
|
ADDQ $0x02, CX
|
|
|
|
match_nolit_emitcopy_end_encodeBetterBlockAsm10B:
|
|
CMPL DX, 8(SP)
|
|
JAE emit_remainder_encodeBetterBlockAsm10B
|
|
CMPQ CX, (SP)
|
|
JB match_nolit_dst_ok_encodeBetterBlockAsm10B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeBetterBlockAsm10B:
|
|
MOVQ $0x0000cf1bbcdcbf9b, SI
|
|
MOVQ $0x9e3779b1, R8
|
|
LEAQ 1(DI), DI
|
|
LEAQ -2(DX), R9
|
|
MOVQ (BX)(DI*1), R10
|
|
MOVQ 1(BX)(DI*1), R11
|
|
MOVQ (BX)(R9*1), R12
|
|
MOVQ 1(BX)(R9*1), R13
|
|
SHLQ $0x10, R10
|
|
IMULQ SI, R10
|
|
SHRQ $0x34, R10
|
|
SHLQ $0x20, R11
|
|
IMULQ R8, R11
|
|
SHRQ $0x36, R11
|
|
SHLQ $0x10, R12
|
|
IMULQ SI, R12
|
|
SHRQ $0x34, R12
|
|
SHLQ $0x20, R13
|
|
IMULQ R8, R13
|
|
SHRQ $0x36, R13
|
|
LEAQ 1(DI), R8
|
|
LEAQ 1(R9), R14
|
|
MOVL DI, (AX)(R10*4)
|
|
MOVL R9, (AX)(R12*4)
|
|
MOVL R8, 16384(AX)(R11*4)
|
|
MOVL R14, 16384(AX)(R13*4)
|
|
LEAQ 1(R9)(DI*1), R8
|
|
SHRQ $0x01, R8
|
|
ADDQ $0x01, DI
|
|
SUBQ $0x01, R9
|
|
|
|
index_loop_encodeBetterBlockAsm10B:
|
|
CMPQ R8, R9
|
|
JAE search_loop_encodeBetterBlockAsm10B
|
|
MOVQ (BX)(DI*1), R10
|
|
MOVQ (BX)(R8*1), R11
|
|
SHLQ $0x10, R10
|
|
IMULQ SI, R10
|
|
SHRQ $0x34, R10
|
|
SHLQ $0x10, R11
|
|
IMULQ SI, R11
|
|
SHRQ $0x34, R11
|
|
MOVL DI, (AX)(R10*4)
|
|
MOVL R8, (AX)(R11*4)
|
|
ADDQ $0x02, DI
|
|
ADDQ $0x02, R8
|
|
JMP index_loop_encodeBetterBlockAsm10B
|
|
|
|
emit_remainder_encodeBetterBlockAsm10B:
|
|
MOVQ src_len+32(FP), AX
|
|
SUBL 12(SP), AX
|
|
LEAQ 3(CX)(AX*1), AX
|
|
CMPQ AX, (SP)
|
|
JB emit_remainder_ok_encodeBetterBlockAsm10B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeBetterBlockAsm10B:
|
|
MOVQ src_len+32(FP), AX
|
|
MOVL 12(SP), DX
|
|
CMPL DX, AX
|
|
JEQ emit_literal_done_emit_remainder_encodeBetterBlockAsm10B
|
|
MOVL AX, SI
|
|
MOVL AX, 12(SP)
|
|
LEAQ (BX)(DX*1), AX
|
|
SUBL DX, SI
|
|
LEAL -1(SI), DX
|
|
CMPL DX, $0x3c
|
|
JB one_byte_emit_remainder_encodeBetterBlockAsm10B
|
|
CMPL DX, $0x00000100
|
|
JB two_bytes_emit_remainder_encodeBetterBlockAsm10B
|
|
JB three_bytes_emit_remainder_encodeBetterBlockAsm10B
|
|
|
|
three_bytes_emit_remainder_encodeBetterBlockAsm10B:
|
|
MOVB $0xf4, (CX)
|
|
MOVW DX, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_emit_remainder_encodeBetterBlockAsm10B
|
|
|
|
two_bytes_emit_remainder_encodeBetterBlockAsm10B:
|
|
MOVB $0xf0, (CX)
|
|
MOVB DL, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL DX, $0x40
|
|
JB memmove_emit_remainder_encodeBetterBlockAsm10B
|
|
JMP memmove_long_emit_remainder_encodeBetterBlockAsm10B
|
|
|
|
one_byte_emit_remainder_encodeBetterBlockAsm10B:
|
|
SHLB $0x02, DL
|
|
MOVB DL, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_emit_remainder_encodeBetterBlockAsm10B:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveShort
|
|
CMPQ BX, $0x03
|
|
JB emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_1or2
|
|
JE emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_3
|
|
CMPQ BX, $0x08
|
|
JB emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_4through7
|
|
CMPQ BX, $0x10
|
|
JBE emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_8through16
|
|
CMPQ BX, $0x20
|
|
JBE emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_1or2:
|
|
MOVB (AX), SI
|
|
MOVB -1(AX)(BX*1), AL
|
|
MOVB SI, (CX)
|
|
MOVB AL, -1(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_3:
|
|
MOVW (AX), SI
|
|
MOVB 2(AX), AL
|
|
MOVW SI, (CX)
|
|
MOVB AL, 2(CX)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_4through7:
|
|
MOVL (AX), SI
|
|
MOVL -4(AX)(BX*1), AX
|
|
MOVL SI, (CX)
|
|
MOVL AX, -4(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_8through16:
|
|
MOVQ (AX), SI
|
|
MOVQ -8(AX)(BX*1), AX
|
|
MOVQ SI, (CX)
|
|
MOVQ AX, -8(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_17through32:
|
|
MOVOU (AX), X0
|
|
MOVOU -16(AX)(BX*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_33through64:
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
|
|
memmove_end_copy_emit_remainder_encodeBetterBlockAsm10B:
|
|
MOVQ DX, CX
|
|
JMP emit_literal_done_emit_remainder_encodeBetterBlockAsm10B
|
|
|
|
memmove_long_emit_remainder_encodeBetterBlockAsm10B:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveLong
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVQ BX, DI
|
|
SHRQ $0x05, DI
|
|
MOVQ CX, SI
|
|
ANDL $0x0000001f, SI
|
|
MOVQ $0x00000040, R8
|
|
SUBQ SI, R8
|
|
DECQ DI
|
|
JA emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm10Blarge_forward_sse_loop_32
|
|
LEAQ -32(AX)(R8*1), SI
|
|
LEAQ -32(CX)(R8*1), R9
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm10Blarge_big_loop_back:
|
|
MOVOU (SI), X4
|
|
MOVOU 16(SI), X5
|
|
MOVOA X4, (R9)
|
|
MOVOA X5, 16(R9)
|
|
ADDQ $0x20, R9
|
|
ADDQ $0x20, SI
|
|
ADDQ $0x20, R8
|
|
DECQ DI
|
|
JNA emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm10Blarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm10Blarge_forward_sse_loop_32:
|
|
MOVOU -32(AX)(R8*1), X4
|
|
MOVOU -16(AX)(R8*1), X5
|
|
MOVOA X4, -32(CX)(R8*1)
|
|
MOVOA X5, -16(CX)(R8*1)
|
|
ADDQ $0x20, R8
|
|
CMPQ BX, R8
|
|
JAE emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm10Blarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
MOVQ DX, CX
|
|
|
|
emit_literal_done_emit_remainder_encodeBetterBlockAsm10B:
|
|
MOVQ dst_base+0(FP), AX
|
|
SUBQ AX, CX
|
|
MOVQ CX, ret+56(FP)
|
|
RET
|
|
|
|
// func encodeBetterBlockAsm8B(dst []byte, src []byte, tmp *[5120]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeBetterBlockAsm8B(SB), $24-64
|
|
MOVQ tmp+48(FP), AX
|
|
MOVQ dst_base+0(FP), CX
|
|
MOVQ $0x00000028, DX
|
|
MOVQ AX, BX
|
|
PXOR X0, X0
|
|
|
|
zero_loop_encodeBetterBlockAsm8B:
|
|
MOVOU X0, (BX)
|
|
MOVOU X0, 16(BX)
|
|
MOVOU X0, 32(BX)
|
|
MOVOU X0, 48(BX)
|
|
MOVOU X0, 64(BX)
|
|
MOVOU X0, 80(BX)
|
|
MOVOU X0, 96(BX)
|
|
MOVOU X0, 112(BX)
|
|
ADDQ $0x80, BX
|
|
DECQ DX
|
|
JNZ zero_loop_encodeBetterBlockAsm8B
|
|
MOVL $0x00000000, 12(SP)
|
|
MOVQ src_len+32(FP), DX
|
|
LEAQ -6(DX), BX
|
|
LEAQ -8(DX), SI
|
|
MOVL SI, 8(SP)
|
|
SHRQ $0x05, DX
|
|
SUBL DX, BX
|
|
LEAQ (CX)(BX*1), BX
|
|
MOVQ BX, (SP)
|
|
MOVL $0x00000001, DX
|
|
MOVL $0x00000000, 16(SP)
|
|
MOVQ src_base+24(FP), BX
|
|
|
|
search_loop_encodeBetterBlockAsm8B:
|
|
MOVL DX, SI
|
|
SUBL 12(SP), SI
|
|
SHRL $0x04, SI
|
|
LEAL 1(DX)(SI*1), SI
|
|
CMPL SI, 8(SP)
|
|
JAE emit_remainder_encodeBetterBlockAsm8B
|
|
MOVQ (BX)(DX*1), DI
|
|
MOVL SI, 20(SP)
|
|
MOVQ $0x0000cf1bbcdcbf9b, R9
|
|
MOVQ $0x9e3779b1, SI
|
|
MOVQ DI, R10
|
|
MOVQ DI, R11
|
|
SHLQ $0x10, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x36, R10
|
|
SHLQ $0x20, R11
|
|
IMULQ SI, R11
|
|
SHRQ $0x38, R11
|
|
MOVL (AX)(R10*4), SI
|
|
MOVL 4096(AX)(R11*4), R8
|
|
MOVL DX, (AX)(R10*4)
|
|
MOVL DX, 4096(AX)(R11*4)
|
|
MOVQ (BX)(SI*1), R10
|
|
MOVQ (BX)(R8*1), R11
|
|
CMPQ R10, DI
|
|
JEQ candidate_match_encodeBetterBlockAsm8B
|
|
CMPQ R11, DI
|
|
JNE no_short_found_encodeBetterBlockAsm8B
|
|
MOVL R8, SI
|
|
JMP candidate_match_encodeBetterBlockAsm8B
|
|
|
|
no_short_found_encodeBetterBlockAsm8B:
|
|
CMPL R10, DI
|
|
JEQ candidate_match_encodeBetterBlockAsm8B
|
|
CMPL R11, DI
|
|
JEQ candidateS_match_encodeBetterBlockAsm8B
|
|
MOVL 20(SP), DX
|
|
JMP search_loop_encodeBetterBlockAsm8B
|
|
|
|
candidateS_match_encodeBetterBlockAsm8B:
|
|
SHRQ $0x08, DI
|
|
MOVQ DI, R10
|
|
SHLQ $0x10, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x36, R10
|
|
MOVL (AX)(R10*4), SI
|
|
INCL DX
|
|
MOVL DX, (AX)(R10*4)
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ candidate_match_encodeBetterBlockAsm8B
|
|
DECL DX
|
|
MOVL R8, SI
|
|
|
|
candidate_match_encodeBetterBlockAsm8B:
|
|
MOVL 12(SP), DI
|
|
TESTL SI, SI
|
|
JZ match_extend_back_end_encodeBetterBlockAsm8B
|
|
|
|
match_extend_back_loop_encodeBetterBlockAsm8B:
|
|
CMPL DX, DI
|
|
JBE match_extend_back_end_encodeBetterBlockAsm8B
|
|
MOVB -1(BX)(SI*1), R8
|
|
MOVB -1(BX)(DX*1), R9
|
|
CMPB R8, R9
|
|
JNE match_extend_back_end_encodeBetterBlockAsm8B
|
|
LEAL -1(DX), DX
|
|
DECL SI
|
|
JZ match_extend_back_end_encodeBetterBlockAsm8B
|
|
JMP match_extend_back_loop_encodeBetterBlockAsm8B
|
|
|
|
match_extend_back_end_encodeBetterBlockAsm8B:
|
|
MOVL DX, DI
|
|
SUBL 12(SP), DI
|
|
LEAQ 3(CX)(DI*1), DI
|
|
CMPQ DI, (SP)
|
|
JB match_dst_size_check_encodeBetterBlockAsm8B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeBetterBlockAsm8B:
|
|
MOVL DX, DI
|
|
ADDL $0x04, DX
|
|
ADDL $0x04, SI
|
|
MOVQ src_len+32(FP), R8
|
|
SUBL DX, R8
|
|
LEAQ (BX)(DX*1), R9
|
|
LEAQ (BX)(SI*1), R10
|
|
|
|
// matchLen
|
|
XORL R12, R12
|
|
|
|
matchlen_loopback_16_match_nolit_encodeBetterBlockAsm8B:
|
|
CMPL R8, $0x10
|
|
JB matchlen_match8_match_nolit_encodeBetterBlockAsm8B
|
|
MOVQ (R9)(R12*1), R11
|
|
MOVQ 8(R9)(R12*1), R13
|
|
XORQ (R10)(R12*1), R11
|
|
JNZ matchlen_bsf_8_match_nolit_encodeBetterBlockAsm8B
|
|
XORQ 8(R10)(R12*1), R13
|
|
JNZ matchlen_bsf_16match_nolit_encodeBetterBlockAsm8B
|
|
LEAL -16(R8), R8
|
|
LEAL 16(R12), R12
|
|
JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm8B
|
|
|
|
matchlen_bsf_16match_nolit_encodeBetterBlockAsm8B:
|
|
TZCNTQ R13, R13
|
|
SARQ $0x03, R13
|
|
LEAL 8(R12)(R13*1), R12
|
|
JMP match_nolit_end_encodeBetterBlockAsm8B
|
|
|
|
matchlen_match8_match_nolit_encodeBetterBlockAsm8B:
|
|
CMPL R8, $0x08
|
|
JB matchlen_match4_match_nolit_encodeBetterBlockAsm8B
|
|
MOVQ (R9)(R12*1), R11
|
|
XORQ (R10)(R12*1), R11
|
|
JNZ matchlen_bsf_8_match_nolit_encodeBetterBlockAsm8B
|
|
LEAL -8(R8), R8
|
|
LEAL 8(R12), R12
|
|
JMP matchlen_match4_match_nolit_encodeBetterBlockAsm8B
|
|
|
|
matchlen_bsf_8_match_nolit_encodeBetterBlockAsm8B:
|
|
TZCNTQ R11, R11
|
|
SARQ $0x03, R11
|
|
LEAL (R12)(R11*1), R12
|
|
JMP match_nolit_end_encodeBetterBlockAsm8B
|
|
|
|
matchlen_match4_match_nolit_encodeBetterBlockAsm8B:
|
|
CMPL R8, $0x04
|
|
JB matchlen_match2_match_nolit_encodeBetterBlockAsm8B
|
|
MOVL (R9)(R12*1), R11
|
|
CMPL (R10)(R12*1), R11
|
|
JNE matchlen_match2_match_nolit_encodeBetterBlockAsm8B
|
|
LEAL -4(R8), R8
|
|
LEAL 4(R12), R12
|
|
|
|
matchlen_match2_match_nolit_encodeBetterBlockAsm8B:
|
|
CMPL R8, $0x01
|
|
JE matchlen_match1_match_nolit_encodeBetterBlockAsm8B
|
|
JB match_nolit_end_encodeBetterBlockAsm8B
|
|
MOVW (R9)(R12*1), R11
|
|
CMPW (R10)(R12*1), R11
|
|
JNE matchlen_match1_match_nolit_encodeBetterBlockAsm8B
|
|
LEAL 2(R12), R12
|
|
SUBL $0x02, R8
|
|
JZ match_nolit_end_encodeBetterBlockAsm8B
|
|
|
|
matchlen_match1_match_nolit_encodeBetterBlockAsm8B:
|
|
MOVB (R9)(R12*1), R11
|
|
CMPB (R10)(R12*1), R11
|
|
JNE match_nolit_end_encodeBetterBlockAsm8B
|
|
LEAL 1(R12), R12
|
|
|
|
match_nolit_end_encodeBetterBlockAsm8B:
|
|
MOVL DX, R8
|
|
SUBL SI, R8
|
|
|
|
// Check if repeat
|
|
CMPL 16(SP), R8
|
|
JEQ match_is_repeat_encodeBetterBlockAsm8B
|
|
MOVL R8, 16(SP)
|
|
MOVL 12(SP), SI
|
|
CMPL SI, DI
|
|
JEQ emit_literal_done_match_emit_encodeBetterBlockAsm8B
|
|
MOVL DI, R9
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(SI*1), R10
|
|
SUBL SI, R9
|
|
LEAL -1(R9), SI
|
|
CMPL SI, $0x3c
|
|
JB one_byte_match_emit_encodeBetterBlockAsm8B
|
|
CMPL SI, $0x00000100
|
|
JB two_bytes_match_emit_encodeBetterBlockAsm8B
|
|
JB three_bytes_match_emit_encodeBetterBlockAsm8B
|
|
|
|
three_bytes_match_emit_encodeBetterBlockAsm8B:
|
|
MOVB $0xf4, (CX)
|
|
MOVW SI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_match_emit_encodeBetterBlockAsm8B
|
|
|
|
two_bytes_match_emit_encodeBetterBlockAsm8B:
|
|
MOVB $0xf0, (CX)
|
|
MOVB SI, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL SI, $0x40
|
|
JB memmove_match_emit_encodeBetterBlockAsm8B
|
|
JMP memmove_long_match_emit_encodeBetterBlockAsm8B
|
|
|
|
one_byte_match_emit_encodeBetterBlockAsm8B:
|
|
SHLB $0x02, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_match_emit_encodeBetterBlockAsm8B:
|
|
LEAQ (CX)(R9*1), SI
|
|
|
|
// genMemMoveShort
|
|
CMPQ R9, $0x04
|
|
JBE emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_4
|
|
CMPQ R9, $0x08
|
|
JB emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_4through7
|
|
CMPQ R9, $0x10
|
|
JBE emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_8through16
|
|
CMPQ R9, $0x20
|
|
JBE emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_4:
|
|
MOVL (R10), R11
|
|
MOVL R11, (CX)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_4through7:
|
|
MOVL (R10), R11
|
|
MOVL -4(R10)(R9*1), R10
|
|
MOVL R11, (CX)
|
|
MOVL R10, -4(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_8through16:
|
|
MOVQ (R10), R11
|
|
MOVQ -8(R10)(R9*1), R10
|
|
MOVQ R11, (CX)
|
|
MOVQ R10, -8(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_17through32:
|
|
MOVOU (R10), X0
|
|
MOVOU -16(R10)(R9*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_33through64:
|
|
MOVOU (R10), X0
|
|
MOVOU 16(R10), X1
|
|
MOVOU -32(R10)(R9*1), X2
|
|
MOVOU -16(R10)(R9*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
|
|
memmove_end_copy_match_emit_encodeBetterBlockAsm8B:
|
|
MOVQ SI, CX
|
|
JMP emit_literal_done_match_emit_encodeBetterBlockAsm8B
|
|
|
|
memmove_long_match_emit_encodeBetterBlockAsm8B:
|
|
LEAQ (CX)(R9*1), SI
|
|
|
|
// genMemMoveLong
|
|
MOVOU (R10), X0
|
|
MOVOU 16(R10), X1
|
|
MOVOU -32(R10)(R9*1), X2
|
|
MOVOU -16(R10)(R9*1), X3
|
|
MOVQ R9, R13
|
|
SHRQ $0x05, R13
|
|
MOVQ CX, R11
|
|
ANDL $0x0000001f, R11
|
|
MOVQ $0x00000040, R14
|
|
SUBQ R11, R14
|
|
DECQ R13
|
|
JA emit_lit_memmove_long_match_emit_encodeBetterBlockAsm8Blarge_forward_sse_loop_32
|
|
LEAQ -32(R10)(R14*1), R11
|
|
LEAQ -32(CX)(R14*1), R15
|
|
|
|
emit_lit_memmove_long_match_emit_encodeBetterBlockAsm8Blarge_big_loop_back:
|
|
MOVOU (R11), X4
|
|
MOVOU 16(R11), X5
|
|
MOVOA X4, (R15)
|
|
MOVOA X5, 16(R15)
|
|
ADDQ $0x20, R15
|
|
ADDQ $0x20, R11
|
|
ADDQ $0x20, R14
|
|
DECQ R13
|
|
JNA emit_lit_memmove_long_match_emit_encodeBetterBlockAsm8Blarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_match_emit_encodeBetterBlockAsm8Blarge_forward_sse_loop_32:
|
|
MOVOU -32(R10)(R14*1), X4
|
|
MOVOU -16(R10)(R14*1), X5
|
|
MOVOA X4, -32(CX)(R14*1)
|
|
MOVOA X5, -16(CX)(R14*1)
|
|
ADDQ $0x20, R14
|
|
CMPQ R9, R14
|
|
JAE emit_lit_memmove_long_match_emit_encodeBetterBlockAsm8Blarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
MOVQ SI, CX
|
|
|
|
emit_literal_done_match_emit_encodeBetterBlockAsm8B:
|
|
ADDL R12, DX
|
|
ADDL $0x04, R12
|
|
MOVL DX, 12(SP)
|
|
|
|
// emitCopy
|
|
CMPL R12, $0x40
|
|
JBE two_byte_offset_short_match_nolit_encodeBetterBlockAsm8B
|
|
CMPL R8, $0x00000800
|
|
JAE long_offset_short_match_nolit_encodeBetterBlockAsm8B
|
|
MOVL $0x00000001, SI
|
|
LEAL 16(SI), SI
|
|
MOVB R8, 1(CX)
|
|
SHRL $0x08, R8
|
|
SHLL $0x05, R8
|
|
ORL R8, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x02, CX
|
|
SUBL $0x08, R12
|
|
|
|
// emitRepeat
|
|
LEAL -4(R12), R12
|
|
JMP cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b
|
|
MOVL R12, SI
|
|
LEAL -4(R12), R12
|
|
CMPL SI, $0x08
|
|
JBE repeat_two_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b
|
|
CMPL SI, $0x0c
|
|
JAE cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b:
|
|
CMPL R12, $0x00000104
|
|
JB repeat_three_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b
|
|
LEAL -256(R12), R12
|
|
MOVW $0x0019, (CX)
|
|
MOVW R12, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B
|
|
|
|
repeat_three_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b:
|
|
LEAL -4(R12), R12
|
|
MOVW $0x0015, (CX)
|
|
MOVB R12, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B
|
|
|
|
repeat_two_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b:
|
|
SHLL $0x02, R12
|
|
ORL $0x01, R12
|
|
MOVW R12, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B
|
|
XORQ SI, SI
|
|
LEAL 1(SI)(R12*4), R12
|
|
MOVB R8, 1(CX)
|
|
SARL $0x08, R8
|
|
SHLL $0x05, R8
|
|
ORL R8, R12
|
|
MOVB R12, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B
|
|
|
|
long_offset_short_match_nolit_encodeBetterBlockAsm8B:
|
|
MOVB $0xee, (CX)
|
|
MOVW R8, 1(CX)
|
|
LEAL -60(R12), R12
|
|
ADDQ $0x03, CX
|
|
|
|
// emitRepeat
|
|
MOVL R12, SI
|
|
LEAL -4(R12), R12
|
|
CMPL SI, $0x08
|
|
JBE repeat_two_match_nolit_encodeBetterBlockAsm8B_emit_copy_short
|
|
CMPL SI, $0x0c
|
|
JAE cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm8B_emit_copy_short
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm8B_emit_copy_short:
|
|
CMPL R12, $0x00000104
|
|
JB repeat_three_match_nolit_encodeBetterBlockAsm8B_emit_copy_short
|
|
LEAL -256(R12), R12
|
|
MOVW $0x0019, (CX)
|
|
MOVW R12, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B
|
|
|
|
repeat_three_match_nolit_encodeBetterBlockAsm8B_emit_copy_short:
|
|
LEAL -4(R12), R12
|
|
MOVW $0x0015, (CX)
|
|
MOVB R12, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B
|
|
|
|
repeat_two_match_nolit_encodeBetterBlockAsm8B_emit_copy_short:
|
|
SHLL $0x02, R12
|
|
ORL $0x01, R12
|
|
MOVW R12, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B
|
|
XORQ SI, SI
|
|
LEAL 1(SI)(R12*4), R12
|
|
MOVB R8, 1(CX)
|
|
SARL $0x08, R8
|
|
SHLL $0x05, R8
|
|
ORL R8, R12
|
|
MOVB R12, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B
|
|
|
|
two_byte_offset_short_match_nolit_encodeBetterBlockAsm8B:
|
|
MOVL R12, SI
|
|
SHLL $0x02, SI
|
|
CMPL R12, $0x0c
|
|
JAE emit_copy_three_match_nolit_encodeBetterBlockAsm8B
|
|
LEAL -15(SI), SI
|
|
MOVB R8, 1(CX)
|
|
SHRL $0x08, R8
|
|
SHLL $0x05, R8
|
|
ORL R8, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B
|
|
|
|
emit_copy_three_match_nolit_encodeBetterBlockAsm8B:
|
|
LEAL -2(SI), SI
|
|
MOVB SI, (CX)
|
|
MOVW R8, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B
|
|
|
|
match_is_repeat_encodeBetterBlockAsm8B:
|
|
MOVL 12(SP), SI
|
|
CMPL SI, DI
|
|
JEQ emit_literal_done_match_emit_repeat_encodeBetterBlockAsm8B
|
|
MOVL DI, R8
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(SI*1), R9
|
|
SUBL SI, R8
|
|
LEAL -1(R8), SI
|
|
CMPL SI, $0x3c
|
|
JB one_byte_match_emit_repeat_encodeBetterBlockAsm8B
|
|
CMPL SI, $0x00000100
|
|
JB two_bytes_match_emit_repeat_encodeBetterBlockAsm8B
|
|
JB three_bytes_match_emit_repeat_encodeBetterBlockAsm8B
|
|
|
|
three_bytes_match_emit_repeat_encodeBetterBlockAsm8B:
|
|
MOVB $0xf4, (CX)
|
|
MOVW SI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm8B
|
|
|
|
two_bytes_match_emit_repeat_encodeBetterBlockAsm8B:
|
|
MOVB $0xf0, (CX)
|
|
MOVB SI, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL SI, $0x40
|
|
JB memmove_match_emit_repeat_encodeBetterBlockAsm8B
|
|
JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm8B
|
|
|
|
one_byte_match_emit_repeat_encodeBetterBlockAsm8B:
|
|
SHLB $0x02, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_match_emit_repeat_encodeBetterBlockAsm8B:
|
|
LEAQ (CX)(R8*1), SI
|
|
|
|
// genMemMoveShort
|
|
CMPQ R8, $0x04
|
|
JBE emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_4
|
|
CMPQ R8, $0x08
|
|
JB emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_4through7
|
|
CMPQ R8, $0x10
|
|
JBE emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_8through16
|
|
CMPQ R8, $0x20
|
|
JBE emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_4:
|
|
MOVL (R9), R10
|
|
MOVL R10, (CX)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_4through7:
|
|
MOVL (R9), R10
|
|
MOVL -4(R9)(R8*1), R9
|
|
MOVL R10, (CX)
|
|
MOVL R9, -4(CX)(R8*1)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_8through16:
|
|
MOVQ (R9), R10
|
|
MOVQ -8(R9)(R8*1), R9
|
|
MOVQ R10, (CX)
|
|
MOVQ R9, -8(CX)(R8*1)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_17through32:
|
|
MOVOU (R9), X0
|
|
MOVOU -16(R9)(R8*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(R8*1)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_33through64:
|
|
MOVOU (R9), X0
|
|
MOVOU 16(R9), X1
|
|
MOVOU -32(R9)(R8*1), X2
|
|
MOVOU -16(R9)(R8*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R8*1)
|
|
MOVOU X3, -16(CX)(R8*1)
|
|
|
|
memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm8B:
|
|
MOVQ SI, CX
|
|
JMP emit_literal_done_match_emit_repeat_encodeBetterBlockAsm8B
|
|
|
|
memmove_long_match_emit_repeat_encodeBetterBlockAsm8B:
|
|
LEAQ (CX)(R8*1), SI
|
|
|
|
// genMemMoveLong
|
|
MOVOU (R9), X0
|
|
MOVOU 16(R9), X1
|
|
MOVOU -32(R9)(R8*1), X2
|
|
MOVOU -16(R9)(R8*1), X3
|
|
MOVQ R8, R11
|
|
SHRQ $0x05, R11
|
|
MOVQ CX, R10
|
|
ANDL $0x0000001f, R10
|
|
MOVQ $0x00000040, R13
|
|
SUBQ R10, R13
|
|
DECQ R11
|
|
JA emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm8Blarge_forward_sse_loop_32
|
|
LEAQ -32(R9)(R13*1), R10
|
|
LEAQ -32(CX)(R13*1), R14
|
|
|
|
emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm8Blarge_big_loop_back:
|
|
MOVOU (R10), X4
|
|
MOVOU 16(R10), X5
|
|
MOVOA X4, (R14)
|
|
MOVOA X5, 16(R14)
|
|
ADDQ $0x20, R14
|
|
ADDQ $0x20, R10
|
|
ADDQ $0x20, R13
|
|
DECQ R11
|
|
JNA emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm8Blarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm8Blarge_forward_sse_loop_32:
|
|
MOVOU -32(R9)(R13*1), X4
|
|
MOVOU -16(R9)(R13*1), X5
|
|
MOVOA X4, -32(CX)(R13*1)
|
|
MOVOA X5, -16(CX)(R13*1)
|
|
ADDQ $0x20, R13
|
|
CMPQ R8, R13
|
|
JAE emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm8Blarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R8*1)
|
|
MOVOU X3, -16(CX)(R8*1)
|
|
MOVQ SI, CX
|
|
|
|
emit_literal_done_match_emit_repeat_encodeBetterBlockAsm8B:
|
|
ADDL R12, DX
|
|
ADDL $0x04, R12
|
|
MOVL DX, 12(SP)
|
|
|
|
// emitRepeat
|
|
MOVL R12, SI
|
|
LEAL -4(R12), R12
|
|
CMPL SI, $0x08
|
|
JBE repeat_two_match_nolit_repeat_encodeBetterBlockAsm8B
|
|
CMPL SI, $0x0c
|
|
JAE cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm8B
|
|
|
|
cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm8B:
|
|
CMPL R12, $0x00000104
|
|
JB repeat_three_match_nolit_repeat_encodeBetterBlockAsm8B
|
|
LEAL -256(R12), R12
|
|
MOVW $0x0019, (CX)
|
|
MOVW R12, 2(CX)
|
|
ADDQ $0x04, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B
|
|
|
|
repeat_three_match_nolit_repeat_encodeBetterBlockAsm8B:
|
|
LEAL -4(R12), R12
|
|
MOVW $0x0015, (CX)
|
|
MOVB R12, 2(CX)
|
|
ADDQ $0x03, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B
|
|
|
|
repeat_two_match_nolit_repeat_encodeBetterBlockAsm8B:
|
|
SHLL $0x02, R12
|
|
ORL $0x01, R12
|
|
MOVW R12, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B
|
|
XORQ SI, SI
|
|
LEAL 1(SI)(R12*4), R12
|
|
MOVB R8, 1(CX)
|
|
SARL $0x08, R8
|
|
SHLL $0x05, R8
|
|
ORL R8, R12
|
|
MOVB R12, (CX)
|
|
ADDQ $0x02, CX
|
|
|
|
match_nolit_emitcopy_end_encodeBetterBlockAsm8B:
|
|
CMPL DX, 8(SP)
|
|
JAE emit_remainder_encodeBetterBlockAsm8B
|
|
CMPQ CX, (SP)
|
|
JB match_nolit_dst_ok_encodeBetterBlockAsm8B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeBetterBlockAsm8B:
|
|
MOVQ $0x0000cf1bbcdcbf9b, SI
|
|
MOVQ $0x9e3779b1, R8
|
|
LEAQ 1(DI), DI
|
|
LEAQ -2(DX), R9
|
|
MOVQ (BX)(DI*1), R10
|
|
MOVQ 1(BX)(DI*1), R11
|
|
MOVQ (BX)(R9*1), R12
|
|
MOVQ 1(BX)(R9*1), R13
|
|
SHLQ $0x10, R10
|
|
IMULQ SI, R10
|
|
SHRQ $0x36, R10
|
|
SHLQ $0x20, R11
|
|
IMULQ R8, R11
|
|
SHRQ $0x38, R11
|
|
SHLQ $0x10, R12
|
|
IMULQ SI, R12
|
|
SHRQ $0x36, R12
|
|
SHLQ $0x20, R13
|
|
IMULQ R8, R13
|
|
SHRQ $0x38, R13
|
|
LEAQ 1(DI), R8
|
|
LEAQ 1(R9), R14
|
|
MOVL DI, (AX)(R10*4)
|
|
MOVL R9, (AX)(R12*4)
|
|
MOVL R8, 4096(AX)(R11*4)
|
|
MOVL R14, 4096(AX)(R13*4)
|
|
LEAQ 1(R9)(DI*1), R8
|
|
SHRQ $0x01, R8
|
|
ADDQ $0x01, DI
|
|
SUBQ $0x01, R9
|
|
|
|
index_loop_encodeBetterBlockAsm8B:
|
|
CMPQ R8, R9
|
|
JAE search_loop_encodeBetterBlockAsm8B
|
|
MOVQ (BX)(DI*1), R10
|
|
MOVQ (BX)(R8*1), R11
|
|
SHLQ $0x10, R10
|
|
IMULQ SI, R10
|
|
SHRQ $0x36, R10
|
|
SHLQ $0x10, R11
|
|
IMULQ SI, R11
|
|
SHRQ $0x36, R11
|
|
MOVL DI, (AX)(R10*4)
|
|
MOVL R8, (AX)(R11*4)
|
|
ADDQ $0x02, DI
|
|
ADDQ $0x02, R8
|
|
JMP index_loop_encodeBetterBlockAsm8B
|
|
|
|
emit_remainder_encodeBetterBlockAsm8B:
|
|
MOVQ src_len+32(FP), AX
|
|
SUBL 12(SP), AX
|
|
LEAQ 3(CX)(AX*1), AX
|
|
CMPQ AX, (SP)
|
|
JB emit_remainder_ok_encodeBetterBlockAsm8B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeBetterBlockAsm8B:
|
|
MOVQ src_len+32(FP), AX
|
|
MOVL 12(SP), DX
|
|
CMPL DX, AX
|
|
JEQ emit_literal_done_emit_remainder_encodeBetterBlockAsm8B
|
|
MOVL AX, SI
|
|
MOVL AX, 12(SP)
|
|
LEAQ (BX)(DX*1), AX
|
|
SUBL DX, SI
|
|
LEAL -1(SI), DX
|
|
CMPL DX, $0x3c
|
|
JB one_byte_emit_remainder_encodeBetterBlockAsm8B
|
|
CMPL DX, $0x00000100
|
|
JB two_bytes_emit_remainder_encodeBetterBlockAsm8B
|
|
JB three_bytes_emit_remainder_encodeBetterBlockAsm8B
|
|
|
|
three_bytes_emit_remainder_encodeBetterBlockAsm8B:
|
|
MOVB $0xf4, (CX)
|
|
MOVW DX, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_emit_remainder_encodeBetterBlockAsm8B
|
|
|
|
two_bytes_emit_remainder_encodeBetterBlockAsm8B:
|
|
MOVB $0xf0, (CX)
|
|
MOVB DL, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL DX, $0x40
|
|
JB memmove_emit_remainder_encodeBetterBlockAsm8B
|
|
JMP memmove_long_emit_remainder_encodeBetterBlockAsm8B
|
|
|
|
one_byte_emit_remainder_encodeBetterBlockAsm8B:
|
|
SHLB $0x02, DL
|
|
MOVB DL, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_emit_remainder_encodeBetterBlockAsm8B:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveShort
|
|
CMPQ BX, $0x03
|
|
JB emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_1or2
|
|
JE emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_3
|
|
CMPQ BX, $0x08
|
|
JB emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_4through7
|
|
CMPQ BX, $0x10
|
|
JBE emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_8through16
|
|
CMPQ BX, $0x20
|
|
JBE emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_1or2:
|
|
MOVB (AX), SI
|
|
MOVB -1(AX)(BX*1), AL
|
|
MOVB SI, (CX)
|
|
MOVB AL, -1(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_3:
|
|
MOVW (AX), SI
|
|
MOVB 2(AX), AL
|
|
MOVW SI, (CX)
|
|
MOVB AL, 2(CX)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_4through7:
|
|
MOVL (AX), SI
|
|
MOVL -4(AX)(BX*1), AX
|
|
MOVL SI, (CX)
|
|
MOVL AX, -4(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_8through16:
|
|
MOVQ (AX), SI
|
|
MOVQ -8(AX)(BX*1), AX
|
|
MOVQ SI, (CX)
|
|
MOVQ AX, -8(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_17through32:
|
|
MOVOU (AX), X0
|
|
MOVOU -16(AX)(BX*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_33through64:
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
|
|
memmove_end_copy_emit_remainder_encodeBetterBlockAsm8B:
|
|
MOVQ DX, CX
|
|
JMP emit_literal_done_emit_remainder_encodeBetterBlockAsm8B
|
|
|
|
memmove_long_emit_remainder_encodeBetterBlockAsm8B:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveLong
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVQ BX, DI
|
|
SHRQ $0x05, DI
|
|
MOVQ CX, SI
|
|
ANDL $0x0000001f, SI
|
|
MOVQ $0x00000040, R8
|
|
SUBQ SI, R8
|
|
DECQ DI
|
|
JA emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm8Blarge_forward_sse_loop_32
|
|
LEAQ -32(AX)(R8*1), SI
|
|
LEAQ -32(CX)(R8*1), R9
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm8Blarge_big_loop_back:
|
|
MOVOU (SI), X4
|
|
MOVOU 16(SI), X5
|
|
MOVOA X4, (R9)
|
|
MOVOA X5, 16(R9)
|
|
ADDQ $0x20, R9
|
|
ADDQ $0x20, SI
|
|
ADDQ $0x20, R8
|
|
DECQ DI
|
|
JNA emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm8Blarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm8Blarge_forward_sse_loop_32:
|
|
MOVOU -32(AX)(R8*1), X4
|
|
MOVOU -16(AX)(R8*1), X5
|
|
MOVOA X4, -32(CX)(R8*1)
|
|
MOVOA X5, -16(CX)(R8*1)
|
|
ADDQ $0x20, R8
|
|
CMPQ BX, R8
|
|
JAE emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm8Blarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
MOVQ DX, CX
|
|
|
|
emit_literal_done_emit_remainder_encodeBetterBlockAsm8B:
|
|
MOVQ dst_base+0(FP), AX
|
|
SUBQ AX, CX
|
|
MOVQ CX, ret+56(FP)
|
|
RET
|
|
|
|
// func encodeSnappyBlockAsm(dst []byte, src []byte, tmp *[65536]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeSnappyBlockAsm(SB), $24-64
|
|
MOVQ tmp+48(FP), AX
|
|
MOVQ dst_base+0(FP), CX
|
|
MOVQ $0x00000200, DX
|
|
MOVQ AX, BX
|
|
PXOR X0, X0
|
|
|
|
zero_loop_encodeSnappyBlockAsm:
|
|
MOVOU X0, (BX)
|
|
MOVOU X0, 16(BX)
|
|
MOVOU X0, 32(BX)
|
|
MOVOU X0, 48(BX)
|
|
MOVOU X0, 64(BX)
|
|
MOVOU X0, 80(BX)
|
|
MOVOU X0, 96(BX)
|
|
MOVOU X0, 112(BX)
|
|
ADDQ $0x80, BX
|
|
DECQ DX
|
|
JNZ zero_loop_encodeSnappyBlockAsm
|
|
MOVL $0x00000000, 12(SP)
|
|
MOVQ src_len+32(FP), DX
|
|
LEAQ -9(DX), BX
|
|
LEAQ -8(DX), SI
|
|
MOVL SI, 8(SP)
|
|
SHRQ $0x05, DX
|
|
SUBL DX, BX
|
|
LEAQ (CX)(BX*1), BX
|
|
MOVQ BX, (SP)
|
|
MOVL $0x00000001, DX
|
|
MOVL DX, 16(SP)
|
|
MOVQ src_base+24(FP), BX
|
|
|
|
search_loop_encodeSnappyBlockAsm:
|
|
MOVL DX, SI
|
|
SUBL 12(SP), SI
|
|
SHRL $0x06, SI
|
|
LEAL 4(DX)(SI*1), SI
|
|
CMPL SI, 8(SP)
|
|
JAE emit_remainder_encodeSnappyBlockAsm
|
|
MOVQ (BX)(DX*1), DI
|
|
MOVL SI, 20(SP)
|
|
MOVQ $0x0000cf1bbcdcbf9b, R9
|
|
MOVQ DI, R10
|
|
MOVQ DI, R11
|
|
SHRQ $0x08, R11
|
|
SHLQ $0x10, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x32, R10
|
|
SHLQ $0x10, R11
|
|
IMULQ R9, R11
|
|
SHRQ $0x32, R11
|
|
MOVL (AX)(R10*4), SI
|
|
MOVL (AX)(R11*4), R8
|
|
MOVL DX, (AX)(R10*4)
|
|
LEAL 1(DX), R10
|
|
MOVL R10, (AX)(R11*4)
|
|
MOVQ DI, R10
|
|
SHRQ $0x10, R10
|
|
SHLQ $0x10, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x32, R10
|
|
MOVL DX, R9
|
|
SUBL 16(SP), R9
|
|
MOVL 1(BX)(R9*1), R11
|
|
MOVQ DI, R9
|
|
SHRQ $0x08, R9
|
|
CMPL R9, R11
|
|
JNE no_repeat_found_encodeSnappyBlockAsm
|
|
LEAL 1(DX), DI
|
|
MOVL 12(SP), SI
|
|
MOVL DI, R8
|
|
SUBL 16(SP), R8
|
|
JZ repeat_extend_back_end_encodeSnappyBlockAsm
|
|
|
|
repeat_extend_back_loop_encodeSnappyBlockAsm:
|
|
CMPL DI, SI
|
|
JBE repeat_extend_back_end_encodeSnappyBlockAsm
|
|
MOVB -1(BX)(R8*1), R9
|
|
MOVB -1(BX)(DI*1), R10
|
|
CMPB R9, R10
|
|
JNE repeat_extend_back_end_encodeSnappyBlockAsm
|
|
LEAL -1(DI), DI
|
|
DECL R8
|
|
JNZ repeat_extend_back_loop_encodeSnappyBlockAsm
|
|
|
|
repeat_extend_back_end_encodeSnappyBlockAsm:
|
|
MOVL DI, SI
|
|
SUBL 12(SP), SI
|
|
LEAQ 5(CX)(SI*1), SI
|
|
CMPQ SI, (SP)
|
|
JB repeat_dst_size_check_encodeSnappyBlockAsm
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
repeat_dst_size_check_encodeSnappyBlockAsm:
|
|
MOVL 12(SP), SI
|
|
CMPL SI, DI
|
|
JEQ emit_literal_done_repeat_emit_encodeSnappyBlockAsm
|
|
MOVL DI, R8
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(SI*1), R9
|
|
SUBL SI, R8
|
|
LEAL -1(R8), SI
|
|
CMPL SI, $0x3c
|
|
JB one_byte_repeat_emit_encodeSnappyBlockAsm
|
|
CMPL SI, $0x00000100
|
|
JB two_bytes_repeat_emit_encodeSnappyBlockAsm
|
|
CMPL SI, $0x00010000
|
|
JB three_bytes_repeat_emit_encodeSnappyBlockAsm
|
|
CMPL SI, $0x01000000
|
|
JB four_bytes_repeat_emit_encodeSnappyBlockAsm
|
|
MOVB $0xfc, (CX)
|
|
MOVL SI, 1(CX)
|
|
ADDQ $0x05, CX
|
|
JMP memmove_long_repeat_emit_encodeSnappyBlockAsm
|
|
|
|
four_bytes_repeat_emit_encodeSnappyBlockAsm:
|
|
MOVL SI, R10
|
|
SHRL $0x10, R10
|
|
MOVB $0xf8, (CX)
|
|
MOVW SI, 1(CX)
|
|
MOVB R10, 3(CX)
|
|
ADDQ $0x04, CX
|
|
JMP memmove_long_repeat_emit_encodeSnappyBlockAsm
|
|
|
|
three_bytes_repeat_emit_encodeSnappyBlockAsm:
|
|
MOVB $0xf4, (CX)
|
|
MOVW SI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_repeat_emit_encodeSnappyBlockAsm
|
|
|
|
two_bytes_repeat_emit_encodeSnappyBlockAsm:
|
|
MOVB $0xf0, (CX)
|
|
MOVB SI, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL SI, $0x40
|
|
JB memmove_repeat_emit_encodeSnappyBlockAsm
|
|
JMP memmove_long_repeat_emit_encodeSnappyBlockAsm
|
|
|
|
one_byte_repeat_emit_encodeSnappyBlockAsm:
|
|
SHLB $0x02, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_repeat_emit_encodeSnappyBlockAsm:
|
|
LEAQ (CX)(R8*1), SI
|
|
|
|
// genMemMoveShort
|
|
CMPQ R8, $0x08
|
|
JBE emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_8
|
|
CMPQ R8, $0x10
|
|
JBE emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_8through16
|
|
CMPQ R8, $0x20
|
|
JBE emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_17through32
|
|
JMP emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_33through64
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_8:
|
|
MOVQ (R9), R10
|
|
MOVQ R10, (CX)
|
|
JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_8through16:
|
|
MOVQ (R9), R10
|
|
MOVQ -8(R9)(R8*1), R9
|
|
MOVQ R10, (CX)
|
|
MOVQ R9, -8(CX)(R8*1)
|
|
JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_17through32:
|
|
MOVOU (R9), X0
|
|
MOVOU -16(R9)(R8*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(R8*1)
|
|
JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_33through64:
|
|
MOVOU (R9), X0
|
|
MOVOU 16(R9), X1
|
|
MOVOU -32(R9)(R8*1), X2
|
|
MOVOU -16(R9)(R8*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R8*1)
|
|
MOVOU X3, -16(CX)(R8*1)
|
|
|
|
memmove_end_copy_repeat_emit_encodeSnappyBlockAsm:
|
|
MOVQ SI, CX
|
|
JMP emit_literal_done_repeat_emit_encodeSnappyBlockAsm
|
|
|
|
memmove_long_repeat_emit_encodeSnappyBlockAsm:
|
|
LEAQ (CX)(R8*1), SI
|
|
|
|
// genMemMoveLong
|
|
MOVOU (R9), X0
|
|
MOVOU 16(R9), X1
|
|
MOVOU -32(R9)(R8*1), X2
|
|
MOVOU -16(R9)(R8*1), X3
|
|
MOVQ R8, R11
|
|
SHRQ $0x05, R11
|
|
MOVQ CX, R10
|
|
ANDL $0x0000001f, R10
|
|
MOVQ $0x00000040, R12
|
|
SUBQ R10, R12
|
|
DECQ R11
|
|
JA emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsmlarge_forward_sse_loop_32
|
|
LEAQ -32(R9)(R12*1), R10
|
|
LEAQ -32(CX)(R12*1), R13
|
|
|
|
emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsmlarge_big_loop_back:
|
|
MOVOU (R10), X4
|
|
MOVOU 16(R10), X5
|
|
MOVOA X4, (R13)
|
|
MOVOA X5, 16(R13)
|
|
ADDQ $0x20, R13
|
|
ADDQ $0x20, R10
|
|
ADDQ $0x20, R12
|
|
DECQ R11
|
|
JNA emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsmlarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsmlarge_forward_sse_loop_32:
|
|
MOVOU -32(R9)(R12*1), X4
|
|
MOVOU -16(R9)(R12*1), X5
|
|
MOVOA X4, -32(CX)(R12*1)
|
|
MOVOA X5, -16(CX)(R12*1)
|
|
ADDQ $0x20, R12
|
|
CMPQ R8, R12
|
|
JAE emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsmlarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R8*1)
|
|
MOVOU X3, -16(CX)(R8*1)
|
|
MOVQ SI, CX
|
|
|
|
emit_literal_done_repeat_emit_encodeSnappyBlockAsm:
|
|
ADDL $0x05, DX
|
|
MOVL DX, SI
|
|
SUBL 16(SP), SI
|
|
MOVQ src_len+32(FP), R8
|
|
SUBL DX, R8
|
|
LEAQ (BX)(DX*1), R9
|
|
LEAQ (BX)(SI*1), SI
|
|
|
|
// matchLen
|
|
XORL R11, R11
|
|
|
|
matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm:
|
|
CMPL R8, $0x10
|
|
JB matchlen_match8_repeat_extend_encodeSnappyBlockAsm
|
|
MOVQ (R9)(R11*1), R10
|
|
MOVQ 8(R9)(R11*1), R12
|
|
XORQ (SI)(R11*1), R10
|
|
JNZ matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm
|
|
XORQ 8(SI)(R11*1), R12
|
|
JNZ matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm
|
|
LEAL -16(R8), R8
|
|
LEAL 16(R11), R11
|
|
JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm
|
|
|
|
matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm:
|
|
TZCNTQ R12, R12
|
|
SARQ $0x03, R12
|
|
LEAL 8(R11)(R12*1), R11
|
|
JMP repeat_extend_forward_end_encodeSnappyBlockAsm
|
|
|
|
matchlen_match8_repeat_extend_encodeSnappyBlockAsm:
|
|
CMPL R8, $0x08
|
|
JB matchlen_match4_repeat_extend_encodeSnappyBlockAsm
|
|
MOVQ (R9)(R11*1), R10
|
|
XORQ (SI)(R11*1), R10
|
|
JNZ matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm
|
|
LEAL -8(R8), R8
|
|
LEAL 8(R11), R11
|
|
JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm
|
|
|
|
matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm:
|
|
TZCNTQ R10, R10
|
|
SARQ $0x03, R10
|
|
LEAL (R11)(R10*1), R11
|
|
JMP repeat_extend_forward_end_encodeSnappyBlockAsm
|
|
|
|
matchlen_match4_repeat_extend_encodeSnappyBlockAsm:
|
|
CMPL R8, $0x04
|
|
JB matchlen_match2_repeat_extend_encodeSnappyBlockAsm
|
|
MOVL (R9)(R11*1), R10
|
|
CMPL (SI)(R11*1), R10
|
|
JNE matchlen_match2_repeat_extend_encodeSnappyBlockAsm
|
|
LEAL -4(R8), R8
|
|
LEAL 4(R11), R11
|
|
|
|
matchlen_match2_repeat_extend_encodeSnappyBlockAsm:
|
|
CMPL R8, $0x01
|
|
JE matchlen_match1_repeat_extend_encodeSnappyBlockAsm
|
|
JB repeat_extend_forward_end_encodeSnappyBlockAsm
|
|
MOVW (R9)(R11*1), R10
|
|
CMPW (SI)(R11*1), R10
|
|
JNE matchlen_match1_repeat_extend_encodeSnappyBlockAsm
|
|
LEAL 2(R11), R11
|
|
SUBL $0x02, R8
|
|
JZ repeat_extend_forward_end_encodeSnappyBlockAsm
|
|
|
|
matchlen_match1_repeat_extend_encodeSnappyBlockAsm:
|
|
MOVB (R9)(R11*1), R10
|
|
CMPB (SI)(R11*1), R10
|
|
JNE repeat_extend_forward_end_encodeSnappyBlockAsm
|
|
LEAL 1(R11), R11
|
|
|
|
repeat_extend_forward_end_encodeSnappyBlockAsm:
|
|
ADDL R11, DX
|
|
MOVL DX, SI
|
|
SUBL DI, SI
|
|
MOVL 16(SP), DI
|
|
|
|
// emitCopy
|
|
CMPL DI, $0x00010000
|
|
JB two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm
|
|
|
|
four_bytes_loop_back_repeat_as_copy_encodeSnappyBlockAsm:
|
|
CMPL SI, $0x40
|
|
JBE four_bytes_remain_repeat_as_copy_encodeSnappyBlockAsm
|
|
MOVB $0xff, (CX)
|
|
MOVL DI, 1(CX)
|
|
LEAL -64(SI), SI
|
|
ADDQ $0x05, CX
|
|
CMPL SI, $0x04
|
|
JB four_bytes_remain_repeat_as_copy_encodeSnappyBlockAsm
|
|
JMP four_bytes_loop_back_repeat_as_copy_encodeSnappyBlockAsm
|
|
|
|
four_bytes_remain_repeat_as_copy_encodeSnappyBlockAsm:
|
|
TESTL SI, SI
|
|
JZ repeat_end_emit_encodeSnappyBlockAsm
|
|
XORL R8, R8
|
|
LEAL -1(R8)(SI*4), SI
|
|
MOVB SI, (CX)
|
|
MOVL DI, 1(CX)
|
|
ADDQ $0x05, CX
|
|
JMP repeat_end_emit_encodeSnappyBlockAsm
|
|
|
|
two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm:
|
|
CMPL SI, $0x40
|
|
JBE two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm
|
|
MOVB $0xee, (CX)
|
|
MOVW DI, 1(CX)
|
|
LEAL -60(SI), SI
|
|
ADDQ $0x03, CX
|
|
JMP two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm
|
|
|
|
two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm:
|
|
MOVL SI, R8
|
|
SHLL $0x02, R8
|
|
CMPL SI, $0x0c
|
|
JAE emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm
|
|
CMPL DI, $0x00000800
|
|
JAE emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm
|
|
LEAL -15(R8), R8
|
|
MOVB DI, 1(CX)
|
|
SHRL $0x08, DI
|
|
SHLL $0x05, DI
|
|
ORL DI, R8
|
|
MOVB R8, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeSnappyBlockAsm
|
|
|
|
emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm:
|
|
LEAL -2(R8), R8
|
|
MOVB R8, (CX)
|
|
MOVW DI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
|
|
repeat_end_emit_encodeSnappyBlockAsm:
|
|
MOVL DX, 12(SP)
|
|
JMP search_loop_encodeSnappyBlockAsm
|
|
|
|
no_repeat_found_encodeSnappyBlockAsm:
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ candidate_match_encodeSnappyBlockAsm
|
|
SHRQ $0x08, DI
|
|
MOVL (AX)(R10*4), SI
|
|
LEAL 2(DX), R9
|
|
CMPL (BX)(R8*1), DI
|
|
JEQ candidate2_match_encodeSnappyBlockAsm
|
|
MOVL R9, (AX)(R10*4)
|
|
SHRQ $0x08, DI
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ candidate3_match_encodeSnappyBlockAsm
|
|
MOVL 20(SP), DX
|
|
JMP search_loop_encodeSnappyBlockAsm
|
|
|
|
candidate3_match_encodeSnappyBlockAsm:
|
|
ADDL $0x02, DX
|
|
JMP candidate_match_encodeSnappyBlockAsm
|
|
|
|
candidate2_match_encodeSnappyBlockAsm:
|
|
MOVL R9, (AX)(R10*4)
|
|
INCL DX
|
|
MOVL R8, SI
|
|
|
|
candidate_match_encodeSnappyBlockAsm:
|
|
MOVL 12(SP), DI
|
|
TESTL SI, SI
|
|
JZ match_extend_back_end_encodeSnappyBlockAsm
|
|
|
|
match_extend_back_loop_encodeSnappyBlockAsm:
|
|
CMPL DX, DI
|
|
JBE match_extend_back_end_encodeSnappyBlockAsm
|
|
MOVB -1(BX)(SI*1), R8
|
|
MOVB -1(BX)(DX*1), R9
|
|
CMPB R8, R9
|
|
JNE match_extend_back_end_encodeSnappyBlockAsm
|
|
LEAL -1(DX), DX
|
|
DECL SI
|
|
JZ match_extend_back_end_encodeSnappyBlockAsm
|
|
JMP match_extend_back_loop_encodeSnappyBlockAsm
|
|
|
|
match_extend_back_end_encodeSnappyBlockAsm:
|
|
MOVL DX, DI
|
|
SUBL 12(SP), DI
|
|
LEAQ 5(CX)(DI*1), DI
|
|
CMPQ DI, (SP)
|
|
JB match_dst_size_check_encodeSnappyBlockAsm
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeSnappyBlockAsm:
|
|
MOVL DX, DI
|
|
MOVL 12(SP), R8
|
|
CMPL R8, DI
|
|
JEQ emit_literal_done_match_emit_encodeSnappyBlockAsm
|
|
MOVL DI, R9
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(R8*1), DI
|
|
SUBL R8, R9
|
|
LEAL -1(R9), R8
|
|
CMPL R8, $0x3c
|
|
JB one_byte_match_emit_encodeSnappyBlockAsm
|
|
CMPL R8, $0x00000100
|
|
JB two_bytes_match_emit_encodeSnappyBlockAsm
|
|
CMPL R8, $0x00010000
|
|
JB three_bytes_match_emit_encodeSnappyBlockAsm
|
|
CMPL R8, $0x01000000
|
|
JB four_bytes_match_emit_encodeSnappyBlockAsm
|
|
MOVB $0xfc, (CX)
|
|
MOVL R8, 1(CX)
|
|
ADDQ $0x05, CX
|
|
JMP memmove_long_match_emit_encodeSnappyBlockAsm
|
|
|
|
four_bytes_match_emit_encodeSnappyBlockAsm:
|
|
MOVL R8, R10
|
|
SHRL $0x10, R10
|
|
MOVB $0xf8, (CX)
|
|
MOVW R8, 1(CX)
|
|
MOVB R10, 3(CX)
|
|
ADDQ $0x04, CX
|
|
JMP memmove_long_match_emit_encodeSnappyBlockAsm
|
|
|
|
three_bytes_match_emit_encodeSnappyBlockAsm:
|
|
MOVB $0xf4, (CX)
|
|
MOVW R8, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_match_emit_encodeSnappyBlockAsm
|
|
|
|
two_bytes_match_emit_encodeSnappyBlockAsm:
|
|
MOVB $0xf0, (CX)
|
|
MOVB R8, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL R8, $0x40
|
|
JB memmove_match_emit_encodeSnappyBlockAsm
|
|
JMP memmove_long_match_emit_encodeSnappyBlockAsm
|
|
|
|
one_byte_match_emit_encodeSnappyBlockAsm:
|
|
SHLB $0x02, R8
|
|
MOVB R8, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_match_emit_encodeSnappyBlockAsm:
|
|
LEAQ (CX)(R9*1), R8
|
|
|
|
// genMemMoveShort
|
|
CMPQ R9, $0x08
|
|
JBE emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_8
|
|
CMPQ R9, $0x10
|
|
JBE emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_8through16
|
|
CMPQ R9, $0x20
|
|
JBE emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_8:
|
|
MOVQ (DI), R10
|
|
MOVQ R10, (CX)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_8through16:
|
|
MOVQ (DI), R10
|
|
MOVQ -8(DI)(R9*1), DI
|
|
MOVQ R10, (CX)
|
|
MOVQ DI, -8(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_17through32:
|
|
MOVOU (DI), X0
|
|
MOVOU -16(DI)(R9*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_33through64:
|
|
MOVOU (DI), X0
|
|
MOVOU 16(DI), X1
|
|
MOVOU -32(DI)(R9*1), X2
|
|
MOVOU -16(DI)(R9*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
|
|
memmove_end_copy_match_emit_encodeSnappyBlockAsm:
|
|
MOVQ R8, CX
|
|
JMP emit_literal_done_match_emit_encodeSnappyBlockAsm
|
|
|
|
memmove_long_match_emit_encodeSnappyBlockAsm:
|
|
LEAQ (CX)(R9*1), R8
|
|
|
|
// genMemMoveLong
|
|
MOVOU (DI), X0
|
|
MOVOU 16(DI), X1
|
|
MOVOU -32(DI)(R9*1), X2
|
|
MOVOU -16(DI)(R9*1), X3
|
|
MOVQ R9, R11
|
|
SHRQ $0x05, R11
|
|
MOVQ CX, R10
|
|
ANDL $0x0000001f, R10
|
|
MOVQ $0x00000040, R12
|
|
SUBQ R10, R12
|
|
DECQ R11
|
|
JA emit_lit_memmove_long_match_emit_encodeSnappyBlockAsmlarge_forward_sse_loop_32
|
|
LEAQ -32(DI)(R12*1), R10
|
|
LEAQ -32(CX)(R12*1), R13
|
|
|
|
emit_lit_memmove_long_match_emit_encodeSnappyBlockAsmlarge_big_loop_back:
|
|
MOVOU (R10), X4
|
|
MOVOU 16(R10), X5
|
|
MOVOA X4, (R13)
|
|
MOVOA X5, 16(R13)
|
|
ADDQ $0x20, R13
|
|
ADDQ $0x20, R10
|
|
ADDQ $0x20, R12
|
|
DECQ R11
|
|
JNA emit_lit_memmove_long_match_emit_encodeSnappyBlockAsmlarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_match_emit_encodeSnappyBlockAsmlarge_forward_sse_loop_32:
|
|
MOVOU -32(DI)(R12*1), X4
|
|
MOVOU -16(DI)(R12*1), X5
|
|
MOVOA X4, -32(CX)(R12*1)
|
|
MOVOA X5, -16(CX)(R12*1)
|
|
ADDQ $0x20, R12
|
|
CMPQ R9, R12
|
|
JAE emit_lit_memmove_long_match_emit_encodeSnappyBlockAsmlarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
MOVQ R8, CX
|
|
|
|
emit_literal_done_match_emit_encodeSnappyBlockAsm:
|
|
match_nolit_loop_encodeSnappyBlockAsm:
|
|
MOVL DX, DI
|
|
SUBL SI, DI
|
|
MOVL DI, 16(SP)
|
|
ADDL $0x04, DX
|
|
ADDL $0x04, SI
|
|
MOVQ src_len+32(FP), DI
|
|
SUBL DX, DI
|
|
LEAQ (BX)(DX*1), R8
|
|
LEAQ (BX)(SI*1), SI
|
|
|
|
// matchLen
|
|
XORL R10, R10
|
|
|
|
matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm:
|
|
CMPL DI, $0x10
|
|
JB matchlen_match8_match_nolit_encodeSnappyBlockAsm
|
|
MOVQ (R8)(R10*1), R9
|
|
MOVQ 8(R8)(R10*1), R11
|
|
XORQ (SI)(R10*1), R9
|
|
JNZ matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm
|
|
XORQ 8(SI)(R10*1), R11
|
|
JNZ matchlen_bsf_16match_nolit_encodeSnappyBlockAsm
|
|
LEAL -16(DI), DI
|
|
LEAL 16(R10), R10
|
|
JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm
|
|
|
|
matchlen_bsf_16match_nolit_encodeSnappyBlockAsm:
|
|
TZCNTQ R11, R11
|
|
SARQ $0x03, R11
|
|
LEAL 8(R10)(R11*1), R10
|
|
JMP match_nolit_end_encodeSnappyBlockAsm
|
|
|
|
matchlen_match8_match_nolit_encodeSnappyBlockAsm:
|
|
CMPL DI, $0x08
|
|
JB matchlen_match4_match_nolit_encodeSnappyBlockAsm
|
|
MOVQ (R8)(R10*1), R9
|
|
XORQ (SI)(R10*1), R9
|
|
JNZ matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm
|
|
LEAL -8(DI), DI
|
|
LEAL 8(R10), R10
|
|
JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm
|
|
|
|
matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm:
|
|
TZCNTQ R9, R9
|
|
SARQ $0x03, R9
|
|
LEAL (R10)(R9*1), R10
|
|
JMP match_nolit_end_encodeSnappyBlockAsm
|
|
|
|
matchlen_match4_match_nolit_encodeSnappyBlockAsm:
|
|
CMPL DI, $0x04
|
|
JB matchlen_match2_match_nolit_encodeSnappyBlockAsm
|
|
MOVL (R8)(R10*1), R9
|
|
CMPL (SI)(R10*1), R9
|
|
JNE matchlen_match2_match_nolit_encodeSnappyBlockAsm
|
|
LEAL -4(DI), DI
|
|
LEAL 4(R10), R10
|
|
|
|
matchlen_match2_match_nolit_encodeSnappyBlockAsm:
|
|
CMPL DI, $0x01
|
|
JE matchlen_match1_match_nolit_encodeSnappyBlockAsm
|
|
JB match_nolit_end_encodeSnappyBlockAsm
|
|
MOVW (R8)(R10*1), R9
|
|
CMPW (SI)(R10*1), R9
|
|
JNE matchlen_match1_match_nolit_encodeSnappyBlockAsm
|
|
LEAL 2(R10), R10
|
|
SUBL $0x02, DI
|
|
JZ match_nolit_end_encodeSnappyBlockAsm
|
|
|
|
matchlen_match1_match_nolit_encodeSnappyBlockAsm:
|
|
MOVB (R8)(R10*1), R9
|
|
CMPB (SI)(R10*1), R9
|
|
JNE match_nolit_end_encodeSnappyBlockAsm
|
|
LEAL 1(R10), R10
|
|
|
|
match_nolit_end_encodeSnappyBlockAsm:
|
|
ADDL R10, DX
|
|
MOVL 16(SP), SI
|
|
ADDL $0x04, R10
|
|
MOVL DX, 12(SP)
|
|
|
|
// emitCopy
|
|
CMPL SI, $0x00010000
|
|
JB two_byte_offset_match_nolit_encodeSnappyBlockAsm
|
|
|
|
four_bytes_loop_back_match_nolit_encodeSnappyBlockAsm:
|
|
CMPL R10, $0x40
|
|
JBE four_bytes_remain_match_nolit_encodeSnappyBlockAsm
|
|
MOVB $0xff, (CX)
|
|
MOVL SI, 1(CX)
|
|
LEAL -64(R10), R10
|
|
ADDQ $0x05, CX
|
|
CMPL R10, $0x04
|
|
JB four_bytes_remain_match_nolit_encodeSnappyBlockAsm
|
|
JMP four_bytes_loop_back_match_nolit_encodeSnappyBlockAsm
|
|
|
|
four_bytes_remain_match_nolit_encodeSnappyBlockAsm:
|
|
TESTL R10, R10
|
|
JZ match_nolit_emitcopy_end_encodeSnappyBlockAsm
|
|
XORL DI, DI
|
|
LEAL -1(DI)(R10*4), R10
|
|
MOVB R10, (CX)
|
|
MOVL SI, 1(CX)
|
|
ADDQ $0x05, CX
|
|
JMP match_nolit_emitcopy_end_encodeSnappyBlockAsm
|
|
|
|
two_byte_offset_match_nolit_encodeSnappyBlockAsm:
|
|
CMPL R10, $0x40
|
|
JBE two_byte_offset_short_match_nolit_encodeSnappyBlockAsm
|
|
MOVB $0xee, (CX)
|
|
MOVW SI, 1(CX)
|
|
LEAL -60(R10), R10
|
|
ADDQ $0x03, CX
|
|
JMP two_byte_offset_match_nolit_encodeSnappyBlockAsm
|
|
|
|
two_byte_offset_short_match_nolit_encodeSnappyBlockAsm:
|
|
MOVL R10, DI
|
|
SHLL $0x02, DI
|
|
CMPL R10, $0x0c
|
|
JAE emit_copy_three_match_nolit_encodeSnappyBlockAsm
|
|
CMPL SI, $0x00000800
|
|
JAE emit_copy_three_match_nolit_encodeSnappyBlockAsm
|
|
LEAL -15(DI), DI
|
|
MOVB SI, 1(CX)
|
|
SHRL $0x08, SI
|
|
SHLL $0x05, SI
|
|
ORL SI, DI
|
|
MOVB DI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeSnappyBlockAsm
|
|
|
|
emit_copy_three_match_nolit_encodeSnappyBlockAsm:
|
|
LEAL -2(DI), DI
|
|
MOVB DI, (CX)
|
|
MOVW SI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
|
|
match_nolit_emitcopy_end_encodeSnappyBlockAsm:
|
|
CMPL DX, 8(SP)
|
|
JAE emit_remainder_encodeSnappyBlockAsm
|
|
MOVQ -2(BX)(DX*1), DI
|
|
CMPQ CX, (SP)
|
|
JB match_nolit_dst_ok_encodeSnappyBlockAsm
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeSnappyBlockAsm:
|
|
MOVQ $0x0000cf1bbcdcbf9b, R9
|
|
MOVQ DI, R8
|
|
SHRQ $0x10, DI
|
|
MOVQ DI, SI
|
|
SHLQ $0x10, R8
|
|
IMULQ R9, R8
|
|
SHRQ $0x32, R8
|
|
SHLQ $0x10, SI
|
|
IMULQ R9, SI
|
|
SHRQ $0x32, SI
|
|
LEAL -2(DX), R9
|
|
LEAQ (AX)(SI*4), R10
|
|
MOVL (R10), SI
|
|
MOVL R9, (AX)(R8*4)
|
|
MOVL DX, (R10)
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ match_nolit_loop_encodeSnappyBlockAsm
|
|
INCL DX
|
|
JMP search_loop_encodeSnappyBlockAsm
|
|
|
|
emit_remainder_encodeSnappyBlockAsm:
|
|
MOVQ src_len+32(FP), AX
|
|
SUBL 12(SP), AX
|
|
LEAQ 5(CX)(AX*1), AX
|
|
CMPQ AX, (SP)
|
|
JB emit_remainder_ok_encodeSnappyBlockAsm
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeSnappyBlockAsm:
|
|
MOVQ src_len+32(FP), AX
|
|
MOVL 12(SP), DX
|
|
CMPL DX, AX
|
|
JEQ emit_literal_done_emit_remainder_encodeSnappyBlockAsm
|
|
MOVL AX, SI
|
|
MOVL AX, 12(SP)
|
|
LEAQ (BX)(DX*1), AX
|
|
SUBL DX, SI
|
|
LEAL -1(SI), DX
|
|
CMPL DX, $0x3c
|
|
JB one_byte_emit_remainder_encodeSnappyBlockAsm
|
|
CMPL DX, $0x00000100
|
|
JB two_bytes_emit_remainder_encodeSnappyBlockAsm
|
|
CMPL DX, $0x00010000
|
|
JB three_bytes_emit_remainder_encodeSnappyBlockAsm
|
|
CMPL DX, $0x01000000
|
|
JB four_bytes_emit_remainder_encodeSnappyBlockAsm
|
|
MOVB $0xfc, (CX)
|
|
MOVL DX, 1(CX)
|
|
ADDQ $0x05, CX
|
|
JMP memmove_long_emit_remainder_encodeSnappyBlockAsm
|
|
|
|
four_bytes_emit_remainder_encodeSnappyBlockAsm:
|
|
MOVL DX, BX
|
|
SHRL $0x10, BX
|
|
MOVB $0xf8, (CX)
|
|
MOVW DX, 1(CX)
|
|
MOVB BL, 3(CX)
|
|
ADDQ $0x04, CX
|
|
JMP memmove_long_emit_remainder_encodeSnappyBlockAsm
|
|
|
|
three_bytes_emit_remainder_encodeSnappyBlockAsm:
|
|
MOVB $0xf4, (CX)
|
|
MOVW DX, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_emit_remainder_encodeSnappyBlockAsm
|
|
|
|
two_bytes_emit_remainder_encodeSnappyBlockAsm:
|
|
MOVB $0xf0, (CX)
|
|
MOVB DL, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL DX, $0x40
|
|
JB memmove_emit_remainder_encodeSnappyBlockAsm
|
|
JMP memmove_long_emit_remainder_encodeSnappyBlockAsm
|
|
|
|
one_byte_emit_remainder_encodeSnappyBlockAsm:
|
|
SHLB $0x02, DL
|
|
MOVB DL, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_emit_remainder_encodeSnappyBlockAsm:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveShort
|
|
CMPQ BX, $0x03
|
|
JB emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_1or2
|
|
JE emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_3
|
|
CMPQ BX, $0x08
|
|
JB emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_4through7
|
|
CMPQ BX, $0x10
|
|
JBE emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_8through16
|
|
CMPQ BX, $0x20
|
|
JBE emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_1or2:
|
|
MOVB (AX), SI
|
|
MOVB -1(AX)(BX*1), AL
|
|
MOVB SI, (CX)
|
|
MOVB AL, -1(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_3:
|
|
MOVW (AX), SI
|
|
MOVB 2(AX), AL
|
|
MOVW SI, (CX)
|
|
MOVB AL, 2(CX)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_4through7:
|
|
MOVL (AX), SI
|
|
MOVL -4(AX)(BX*1), AX
|
|
MOVL SI, (CX)
|
|
MOVL AX, -4(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_8through16:
|
|
MOVQ (AX), SI
|
|
MOVQ -8(AX)(BX*1), AX
|
|
MOVQ SI, (CX)
|
|
MOVQ AX, -8(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_17through32:
|
|
MOVOU (AX), X0
|
|
MOVOU -16(AX)(BX*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_33through64:
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
|
|
memmove_end_copy_emit_remainder_encodeSnappyBlockAsm:
|
|
MOVQ DX, CX
|
|
JMP emit_literal_done_emit_remainder_encodeSnappyBlockAsm
|
|
|
|
memmove_long_emit_remainder_encodeSnappyBlockAsm:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveLong
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVQ BX, DI
|
|
SHRQ $0x05, DI
|
|
MOVQ CX, SI
|
|
ANDL $0x0000001f, SI
|
|
MOVQ $0x00000040, R8
|
|
SUBQ SI, R8
|
|
DECQ DI
|
|
JA emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsmlarge_forward_sse_loop_32
|
|
LEAQ -32(AX)(R8*1), SI
|
|
LEAQ -32(CX)(R8*1), R9
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsmlarge_big_loop_back:
|
|
MOVOU (SI), X4
|
|
MOVOU 16(SI), X5
|
|
MOVOA X4, (R9)
|
|
MOVOA X5, 16(R9)
|
|
ADDQ $0x20, R9
|
|
ADDQ $0x20, SI
|
|
ADDQ $0x20, R8
|
|
DECQ DI
|
|
JNA emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsmlarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsmlarge_forward_sse_loop_32:
|
|
MOVOU -32(AX)(R8*1), X4
|
|
MOVOU -16(AX)(R8*1), X5
|
|
MOVOA X4, -32(CX)(R8*1)
|
|
MOVOA X5, -16(CX)(R8*1)
|
|
ADDQ $0x20, R8
|
|
CMPQ BX, R8
|
|
JAE emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsmlarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
MOVQ DX, CX
|
|
|
|
emit_literal_done_emit_remainder_encodeSnappyBlockAsm:
|
|
MOVQ dst_base+0(FP), AX
|
|
SUBQ AX, CX
|
|
MOVQ CX, ret+56(FP)
|
|
RET
|
|
|
|
// func encodeSnappyBlockAsm64K(dst []byte, src []byte, tmp *[65536]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeSnappyBlockAsm64K(SB), $24-64
|
|
MOVQ tmp+48(FP), AX
|
|
MOVQ dst_base+0(FP), CX
|
|
MOVQ $0x00000200, DX
|
|
MOVQ AX, BX
|
|
PXOR X0, X0
|
|
|
|
zero_loop_encodeSnappyBlockAsm64K:
|
|
MOVOU X0, (BX)
|
|
MOVOU X0, 16(BX)
|
|
MOVOU X0, 32(BX)
|
|
MOVOU X0, 48(BX)
|
|
MOVOU X0, 64(BX)
|
|
MOVOU X0, 80(BX)
|
|
MOVOU X0, 96(BX)
|
|
MOVOU X0, 112(BX)
|
|
ADDQ $0x80, BX
|
|
DECQ DX
|
|
JNZ zero_loop_encodeSnappyBlockAsm64K
|
|
MOVL $0x00000000, 12(SP)
|
|
MOVQ src_len+32(FP), DX
|
|
LEAQ -9(DX), BX
|
|
LEAQ -8(DX), SI
|
|
MOVL SI, 8(SP)
|
|
SHRQ $0x05, DX
|
|
SUBL DX, BX
|
|
LEAQ (CX)(BX*1), BX
|
|
MOVQ BX, (SP)
|
|
MOVL $0x00000001, DX
|
|
MOVL DX, 16(SP)
|
|
MOVQ src_base+24(FP), BX
|
|
|
|
search_loop_encodeSnappyBlockAsm64K:
|
|
MOVL DX, SI
|
|
SUBL 12(SP), SI
|
|
SHRL $0x06, SI
|
|
LEAL 4(DX)(SI*1), SI
|
|
CMPL SI, 8(SP)
|
|
JAE emit_remainder_encodeSnappyBlockAsm64K
|
|
MOVQ (BX)(DX*1), DI
|
|
MOVL SI, 20(SP)
|
|
MOVQ $0x0000cf1bbcdcbf9b, R9
|
|
MOVQ DI, R10
|
|
MOVQ DI, R11
|
|
SHRQ $0x08, R11
|
|
SHLQ $0x10, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x32, R10
|
|
SHLQ $0x10, R11
|
|
IMULQ R9, R11
|
|
SHRQ $0x32, R11
|
|
MOVL (AX)(R10*4), SI
|
|
MOVL (AX)(R11*4), R8
|
|
MOVL DX, (AX)(R10*4)
|
|
LEAL 1(DX), R10
|
|
MOVL R10, (AX)(R11*4)
|
|
MOVQ DI, R10
|
|
SHRQ $0x10, R10
|
|
SHLQ $0x10, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x32, R10
|
|
MOVL DX, R9
|
|
SUBL 16(SP), R9
|
|
MOVL 1(BX)(R9*1), R11
|
|
MOVQ DI, R9
|
|
SHRQ $0x08, R9
|
|
CMPL R9, R11
|
|
JNE no_repeat_found_encodeSnappyBlockAsm64K
|
|
LEAL 1(DX), DI
|
|
MOVL 12(SP), SI
|
|
MOVL DI, R8
|
|
SUBL 16(SP), R8
|
|
JZ repeat_extend_back_end_encodeSnappyBlockAsm64K
|
|
|
|
repeat_extend_back_loop_encodeSnappyBlockAsm64K:
|
|
CMPL DI, SI
|
|
JBE repeat_extend_back_end_encodeSnappyBlockAsm64K
|
|
MOVB -1(BX)(R8*1), R9
|
|
MOVB -1(BX)(DI*1), R10
|
|
CMPB R9, R10
|
|
JNE repeat_extend_back_end_encodeSnappyBlockAsm64K
|
|
LEAL -1(DI), DI
|
|
DECL R8
|
|
JNZ repeat_extend_back_loop_encodeSnappyBlockAsm64K
|
|
|
|
repeat_extend_back_end_encodeSnappyBlockAsm64K:
|
|
MOVL DI, SI
|
|
SUBL 12(SP), SI
|
|
LEAQ 3(CX)(SI*1), SI
|
|
CMPQ SI, (SP)
|
|
JB repeat_dst_size_check_encodeSnappyBlockAsm64K
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
repeat_dst_size_check_encodeSnappyBlockAsm64K:
|
|
MOVL 12(SP), SI
|
|
CMPL SI, DI
|
|
JEQ emit_literal_done_repeat_emit_encodeSnappyBlockAsm64K
|
|
MOVL DI, R8
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(SI*1), R9
|
|
SUBL SI, R8
|
|
LEAL -1(R8), SI
|
|
CMPL SI, $0x3c
|
|
JB one_byte_repeat_emit_encodeSnappyBlockAsm64K
|
|
CMPL SI, $0x00000100
|
|
JB two_bytes_repeat_emit_encodeSnappyBlockAsm64K
|
|
JB three_bytes_repeat_emit_encodeSnappyBlockAsm64K
|
|
|
|
three_bytes_repeat_emit_encodeSnappyBlockAsm64K:
|
|
MOVB $0xf4, (CX)
|
|
MOVW SI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_repeat_emit_encodeSnappyBlockAsm64K
|
|
|
|
two_bytes_repeat_emit_encodeSnappyBlockAsm64K:
|
|
MOVB $0xf0, (CX)
|
|
MOVB SI, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL SI, $0x40
|
|
JB memmove_repeat_emit_encodeSnappyBlockAsm64K
|
|
JMP memmove_long_repeat_emit_encodeSnappyBlockAsm64K
|
|
|
|
one_byte_repeat_emit_encodeSnappyBlockAsm64K:
|
|
SHLB $0x02, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_repeat_emit_encodeSnappyBlockAsm64K:
|
|
LEAQ (CX)(R8*1), SI
|
|
|
|
// genMemMoveShort
|
|
CMPQ R8, $0x08
|
|
JBE emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_8
|
|
CMPQ R8, $0x10
|
|
JBE emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_8through16
|
|
CMPQ R8, $0x20
|
|
JBE emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_17through32
|
|
JMP emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_33through64
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_8:
|
|
MOVQ (R9), R10
|
|
MOVQ R10, (CX)
|
|
JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm64K
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_8through16:
|
|
MOVQ (R9), R10
|
|
MOVQ -8(R9)(R8*1), R9
|
|
MOVQ R10, (CX)
|
|
MOVQ R9, -8(CX)(R8*1)
|
|
JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm64K
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_17through32:
|
|
MOVOU (R9), X0
|
|
MOVOU -16(R9)(R8*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(R8*1)
|
|
JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm64K
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_33through64:
|
|
MOVOU (R9), X0
|
|
MOVOU 16(R9), X1
|
|
MOVOU -32(R9)(R8*1), X2
|
|
MOVOU -16(R9)(R8*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R8*1)
|
|
MOVOU X3, -16(CX)(R8*1)
|
|
|
|
memmove_end_copy_repeat_emit_encodeSnappyBlockAsm64K:
|
|
MOVQ SI, CX
|
|
JMP emit_literal_done_repeat_emit_encodeSnappyBlockAsm64K
|
|
|
|
memmove_long_repeat_emit_encodeSnappyBlockAsm64K:
|
|
LEAQ (CX)(R8*1), SI
|
|
|
|
// genMemMoveLong
|
|
MOVOU (R9), X0
|
|
MOVOU 16(R9), X1
|
|
MOVOU -32(R9)(R8*1), X2
|
|
MOVOU -16(R9)(R8*1), X3
|
|
MOVQ R8, R11
|
|
SHRQ $0x05, R11
|
|
MOVQ CX, R10
|
|
ANDL $0x0000001f, R10
|
|
MOVQ $0x00000040, R12
|
|
SUBQ R10, R12
|
|
DECQ R11
|
|
JA emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm64Klarge_forward_sse_loop_32
|
|
LEAQ -32(R9)(R12*1), R10
|
|
LEAQ -32(CX)(R12*1), R13
|
|
|
|
emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm64Klarge_big_loop_back:
|
|
MOVOU (R10), X4
|
|
MOVOU 16(R10), X5
|
|
MOVOA X4, (R13)
|
|
MOVOA X5, 16(R13)
|
|
ADDQ $0x20, R13
|
|
ADDQ $0x20, R10
|
|
ADDQ $0x20, R12
|
|
DECQ R11
|
|
JNA emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm64Klarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm64Klarge_forward_sse_loop_32:
|
|
MOVOU -32(R9)(R12*1), X4
|
|
MOVOU -16(R9)(R12*1), X5
|
|
MOVOA X4, -32(CX)(R12*1)
|
|
MOVOA X5, -16(CX)(R12*1)
|
|
ADDQ $0x20, R12
|
|
CMPQ R8, R12
|
|
JAE emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm64Klarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R8*1)
|
|
MOVOU X3, -16(CX)(R8*1)
|
|
MOVQ SI, CX
|
|
|
|
emit_literal_done_repeat_emit_encodeSnappyBlockAsm64K:
|
|
ADDL $0x05, DX
|
|
MOVL DX, SI
|
|
SUBL 16(SP), SI
|
|
MOVQ src_len+32(FP), R8
|
|
SUBL DX, R8
|
|
LEAQ (BX)(DX*1), R9
|
|
LEAQ (BX)(SI*1), SI
|
|
|
|
// matchLen
|
|
XORL R11, R11
|
|
|
|
matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm64K:
|
|
CMPL R8, $0x10
|
|
JB matchlen_match8_repeat_extend_encodeSnappyBlockAsm64K
|
|
MOVQ (R9)(R11*1), R10
|
|
MOVQ 8(R9)(R11*1), R12
|
|
XORQ (SI)(R11*1), R10
|
|
JNZ matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm64K
|
|
XORQ 8(SI)(R11*1), R12
|
|
JNZ matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm64K
|
|
LEAL -16(R8), R8
|
|
LEAL 16(R11), R11
|
|
JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm64K
|
|
|
|
matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm64K:
|
|
TZCNTQ R12, R12
|
|
SARQ $0x03, R12
|
|
LEAL 8(R11)(R12*1), R11
|
|
JMP repeat_extend_forward_end_encodeSnappyBlockAsm64K
|
|
|
|
matchlen_match8_repeat_extend_encodeSnappyBlockAsm64K:
|
|
CMPL R8, $0x08
|
|
JB matchlen_match4_repeat_extend_encodeSnappyBlockAsm64K
|
|
MOVQ (R9)(R11*1), R10
|
|
XORQ (SI)(R11*1), R10
|
|
JNZ matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm64K
|
|
LEAL -8(R8), R8
|
|
LEAL 8(R11), R11
|
|
JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm64K
|
|
|
|
matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm64K:
|
|
TZCNTQ R10, R10
|
|
SARQ $0x03, R10
|
|
LEAL (R11)(R10*1), R11
|
|
JMP repeat_extend_forward_end_encodeSnappyBlockAsm64K
|
|
|
|
matchlen_match4_repeat_extend_encodeSnappyBlockAsm64K:
|
|
CMPL R8, $0x04
|
|
JB matchlen_match2_repeat_extend_encodeSnappyBlockAsm64K
|
|
MOVL (R9)(R11*1), R10
|
|
CMPL (SI)(R11*1), R10
|
|
JNE matchlen_match2_repeat_extend_encodeSnappyBlockAsm64K
|
|
LEAL -4(R8), R8
|
|
LEAL 4(R11), R11
|
|
|
|
matchlen_match2_repeat_extend_encodeSnappyBlockAsm64K:
|
|
CMPL R8, $0x01
|
|
JE matchlen_match1_repeat_extend_encodeSnappyBlockAsm64K
|
|
JB repeat_extend_forward_end_encodeSnappyBlockAsm64K
|
|
MOVW (R9)(R11*1), R10
|
|
CMPW (SI)(R11*1), R10
|
|
JNE matchlen_match1_repeat_extend_encodeSnappyBlockAsm64K
|
|
LEAL 2(R11), R11
|
|
SUBL $0x02, R8
|
|
JZ repeat_extend_forward_end_encodeSnappyBlockAsm64K
|
|
|
|
matchlen_match1_repeat_extend_encodeSnappyBlockAsm64K:
|
|
MOVB (R9)(R11*1), R10
|
|
CMPB (SI)(R11*1), R10
|
|
JNE repeat_extend_forward_end_encodeSnappyBlockAsm64K
|
|
LEAL 1(R11), R11
|
|
|
|
repeat_extend_forward_end_encodeSnappyBlockAsm64K:
|
|
ADDL R11, DX
|
|
MOVL DX, SI
|
|
SUBL DI, SI
|
|
MOVL 16(SP), DI
|
|
|
|
// emitCopy
|
|
two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm64K:
|
|
CMPL SI, $0x40
|
|
JBE two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm64K
|
|
MOVB $0xee, (CX)
|
|
MOVW DI, 1(CX)
|
|
LEAL -60(SI), SI
|
|
ADDQ $0x03, CX
|
|
JMP two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm64K
|
|
|
|
two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm64K:
|
|
MOVL SI, R8
|
|
SHLL $0x02, R8
|
|
CMPL SI, $0x0c
|
|
JAE emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm64K
|
|
CMPL DI, $0x00000800
|
|
JAE emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm64K
|
|
LEAL -15(R8), R8
|
|
MOVB DI, 1(CX)
|
|
SHRL $0x08, DI
|
|
SHLL $0x05, DI
|
|
ORL DI, R8
|
|
MOVB R8, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeSnappyBlockAsm64K
|
|
|
|
emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm64K:
|
|
LEAL -2(R8), R8
|
|
MOVB R8, (CX)
|
|
MOVW DI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
|
|
repeat_end_emit_encodeSnappyBlockAsm64K:
|
|
MOVL DX, 12(SP)
|
|
JMP search_loop_encodeSnappyBlockAsm64K
|
|
|
|
no_repeat_found_encodeSnappyBlockAsm64K:
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ candidate_match_encodeSnappyBlockAsm64K
|
|
SHRQ $0x08, DI
|
|
MOVL (AX)(R10*4), SI
|
|
LEAL 2(DX), R9
|
|
CMPL (BX)(R8*1), DI
|
|
JEQ candidate2_match_encodeSnappyBlockAsm64K
|
|
MOVL R9, (AX)(R10*4)
|
|
SHRQ $0x08, DI
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ candidate3_match_encodeSnappyBlockAsm64K
|
|
MOVL 20(SP), DX
|
|
JMP search_loop_encodeSnappyBlockAsm64K
|
|
|
|
candidate3_match_encodeSnappyBlockAsm64K:
|
|
ADDL $0x02, DX
|
|
JMP candidate_match_encodeSnappyBlockAsm64K
|
|
|
|
candidate2_match_encodeSnappyBlockAsm64K:
|
|
MOVL R9, (AX)(R10*4)
|
|
INCL DX
|
|
MOVL R8, SI
|
|
|
|
candidate_match_encodeSnappyBlockAsm64K:
|
|
MOVL 12(SP), DI
|
|
TESTL SI, SI
|
|
JZ match_extend_back_end_encodeSnappyBlockAsm64K
|
|
|
|
match_extend_back_loop_encodeSnappyBlockAsm64K:
|
|
CMPL DX, DI
|
|
JBE match_extend_back_end_encodeSnappyBlockAsm64K
|
|
MOVB -1(BX)(SI*1), R8
|
|
MOVB -1(BX)(DX*1), R9
|
|
CMPB R8, R9
|
|
JNE match_extend_back_end_encodeSnappyBlockAsm64K
|
|
LEAL -1(DX), DX
|
|
DECL SI
|
|
JZ match_extend_back_end_encodeSnappyBlockAsm64K
|
|
JMP match_extend_back_loop_encodeSnappyBlockAsm64K
|
|
|
|
match_extend_back_end_encodeSnappyBlockAsm64K:
|
|
MOVL DX, DI
|
|
SUBL 12(SP), DI
|
|
LEAQ 3(CX)(DI*1), DI
|
|
CMPQ DI, (SP)
|
|
JB match_dst_size_check_encodeSnappyBlockAsm64K
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeSnappyBlockAsm64K:
|
|
MOVL DX, DI
|
|
MOVL 12(SP), R8
|
|
CMPL R8, DI
|
|
JEQ emit_literal_done_match_emit_encodeSnappyBlockAsm64K
|
|
MOVL DI, R9
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(R8*1), DI
|
|
SUBL R8, R9
|
|
LEAL -1(R9), R8
|
|
CMPL R8, $0x3c
|
|
JB one_byte_match_emit_encodeSnappyBlockAsm64K
|
|
CMPL R8, $0x00000100
|
|
JB two_bytes_match_emit_encodeSnappyBlockAsm64K
|
|
JB three_bytes_match_emit_encodeSnappyBlockAsm64K
|
|
|
|
three_bytes_match_emit_encodeSnappyBlockAsm64K:
|
|
MOVB $0xf4, (CX)
|
|
MOVW R8, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_match_emit_encodeSnappyBlockAsm64K
|
|
|
|
two_bytes_match_emit_encodeSnappyBlockAsm64K:
|
|
MOVB $0xf0, (CX)
|
|
MOVB R8, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL R8, $0x40
|
|
JB memmove_match_emit_encodeSnappyBlockAsm64K
|
|
JMP memmove_long_match_emit_encodeSnappyBlockAsm64K
|
|
|
|
one_byte_match_emit_encodeSnappyBlockAsm64K:
|
|
SHLB $0x02, R8
|
|
MOVB R8, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_match_emit_encodeSnappyBlockAsm64K:
|
|
LEAQ (CX)(R9*1), R8
|
|
|
|
// genMemMoveShort
|
|
CMPQ R9, $0x08
|
|
JBE emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_8
|
|
CMPQ R9, $0x10
|
|
JBE emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_8through16
|
|
CMPQ R9, $0x20
|
|
JBE emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_8:
|
|
MOVQ (DI), R10
|
|
MOVQ R10, (CX)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm64K
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_8through16:
|
|
MOVQ (DI), R10
|
|
MOVQ -8(DI)(R9*1), DI
|
|
MOVQ R10, (CX)
|
|
MOVQ DI, -8(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm64K
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_17through32:
|
|
MOVOU (DI), X0
|
|
MOVOU -16(DI)(R9*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm64K
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_33through64:
|
|
MOVOU (DI), X0
|
|
MOVOU 16(DI), X1
|
|
MOVOU -32(DI)(R9*1), X2
|
|
MOVOU -16(DI)(R9*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
|
|
memmove_end_copy_match_emit_encodeSnappyBlockAsm64K:
|
|
MOVQ R8, CX
|
|
JMP emit_literal_done_match_emit_encodeSnappyBlockAsm64K
|
|
|
|
memmove_long_match_emit_encodeSnappyBlockAsm64K:
|
|
LEAQ (CX)(R9*1), R8
|
|
|
|
// genMemMoveLong
|
|
MOVOU (DI), X0
|
|
MOVOU 16(DI), X1
|
|
MOVOU -32(DI)(R9*1), X2
|
|
MOVOU -16(DI)(R9*1), X3
|
|
MOVQ R9, R11
|
|
SHRQ $0x05, R11
|
|
MOVQ CX, R10
|
|
ANDL $0x0000001f, R10
|
|
MOVQ $0x00000040, R12
|
|
SUBQ R10, R12
|
|
DECQ R11
|
|
JA emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm64Klarge_forward_sse_loop_32
|
|
LEAQ -32(DI)(R12*1), R10
|
|
LEAQ -32(CX)(R12*1), R13
|
|
|
|
emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm64Klarge_big_loop_back:
|
|
MOVOU (R10), X4
|
|
MOVOU 16(R10), X5
|
|
MOVOA X4, (R13)
|
|
MOVOA X5, 16(R13)
|
|
ADDQ $0x20, R13
|
|
ADDQ $0x20, R10
|
|
ADDQ $0x20, R12
|
|
DECQ R11
|
|
JNA emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm64Klarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm64Klarge_forward_sse_loop_32:
|
|
MOVOU -32(DI)(R12*1), X4
|
|
MOVOU -16(DI)(R12*1), X5
|
|
MOVOA X4, -32(CX)(R12*1)
|
|
MOVOA X5, -16(CX)(R12*1)
|
|
ADDQ $0x20, R12
|
|
CMPQ R9, R12
|
|
JAE emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm64Klarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
MOVQ R8, CX
|
|
|
|
emit_literal_done_match_emit_encodeSnappyBlockAsm64K:
|
|
match_nolit_loop_encodeSnappyBlockAsm64K:
|
|
MOVL DX, DI
|
|
SUBL SI, DI
|
|
MOVL DI, 16(SP)
|
|
ADDL $0x04, DX
|
|
ADDL $0x04, SI
|
|
MOVQ src_len+32(FP), DI
|
|
SUBL DX, DI
|
|
LEAQ (BX)(DX*1), R8
|
|
LEAQ (BX)(SI*1), SI
|
|
|
|
// matchLen
|
|
XORL R10, R10
|
|
|
|
matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm64K:
|
|
CMPL DI, $0x10
|
|
JB matchlen_match8_match_nolit_encodeSnappyBlockAsm64K
|
|
MOVQ (R8)(R10*1), R9
|
|
MOVQ 8(R8)(R10*1), R11
|
|
XORQ (SI)(R10*1), R9
|
|
JNZ matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm64K
|
|
XORQ 8(SI)(R10*1), R11
|
|
JNZ matchlen_bsf_16match_nolit_encodeSnappyBlockAsm64K
|
|
LEAL -16(DI), DI
|
|
LEAL 16(R10), R10
|
|
JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm64K
|
|
|
|
matchlen_bsf_16match_nolit_encodeSnappyBlockAsm64K:
|
|
TZCNTQ R11, R11
|
|
SARQ $0x03, R11
|
|
LEAL 8(R10)(R11*1), R10
|
|
JMP match_nolit_end_encodeSnappyBlockAsm64K
|
|
|
|
matchlen_match8_match_nolit_encodeSnappyBlockAsm64K:
|
|
CMPL DI, $0x08
|
|
JB matchlen_match4_match_nolit_encodeSnappyBlockAsm64K
|
|
MOVQ (R8)(R10*1), R9
|
|
XORQ (SI)(R10*1), R9
|
|
JNZ matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm64K
|
|
LEAL -8(DI), DI
|
|
LEAL 8(R10), R10
|
|
JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm64K
|
|
|
|
matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm64K:
|
|
TZCNTQ R9, R9
|
|
SARQ $0x03, R9
|
|
LEAL (R10)(R9*1), R10
|
|
JMP match_nolit_end_encodeSnappyBlockAsm64K
|
|
|
|
matchlen_match4_match_nolit_encodeSnappyBlockAsm64K:
|
|
CMPL DI, $0x04
|
|
JB matchlen_match2_match_nolit_encodeSnappyBlockAsm64K
|
|
MOVL (R8)(R10*1), R9
|
|
CMPL (SI)(R10*1), R9
|
|
JNE matchlen_match2_match_nolit_encodeSnappyBlockAsm64K
|
|
LEAL -4(DI), DI
|
|
LEAL 4(R10), R10
|
|
|
|
matchlen_match2_match_nolit_encodeSnappyBlockAsm64K:
|
|
CMPL DI, $0x01
|
|
JE matchlen_match1_match_nolit_encodeSnappyBlockAsm64K
|
|
JB match_nolit_end_encodeSnappyBlockAsm64K
|
|
MOVW (R8)(R10*1), R9
|
|
CMPW (SI)(R10*1), R9
|
|
JNE matchlen_match1_match_nolit_encodeSnappyBlockAsm64K
|
|
LEAL 2(R10), R10
|
|
SUBL $0x02, DI
|
|
JZ match_nolit_end_encodeSnappyBlockAsm64K
|
|
|
|
matchlen_match1_match_nolit_encodeSnappyBlockAsm64K:
|
|
MOVB (R8)(R10*1), R9
|
|
CMPB (SI)(R10*1), R9
|
|
JNE match_nolit_end_encodeSnappyBlockAsm64K
|
|
LEAL 1(R10), R10
|
|
|
|
match_nolit_end_encodeSnappyBlockAsm64K:
|
|
ADDL R10, DX
|
|
MOVL 16(SP), SI
|
|
ADDL $0x04, R10
|
|
MOVL DX, 12(SP)
|
|
|
|
// emitCopy
|
|
two_byte_offset_match_nolit_encodeSnappyBlockAsm64K:
|
|
CMPL R10, $0x40
|
|
JBE two_byte_offset_short_match_nolit_encodeSnappyBlockAsm64K
|
|
MOVB $0xee, (CX)
|
|
MOVW SI, 1(CX)
|
|
LEAL -60(R10), R10
|
|
ADDQ $0x03, CX
|
|
JMP two_byte_offset_match_nolit_encodeSnappyBlockAsm64K
|
|
|
|
two_byte_offset_short_match_nolit_encodeSnappyBlockAsm64K:
|
|
MOVL R10, DI
|
|
SHLL $0x02, DI
|
|
CMPL R10, $0x0c
|
|
JAE emit_copy_three_match_nolit_encodeSnappyBlockAsm64K
|
|
CMPL SI, $0x00000800
|
|
JAE emit_copy_three_match_nolit_encodeSnappyBlockAsm64K
|
|
LEAL -15(DI), DI
|
|
MOVB SI, 1(CX)
|
|
SHRL $0x08, SI
|
|
SHLL $0x05, SI
|
|
ORL SI, DI
|
|
MOVB DI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeSnappyBlockAsm64K
|
|
|
|
emit_copy_three_match_nolit_encodeSnappyBlockAsm64K:
|
|
LEAL -2(DI), DI
|
|
MOVB DI, (CX)
|
|
MOVW SI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
|
|
match_nolit_emitcopy_end_encodeSnappyBlockAsm64K:
|
|
CMPL DX, 8(SP)
|
|
JAE emit_remainder_encodeSnappyBlockAsm64K
|
|
MOVQ -2(BX)(DX*1), DI
|
|
CMPQ CX, (SP)
|
|
JB match_nolit_dst_ok_encodeSnappyBlockAsm64K
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeSnappyBlockAsm64K:
|
|
MOVQ $0x0000cf1bbcdcbf9b, R9
|
|
MOVQ DI, R8
|
|
SHRQ $0x10, DI
|
|
MOVQ DI, SI
|
|
SHLQ $0x10, R8
|
|
IMULQ R9, R8
|
|
SHRQ $0x32, R8
|
|
SHLQ $0x10, SI
|
|
IMULQ R9, SI
|
|
SHRQ $0x32, SI
|
|
LEAL -2(DX), R9
|
|
LEAQ (AX)(SI*4), R10
|
|
MOVL (R10), SI
|
|
MOVL R9, (AX)(R8*4)
|
|
MOVL DX, (R10)
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ match_nolit_loop_encodeSnappyBlockAsm64K
|
|
INCL DX
|
|
JMP search_loop_encodeSnappyBlockAsm64K
|
|
|
|
emit_remainder_encodeSnappyBlockAsm64K:
|
|
MOVQ src_len+32(FP), AX
|
|
SUBL 12(SP), AX
|
|
LEAQ 3(CX)(AX*1), AX
|
|
CMPQ AX, (SP)
|
|
JB emit_remainder_ok_encodeSnappyBlockAsm64K
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeSnappyBlockAsm64K:
|
|
MOVQ src_len+32(FP), AX
|
|
MOVL 12(SP), DX
|
|
CMPL DX, AX
|
|
JEQ emit_literal_done_emit_remainder_encodeSnappyBlockAsm64K
|
|
MOVL AX, SI
|
|
MOVL AX, 12(SP)
|
|
LEAQ (BX)(DX*1), AX
|
|
SUBL DX, SI
|
|
LEAL -1(SI), DX
|
|
CMPL DX, $0x3c
|
|
JB one_byte_emit_remainder_encodeSnappyBlockAsm64K
|
|
CMPL DX, $0x00000100
|
|
JB two_bytes_emit_remainder_encodeSnappyBlockAsm64K
|
|
JB three_bytes_emit_remainder_encodeSnappyBlockAsm64K
|
|
|
|
three_bytes_emit_remainder_encodeSnappyBlockAsm64K:
|
|
MOVB $0xf4, (CX)
|
|
MOVW DX, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_emit_remainder_encodeSnappyBlockAsm64K
|
|
|
|
two_bytes_emit_remainder_encodeSnappyBlockAsm64K:
|
|
MOVB $0xf0, (CX)
|
|
MOVB DL, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL DX, $0x40
|
|
JB memmove_emit_remainder_encodeSnappyBlockAsm64K
|
|
JMP memmove_long_emit_remainder_encodeSnappyBlockAsm64K
|
|
|
|
one_byte_emit_remainder_encodeSnappyBlockAsm64K:
|
|
SHLB $0x02, DL
|
|
MOVB DL, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_emit_remainder_encodeSnappyBlockAsm64K:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveShort
|
|
CMPQ BX, $0x03
|
|
JB emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_1or2
|
|
JE emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_3
|
|
CMPQ BX, $0x08
|
|
JB emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_4through7
|
|
CMPQ BX, $0x10
|
|
JBE emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_8through16
|
|
CMPQ BX, $0x20
|
|
JBE emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_1or2:
|
|
MOVB (AX), SI
|
|
MOVB -1(AX)(BX*1), AL
|
|
MOVB SI, (CX)
|
|
MOVB AL, -1(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm64K
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_3:
|
|
MOVW (AX), SI
|
|
MOVB 2(AX), AL
|
|
MOVW SI, (CX)
|
|
MOVB AL, 2(CX)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm64K
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_4through7:
|
|
MOVL (AX), SI
|
|
MOVL -4(AX)(BX*1), AX
|
|
MOVL SI, (CX)
|
|
MOVL AX, -4(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm64K
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_8through16:
|
|
MOVQ (AX), SI
|
|
MOVQ -8(AX)(BX*1), AX
|
|
MOVQ SI, (CX)
|
|
MOVQ AX, -8(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm64K
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_17through32:
|
|
MOVOU (AX), X0
|
|
MOVOU -16(AX)(BX*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm64K
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_33through64:
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
|
|
memmove_end_copy_emit_remainder_encodeSnappyBlockAsm64K:
|
|
MOVQ DX, CX
|
|
JMP emit_literal_done_emit_remainder_encodeSnappyBlockAsm64K
|
|
|
|
memmove_long_emit_remainder_encodeSnappyBlockAsm64K:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveLong
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVQ BX, DI
|
|
SHRQ $0x05, DI
|
|
MOVQ CX, SI
|
|
ANDL $0x0000001f, SI
|
|
MOVQ $0x00000040, R8
|
|
SUBQ SI, R8
|
|
DECQ DI
|
|
JA emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm64Klarge_forward_sse_loop_32
|
|
LEAQ -32(AX)(R8*1), SI
|
|
LEAQ -32(CX)(R8*1), R9
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm64Klarge_big_loop_back:
|
|
MOVOU (SI), X4
|
|
MOVOU 16(SI), X5
|
|
MOVOA X4, (R9)
|
|
MOVOA X5, 16(R9)
|
|
ADDQ $0x20, R9
|
|
ADDQ $0x20, SI
|
|
ADDQ $0x20, R8
|
|
DECQ DI
|
|
JNA emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm64Klarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm64Klarge_forward_sse_loop_32:
|
|
MOVOU -32(AX)(R8*1), X4
|
|
MOVOU -16(AX)(R8*1), X5
|
|
MOVOA X4, -32(CX)(R8*1)
|
|
MOVOA X5, -16(CX)(R8*1)
|
|
ADDQ $0x20, R8
|
|
CMPQ BX, R8
|
|
JAE emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm64Klarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
MOVQ DX, CX
|
|
|
|
emit_literal_done_emit_remainder_encodeSnappyBlockAsm64K:
|
|
MOVQ dst_base+0(FP), AX
|
|
SUBQ AX, CX
|
|
MOVQ CX, ret+56(FP)
|
|
RET
|
|
|
|
// func encodeSnappyBlockAsm12B(dst []byte, src []byte, tmp *[16384]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeSnappyBlockAsm12B(SB), $24-64
|
|
MOVQ tmp+48(FP), AX
|
|
MOVQ dst_base+0(FP), CX
|
|
MOVQ $0x00000080, DX
|
|
MOVQ AX, BX
|
|
PXOR X0, X0
|
|
|
|
zero_loop_encodeSnappyBlockAsm12B:
|
|
MOVOU X0, (BX)
|
|
MOVOU X0, 16(BX)
|
|
MOVOU X0, 32(BX)
|
|
MOVOU X0, 48(BX)
|
|
MOVOU X0, 64(BX)
|
|
MOVOU X0, 80(BX)
|
|
MOVOU X0, 96(BX)
|
|
MOVOU X0, 112(BX)
|
|
ADDQ $0x80, BX
|
|
DECQ DX
|
|
JNZ zero_loop_encodeSnappyBlockAsm12B
|
|
MOVL $0x00000000, 12(SP)
|
|
MOVQ src_len+32(FP), DX
|
|
LEAQ -9(DX), BX
|
|
LEAQ -8(DX), SI
|
|
MOVL SI, 8(SP)
|
|
SHRQ $0x05, DX
|
|
SUBL DX, BX
|
|
LEAQ (CX)(BX*1), BX
|
|
MOVQ BX, (SP)
|
|
MOVL $0x00000001, DX
|
|
MOVL DX, 16(SP)
|
|
MOVQ src_base+24(FP), BX
|
|
|
|
search_loop_encodeSnappyBlockAsm12B:
|
|
MOVL DX, SI
|
|
SUBL 12(SP), SI
|
|
SHRL $0x05, SI
|
|
LEAL 4(DX)(SI*1), SI
|
|
CMPL SI, 8(SP)
|
|
JAE emit_remainder_encodeSnappyBlockAsm12B
|
|
MOVQ (BX)(DX*1), DI
|
|
MOVL SI, 20(SP)
|
|
MOVQ $0x000000cf1bbcdcbb, R9
|
|
MOVQ DI, R10
|
|
MOVQ DI, R11
|
|
SHRQ $0x08, R11
|
|
SHLQ $0x18, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x34, R10
|
|
SHLQ $0x18, R11
|
|
IMULQ R9, R11
|
|
SHRQ $0x34, R11
|
|
MOVL (AX)(R10*4), SI
|
|
MOVL (AX)(R11*4), R8
|
|
MOVL DX, (AX)(R10*4)
|
|
LEAL 1(DX), R10
|
|
MOVL R10, (AX)(R11*4)
|
|
MOVQ DI, R10
|
|
SHRQ $0x10, R10
|
|
SHLQ $0x18, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x34, R10
|
|
MOVL DX, R9
|
|
SUBL 16(SP), R9
|
|
MOVL 1(BX)(R9*1), R11
|
|
MOVQ DI, R9
|
|
SHRQ $0x08, R9
|
|
CMPL R9, R11
|
|
JNE no_repeat_found_encodeSnappyBlockAsm12B
|
|
LEAL 1(DX), DI
|
|
MOVL 12(SP), SI
|
|
MOVL DI, R8
|
|
SUBL 16(SP), R8
|
|
JZ repeat_extend_back_end_encodeSnappyBlockAsm12B
|
|
|
|
repeat_extend_back_loop_encodeSnappyBlockAsm12B:
|
|
CMPL DI, SI
|
|
JBE repeat_extend_back_end_encodeSnappyBlockAsm12B
|
|
MOVB -1(BX)(R8*1), R9
|
|
MOVB -1(BX)(DI*1), R10
|
|
CMPB R9, R10
|
|
JNE repeat_extend_back_end_encodeSnappyBlockAsm12B
|
|
LEAL -1(DI), DI
|
|
DECL R8
|
|
JNZ repeat_extend_back_loop_encodeSnappyBlockAsm12B
|
|
|
|
repeat_extend_back_end_encodeSnappyBlockAsm12B:
|
|
MOVL DI, SI
|
|
SUBL 12(SP), SI
|
|
LEAQ 3(CX)(SI*1), SI
|
|
CMPQ SI, (SP)
|
|
JB repeat_dst_size_check_encodeSnappyBlockAsm12B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
repeat_dst_size_check_encodeSnappyBlockAsm12B:
|
|
MOVL 12(SP), SI
|
|
CMPL SI, DI
|
|
JEQ emit_literal_done_repeat_emit_encodeSnappyBlockAsm12B
|
|
MOVL DI, R8
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(SI*1), R9
|
|
SUBL SI, R8
|
|
LEAL -1(R8), SI
|
|
CMPL SI, $0x3c
|
|
JB one_byte_repeat_emit_encodeSnappyBlockAsm12B
|
|
CMPL SI, $0x00000100
|
|
JB two_bytes_repeat_emit_encodeSnappyBlockAsm12B
|
|
JB three_bytes_repeat_emit_encodeSnappyBlockAsm12B
|
|
|
|
three_bytes_repeat_emit_encodeSnappyBlockAsm12B:
|
|
MOVB $0xf4, (CX)
|
|
MOVW SI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_repeat_emit_encodeSnappyBlockAsm12B
|
|
|
|
two_bytes_repeat_emit_encodeSnappyBlockAsm12B:
|
|
MOVB $0xf0, (CX)
|
|
MOVB SI, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL SI, $0x40
|
|
JB memmove_repeat_emit_encodeSnappyBlockAsm12B
|
|
JMP memmove_long_repeat_emit_encodeSnappyBlockAsm12B
|
|
|
|
one_byte_repeat_emit_encodeSnappyBlockAsm12B:
|
|
SHLB $0x02, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_repeat_emit_encodeSnappyBlockAsm12B:
|
|
LEAQ (CX)(R8*1), SI
|
|
|
|
// genMemMoveShort
|
|
CMPQ R8, $0x08
|
|
JBE emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_8
|
|
CMPQ R8, $0x10
|
|
JBE emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_8through16
|
|
CMPQ R8, $0x20
|
|
JBE emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_17through32
|
|
JMP emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_8:
|
|
MOVQ (R9), R10
|
|
MOVQ R10, (CX)
|
|
JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm12B
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_8through16:
|
|
MOVQ (R9), R10
|
|
MOVQ -8(R9)(R8*1), R9
|
|
MOVQ R10, (CX)
|
|
MOVQ R9, -8(CX)(R8*1)
|
|
JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm12B
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_17through32:
|
|
MOVOU (R9), X0
|
|
MOVOU -16(R9)(R8*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(R8*1)
|
|
JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm12B
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_33through64:
|
|
MOVOU (R9), X0
|
|
MOVOU 16(R9), X1
|
|
MOVOU -32(R9)(R8*1), X2
|
|
MOVOU -16(R9)(R8*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R8*1)
|
|
MOVOU X3, -16(CX)(R8*1)
|
|
|
|
memmove_end_copy_repeat_emit_encodeSnappyBlockAsm12B:
|
|
MOVQ SI, CX
|
|
JMP emit_literal_done_repeat_emit_encodeSnappyBlockAsm12B
|
|
|
|
memmove_long_repeat_emit_encodeSnappyBlockAsm12B:
|
|
LEAQ (CX)(R8*1), SI
|
|
|
|
// genMemMoveLong
|
|
MOVOU (R9), X0
|
|
MOVOU 16(R9), X1
|
|
MOVOU -32(R9)(R8*1), X2
|
|
MOVOU -16(R9)(R8*1), X3
|
|
MOVQ R8, R11
|
|
SHRQ $0x05, R11
|
|
MOVQ CX, R10
|
|
ANDL $0x0000001f, R10
|
|
MOVQ $0x00000040, R12
|
|
SUBQ R10, R12
|
|
DECQ R11
|
|
JA emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm12Blarge_forward_sse_loop_32
|
|
LEAQ -32(R9)(R12*1), R10
|
|
LEAQ -32(CX)(R12*1), R13
|
|
|
|
emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm12Blarge_big_loop_back:
|
|
MOVOU (R10), X4
|
|
MOVOU 16(R10), X5
|
|
MOVOA X4, (R13)
|
|
MOVOA X5, 16(R13)
|
|
ADDQ $0x20, R13
|
|
ADDQ $0x20, R10
|
|
ADDQ $0x20, R12
|
|
DECQ R11
|
|
JNA emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm12Blarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm12Blarge_forward_sse_loop_32:
|
|
MOVOU -32(R9)(R12*1), X4
|
|
MOVOU -16(R9)(R12*1), X5
|
|
MOVOA X4, -32(CX)(R12*1)
|
|
MOVOA X5, -16(CX)(R12*1)
|
|
ADDQ $0x20, R12
|
|
CMPQ R8, R12
|
|
JAE emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm12Blarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R8*1)
|
|
MOVOU X3, -16(CX)(R8*1)
|
|
MOVQ SI, CX
|
|
|
|
emit_literal_done_repeat_emit_encodeSnappyBlockAsm12B:
|
|
ADDL $0x05, DX
|
|
MOVL DX, SI
|
|
SUBL 16(SP), SI
|
|
MOVQ src_len+32(FP), R8
|
|
SUBL DX, R8
|
|
LEAQ (BX)(DX*1), R9
|
|
LEAQ (BX)(SI*1), SI
|
|
|
|
// matchLen
|
|
XORL R11, R11
|
|
|
|
matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm12B:
|
|
CMPL R8, $0x10
|
|
JB matchlen_match8_repeat_extend_encodeSnappyBlockAsm12B
|
|
MOVQ (R9)(R11*1), R10
|
|
MOVQ 8(R9)(R11*1), R12
|
|
XORQ (SI)(R11*1), R10
|
|
JNZ matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm12B
|
|
XORQ 8(SI)(R11*1), R12
|
|
JNZ matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm12B
|
|
LEAL -16(R8), R8
|
|
LEAL 16(R11), R11
|
|
JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm12B
|
|
|
|
matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm12B:
|
|
TZCNTQ R12, R12
|
|
SARQ $0x03, R12
|
|
LEAL 8(R11)(R12*1), R11
|
|
JMP repeat_extend_forward_end_encodeSnappyBlockAsm12B
|
|
|
|
matchlen_match8_repeat_extend_encodeSnappyBlockAsm12B:
|
|
CMPL R8, $0x08
|
|
JB matchlen_match4_repeat_extend_encodeSnappyBlockAsm12B
|
|
MOVQ (R9)(R11*1), R10
|
|
XORQ (SI)(R11*1), R10
|
|
JNZ matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm12B
|
|
LEAL -8(R8), R8
|
|
LEAL 8(R11), R11
|
|
JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm12B
|
|
|
|
matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm12B:
|
|
TZCNTQ R10, R10
|
|
SARQ $0x03, R10
|
|
LEAL (R11)(R10*1), R11
|
|
JMP repeat_extend_forward_end_encodeSnappyBlockAsm12B
|
|
|
|
matchlen_match4_repeat_extend_encodeSnappyBlockAsm12B:
|
|
CMPL R8, $0x04
|
|
JB matchlen_match2_repeat_extend_encodeSnappyBlockAsm12B
|
|
MOVL (R9)(R11*1), R10
|
|
CMPL (SI)(R11*1), R10
|
|
JNE matchlen_match2_repeat_extend_encodeSnappyBlockAsm12B
|
|
LEAL -4(R8), R8
|
|
LEAL 4(R11), R11
|
|
|
|
matchlen_match2_repeat_extend_encodeSnappyBlockAsm12B:
|
|
CMPL R8, $0x01
|
|
JE matchlen_match1_repeat_extend_encodeSnappyBlockAsm12B
|
|
JB repeat_extend_forward_end_encodeSnappyBlockAsm12B
|
|
MOVW (R9)(R11*1), R10
|
|
CMPW (SI)(R11*1), R10
|
|
JNE matchlen_match1_repeat_extend_encodeSnappyBlockAsm12B
|
|
LEAL 2(R11), R11
|
|
SUBL $0x02, R8
|
|
JZ repeat_extend_forward_end_encodeSnappyBlockAsm12B
|
|
|
|
matchlen_match1_repeat_extend_encodeSnappyBlockAsm12B:
|
|
MOVB (R9)(R11*1), R10
|
|
CMPB (SI)(R11*1), R10
|
|
JNE repeat_extend_forward_end_encodeSnappyBlockAsm12B
|
|
LEAL 1(R11), R11
|
|
|
|
repeat_extend_forward_end_encodeSnappyBlockAsm12B:
|
|
ADDL R11, DX
|
|
MOVL DX, SI
|
|
SUBL DI, SI
|
|
MOVL 16(SP), DI
|
|
|
|
// emitCopy
|
|
two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm12B:
|
|
CMPL SI, $0x40
|
|
JBE two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm12B
|
|
MOVB $0xee, (CX)
|
|
MOVW DI, 1(CX)
|
|
LEAL -60(SI), SI
|
|
ADDQ $0x03, CX
|
|
JMP two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm12B
|
|
|
|
two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm12B:
|
|
MOVL SI, R8
|
|
SHLL $0x02, R8
|
|
CMPL SI, $0x0c
|
|
JAE emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm12B
|
|
CMPL DI, $0x00000800
|
|
JAE emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm12B
|
|
LEAL -15(R8), R8
|
|
MOVB DI, 1(CX)
|
|
SHRL $0x08, DI
|
|
SHLL $0x05, DI
|
|
ORL DI, R8
|
|
MOVB R8, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeSnappyBlockAsm12B
|
|
|
|
emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm12B:
|
|
LEAL -2(R8), R8
|
|
MOVB R8, (CX)
|
|
MOVW DI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
|
|
repeat_end_emit_encodeSnappyBlockAsm12B:
|
|
MOVL DX, 12(SP)
|
|
JMP search_loop_encodeSnappyBlockAsm12B
|
|
|
|
no_repeat_found_encodeSnappyBlockAsm12B:
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ candidate_match_encodeSnappyBlockAsm12B
|
|
SHRQ $0x08, DI
|
|
MOVL (AX)(R10*4), SI
|
|
LEAL 2(DX), R9
|
|
CMPL (BX)(R8*1), DI
|
|
JEQ candidate2_match_encodeSnappyBlockAsm12B
|
|
MOVL R9, (AX)(R10*4)
|
|
SHRQ $0x08, DI
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ candidate3_match_encodeSnappyBlockAsm12B
|
|
MOVL 20(SP), DX
|
|
JMP search_loop_encodeSnappyBlockAsm12B
|
|
|
|
candidate3_match_encodeSnappyBlockAsm12B:
|
|
ADDL $0x02, DX
|
|
JMP candidate_match_encodeSnappyBlockAsm12B
|
|
|
|
candidate2_match_encodeSnappyBlockAsm12B:
|
|
MOVL R9, (AX)(R10*4)
|
|
INCL DX
|
|
MOVL R8, SI
|
|
|
|
candidate_match_encodeSnappyBlockAsm12B:
|
|
MOVL 12(SP), DI
|
|
TESTL SI, SI
|
|
JZ match_extend_back_end_encodeSnappyBlockAsm12B
|
|
|
|
match_extend_back_loop_encodeSnappyBlockAsm12B:
|
|
CMPL DX, DI
|
|
JBE match_extend_back_end_encodeSnappyBlockAsm12B
|
|
MOVB -1(BX)(SI*1), R8
|
|
MOVB -1(BX)(DX*1), R9
|
|
CMPB R8, R9
|
|
JNE match_extend_back_end_encodeSnappyBlockAsm12B
|
|
LEAL -1(DX), DX
|
|
DECL SI
|
|
JZ match_extend_back_end_encodeSnappyBlockAsm12B
|
|
JMP match_extend_back_loop_encodeSnappyBlockAsm12B
|
|
|
|
match_extend_back_end_encodeSnappyBlockAsm12B:
|
|
MOVL DX, DI
|
|
SUBL 12(SP), DI
|
|
LEAQ 3(CX)(DI*1), DI
|
|
CMPQ DI, (SP)
|
|
JB match_dst_size_check_encodeSnappyBlockAsm12B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeSnappyBlockAsm12B:
|
|
MOVL DX, DI
|
|
MOVL 12(SP), R8
|
|
CMPL R8, DI
|
|
JEQ emit_literal_done_match_emit_encodeSnappyBlockAsm12B
|
|
MOVL DI, R9
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(R8*1), DI
|
|
SUBL R8, R9
|
|
LEAL -1(R9), R8
|
|
CMPL R8, $0x3c
|
|
JB one_byte_match_emit_encodeSnappyBlockAsm12B
|
|
CMPL R8, $0x00000100
|
|
JB two_bytes_match_emit_encodeSnappyBlockAsm12B
|
|
JB three_bytes_match_emit_encodeSnappyBlockAsm12B
|
|
|
|
three_bytes_match_emit_encodeSnappyBlockAsm12B:
|
|
MOVB $0xf4, (CX)
|
|
MOVW R8, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_match_emit_encodeSnappyBlockAsm12B
|
|
|
|
two_bytes_match_emit_encodeSnappyBlockAsm12B:
|
|
MOVB $0xf0, (CX)
|
|
MOVB R8, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL R8, $0x40
|
|
JB memmove_match_emit_encodeSnappyBlockAsm12B
|
|
JMP memmove_long_match_emit_encodeSnappyBlockAsm12B
|
|
|
|
one_byte_match_emit_encodeSnappyBlockAsm12B:
|
|
SHLB $0x02, R8
|
|
MOVB R8, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_match_emit_encodeSnappyBlockAsm12B:
|
|
LEAQ (CX)(R9*1), R8
|
|
|
|
// genMemMoveShort
|
|
CMPQ R9, $0x08
|
|
JBE emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_8
|
|
CMPQ R9, $0x10
|
|
JBE emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_8through16
|
|
CMPQ R9, $0x20
|
|
JBE emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_8:
|
|
MOVQ (DI), R10
|
|
MOVQ R10, (CX)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm12B
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_8through16:
|
|
MOVQ (DI), R10
|
|
MOVQ -8(DI)(R9*1), DI
|
|
MOVQ R10, (CX)
|
|
MOVQ DI, -8(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm12B
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_17through32:
|
|
MOVOU (DI), X0
|
|
MOVOU -16(DI)(R9*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm12B
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_33through64:
|
|
MOVOU (DI), X0
|
|
MOVOU 16(DI), X1
|
|
MOVOU -32(DI)(R9*1), X2
|
|
MOVOU -16(DI)(R9*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
|
|
memmove_end_copy_match_emit_encodeSnappyBlockAsm12B:
|
|
MOVQ R8, CX
|
|
JMP emit_literal_done_match_emit_encodeSnappyBlockAsm12B
|
|
|
|
memmove_long_match_emit_encodeSnappyBlockAsm12B:
|
|
LEAQ (CX)(R9*1), R8
|
|
|
|
// genMemMoveLong
|
|
MOVOU (DI), X0
|
|
MOVOU 16(DI), X1
|
|
MOVOU -32(DI)(R9*1), X2
|
|
MOVOU -16(DI)(R9*1), X3
|
|
MOVQ R9, R11
|
|
SHRQ $0x05, R11
|
|
MOVQ CX, R10
|
|
ANDL $0x0000001f, R10
|
|
MOVQ $0x00000040, R12
|
|
SUBQ R10, R12
|
|
DECQ R11
|
|
JA emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm12Blarge_forward_sse_loop_32
|
|
LEAQ -32(DI)(R12*1), R10
|
|
LEAQ -32(CX)(R12*1), R13
|
|
|
|
emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm12Blarge_big_loop_back:
|
|
MOVOU (R10), X4
|
|
MOVOU 16(R10), X5
|
|
MOVOA X4, (R13)
|
|
MOVOA X5, 16(R13)
|
|
ADDQ $0x20, R13
|
|
ADDQ $0x20, R10
|
|
ADDQ $0x20, R12
|
|
DECQ R11
|
|
JNA emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm12Blarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm12Blarge_forward_sse_loop_32:
|
|
MOVOU -32(DI)(R12*1), X4
|
|
MOVOU -16(DI)(R12*1), X5
|
|
MOVOA X4, -32(CX)(R12*1)
|
|
MOVOA X5, -16(CX)(R12*1)
|
|
ADDQ $0x20, R12
|
|
CMPQ R9, R12
|
|
JAE emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm12Blarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
MOVQ R8, CX
|
|
|
|
emit_literal_done_match_emit_encodeSnappyBlockAsm12B:
|
|
match_nolit_loop_encodeSnappyBlockAsm12B:
|
|
MOVL DX, DI
|
|
SUBL SI, DI
|
|
MOVL DI, 16(SP)
|
|
ADDL $0x04, DX
|
|
ADDL $0x04, SI
|
|
MOVQ src_len+32(FP), DI
|
|
SUBL DX, DI
|
|
LEAQ (BX)(DX*1), R8
|
|
LEAQ (BX)(SI*1), SI
|
|
|
|
// matchLen
|
|
XORL R10, R10
|
|
|
|
matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm12B:
|
|
CMPL DI, $0x10
|
|
JB matchlen_match8_match_nolit_encodeSnappyBlockAsm12B
|
|
MOVQ (R8)(R10*1), R9
|
|
MOVQ 8(R8)(R10*1), R11
|
|
XORQ (SI)(R10*1), R9
|
|
JNZ matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm12B
|
|
XORQ 8(SI)(R10*1), R11
|
|
JNZ matchlen_bsf_16match_nolit_encodeSnappyBlockAsm12B
|
|
LEAL -16(DI), DI
|
|
LEAL 16(R10), R10
|
|
JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm12B
|
|
|
|
matchlen_bsf_16match_nolit_encodeSnappyBlockAsm12B:
|
|
TZCNTQ R11, R11
|
|
SARQ $0x03, R11
|
|
LEAL 8(R10)(R11*1), R10
|
|
JMP match_nolit_end_encodeSnappyBlockAsm12B
|
|
|
|
matchlen_match8_match_nolit_encodeSnappyBlockAsm12B:
|
|
CMPL DI, $0x08
|
|
JB matchlen_match4_match_nolit_encodeSnappyBlockAsm12B
|
|
MOVQ (R8)(R10*1), R9
|
|
XORQ (SI)(R10*1), R9
|
|
JNZ matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm12B
|
|
LEAL -8(DI), DI
|
|
LEAL 8(R10), R10
|
|
JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm12B
|
|
|
|
matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm12B:
|
|
TZCNTQ R9, R9
|
|
SARQ $0x03, R9
|
|
LEAL (R10)(R9*1), R10
|
|
JMP match_nolit_end_encodeSnappyBlockAsm12B
|
|
|
|
matchlen_match4_match_nolit_encodeSnappyBlockAsm12B:
|
|
CMPL DI, $0x04
|
|
JB matchlen_match2_match_nolit_encodeSnappyBlockAsm12B
|
|
MOVL (R8)(R10*1), R9
|
|
CMPL (SI)(R10*1), R9
|
|
JNE matchlen_match2_match_nolit_encodeSnappyBlockAsm12B
|
|
LEAL -4(DI), DI
|
|
LEAL 4(R10), R10
|
|
|
|
matchlen_match2_match_nolit_encodeSnappyBlockAsm12B:
|
|
CMPL DI, $0x01
|
|
JE matchlen_match1_match_nolit_encodeSnappyBlockAsm12B
|
|
JB match_nolit_end_encodeSnappyBlockAsm12B
|
|
MOVW (R8)(R10*1), R9
|
|
CMPW (SI)(R10*1), R9
|
|
JNE matchlen_match1_match_nolit_encodeSnappyBlockAsm12B
|
|
LEAL 2(R10), R10
|
|
SUBL $0x02, DI
|
|
JZ match_nolit_end_encodeSnappyBlockAsm12B
|
|
|
|
matchlen_match1_match_nolit_encodeSnappyBlockAsm12B:
|
|
MOVB (R8)(R10*1), R9
|
|
CMPB (SI)(R10*1), R9
|
|
JNE match_nolit_end_encodeSnappyBlockAsm12B
|
|
LEAL 1(R10), R10
|
|
|
|
match_nolit_end_encodeSnappyBlockAsm12B:
|
|
ADDL R10, DX
|
|
MOVL 16(SP), SI
|
|
ADDL $0x04, R10
|
|
MOVL DX, 12(SP)
|
|
|
|
// emitCopy
|
|
two_byte_offset_match_nolit_encodeSnappyBlockAsm12B:
|
|
CMPL R10, $0x40
|
|
JBE two_byte_offset_short_match_nolit_encodeSnappyBlockAsm12B
|
|
MOVB $0xee, (CX)
|
|
MOVW SI, 1(CX)
|
|
LEAL -60(R10), R10
|
|
ADDQ $0x03, CX
|
|
JMP two_byte_offset_match_nolit_encodeSnappyBlockAsm12B
|
|
|
|
two_byte_offset_short_match_nolit_encodeSnappyBlockAsm12B:
|
|
MOVL R10, DI
|
|
SHLL $0x02, DI
|
|
CMPL R10, $0x0c
|
|
JAE emit_copy_three_match_nolit_encodeSnappyBlockAsm12B
|
|
CMPL SI, $0x00000800
|
|
JAE emit_copy_three_match_nolit_encodeSnappyBlockAsm12B
|
|
LEAL -15(DI), DI
|
|
MOVB SI, 1(CX)
|
|
SHRL $0x08, SI
|
|
SHLL $0x05, SI
|
|
ORL SI, DI
|
|
MOVB DI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeSnappyBlockAsm12B
|
|
|
|
emit_copy_three_match_nolit_encodeSnappyBlockAsm12B:
|
|
LEAL -2(DI), DI
|
|
MOVB DI, (CX)
|
|
MOVW SI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
|
|
match_nolit_emitcopy_end_encodeSnappyBlockAsm12B:
|
|
CMPL DX, 8(SP)
|
|
JAE emit_remainder_encodeSnappyBlockAsm12B
|
|
MOVQ -2(BX)(DX*1), DI
|
|
CMPQ CX, (SP)
|
|
JB match_nolit_dst_ok_encodeSnappyBlockAsm12B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeSnappyBlockAsm12B:
|
|
MOVQ $0x000000cf1bbcdcbb, R9
|
|
MOVQ DI, R8
|
|
SHRQ $0x10, DI
|
|
MOVQ DI, SI
|
|
SHLQ $0x18, R8
|
|
IMULQ R9, R8
|
|
SHRQ $0x34, R8
|
|
SHLQ $0x18, SI
|
|
IMULQ R9, SI
|
|
SHRQ $0x34, SI
|
|
LEAL -2(DX), R9
|
|
LEAQ (AX)(SI*4), R10
|
|
MOVL (R10), SI
|
|
MOVL R9, (AX)(R8*4)
|
|
MOVL DX, (R10)
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ match_nolit_loop_encodeSnappyBlockAsm12B
|
|
INCL DX
|
|
JMP search_loop_encodeSnappyBlockAsm12B
|
|
|
|
emit_remainder_encodeSnappyBlockAsm12B:
|
|
MOVQ src_len+32(FP), AX
|
|
SUBL 12(SP), AX
|
|
LEAQ 3(CX)(AX*1), AX
|
|
CMPQ AX, (SP)
|
|
JB emit_remainder_ok_encodeSnappyBlockAsm12B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeSnappyBlockAsm12B:
|
|
MOVQ src_len+32(FP), AX
|
|
MOVL 12(SP), DX
|
|
CMPL DX, AX
|
|
JEQ emit_literal_done_emit_remainder_encodeSnappyBlockAsm12B
|
|
MOVL AX, SI
|
|
MOVL AX, 12(SP)
|
|
LEAQ (BX)(DX*1), AX
|
|
SUBL DX, SI
|
|
LEAL -1(SI), DX
|
|
CMPL DX, $0x3c
|
|
JB one_byte_emit_remainder_encodeSnappyBlockAsm12B
|
|
CMPL DX, $0x00000100
|
|
JB two_bytes_emit_remainder_encodeSnappyBlockAsm12B
|
|
JB three_bytes_emit_remainder_encodeSnappyBlockAsm12B
|
|
|
|
three_bytes_emit_remainder_encodeSnappyBlockAsm12B:
|
|
MOVB $0xf4, (CX)
|
|
MOVW DX, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_emit_remainder_encodeSnappyBlockAsm12B
|
|
|
|
two_bytes_emit_remainder_encodeSnappyBlockAsm12B:
|
|
MOVB $0xf0, (CX)
|
|
MOVB DL, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL DX, $0x40
|
|
JB memmove_emit_remainder_encodeSnappyBlockAsm12B
|
|
JMP memmove_long_emit_remainder_encodeSnappyBlockAsm12B
|
|
|
|
one_byte_emit_remainder_encodeSnappyBlockAsm12B:
|
|
SHLB $0x02, DL
|
|
MOVB DL, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_emit_remainder_encodeSnappyBlockAsm12B:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveShort
|
|
CMPQ BX, $0x03
|
|
JB emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_1or2
|
|
JE emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_3
|
|
CMPQ BX, $0x08
|
|
JB emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_4through7
|
|
CMPQ BX, $0x10
|
|
JBE emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_8through16
|
|
CMPQ BX, $0x20
|
|
JBE emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_1or2:
|
|
MOVB (AX), SI
|
|
MOVB -1(AX)(BX*1), AL
|
|
MOVB SI, (CX)
|
|
MOVB AL, -1(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_3:
|
|
MOVW (AX), SI
|
|
MOVB 2(AX), AL
|
|
MOVW SI, (CX)
|
|
MOVB AL, 2(CX)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_4through7:
|
|
MOVL (AX), SI
|
|
MOVL -4(AX)(BX*1), AX
|
|
MOVL SI, (CX)
|
|
MOVL AX, -4(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_8through16:
|
|
MOVQ (AX), SI
|
|
MOVQ -8(AX)(BX*1), AX
|
|
MOVQ SI, (CX)
|
|
MOVQ AX, -8(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_17through32:
|
|
MOVOU (AX), X0
|
|
MOVOU -16(AX)(BX*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_33through64:
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
|
|
memmove_end_copy_emit_remainder_encodeSnappyBlockAsm12B:
|
|
MOVQ DX, CX
|
|
JMP emit_literal_done_emit_remainder_encodeSnappyBlockAsm12B
|
|
|
|
memmove_long_emit_remainder_encodeSnappyBlockAsm12B:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveLong
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVQ BX, DI
|
|
SHRQ $0x05, DI
|
|
MOVQ CX, SI
|
|
ANDL $0x0000001f, SI
|
|
MOVQ $0x00000040, R8
|
|
SUBQ SI, R8
|
|
DECQ DI
|
|
JA emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm12Blarge_forward_sse_loop_32
|
|
LEAQ -32(AX)(R8*1), SI
|
|
LEAQ -32(CX)(R8*1), R9
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm12Blarge_big_loop_back:
|
|
MOVOU (SI), X4
|
|
MOVOU 16(SI), X5
|
|
MOVOA X4, (R9)
|
|
MOVOA X5, 16(R9)
|
|
ADDQ $0x20, R9
|
|
ADDQ $0x20, SI
|
|
ADDQ $0x20, R8
|
|
DECQ DI
|
|
JNA emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm12Blarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm12Blarge_forward_sse_loop_32:
|
|
MOVOU -32(AX)(R8*1), X4
|
|
MOVOU -16(AX)(R8*1), X5
|
|
MOVOA X4, -32(CX)(R8*1)
|
|
MOVOA X5, -16(CX)(R8*1)
|
|
ADDQ $0x20, R8
|
|
CMPQ BX, R8
|
|
JAE emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm12Blarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
MOVQ DX, CX
|
|
|
|
emit_literal_done_emit_remainder_encodeSnappyBlockAsm12B:
|
|
MOVQ dst_base+0(FP), AX
|
|
SUBQ AX, CX
|
|
MOVQ CX, ret+56(FP)
|
|
RET
|
|
|
|
// func encodeSnappyBlockAsm10B(dst []byte, src []byte, tmp *[4096]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeSnappyBlockAsm10B(SB), $24-64
|
|
MOVQ tmp+48(FP), AX
|
|
MOVQ dst_base+0(FP), CX
|
|
MOVQ $0x00000020, DX
|
|
MOVQ AX, BX
|
|
PXOR X0, X0
|
|
|
|
zero_loop_encodeSnappyBlockAsm10B:
|
|
MOVOU X0, (BX)
|
|
MOVOU X0, 16(BX)
|
|
MOVOU X0, 32(BX)
|
|
MOVOU X0, 48(BX)
|
|
MOVOU X0, 64(BX)
|
|
MOVOU X0, 80(BX)
|
|
MOVOU X0, 96(BX)
|
|
MOVOU X0, 112(BX)
|
|
ADDQ $0x80, BX
|
|
DECQ DX
|
|
JNZ zero_loop_encodeSnappyBlockAsm10B
|
|
MOVL $0x00000000, 12(SP)
|
|
MOVQ src_len+32(FP), DX
|
|
LEAQ -9(DX), BX
|
|
LEAQ -8(DX), SI
|
|
MOVL SI, 8(SP)
|
|
SHRQ $0x05, DX
|
|
SUBL DX, BX
|
|
LEAQ (CX)(BX*1), BX
|
|
MOVQ BX, (SP)
|
|
MOVL $0x00000001, DX
|
|
MOVL DX, 16(SP)
|
|
MOVQ src_base+24(FP), BX
|
|
|
|
search_loop_encodeSnappyBlockAsm10B:
|
|
MOVL DX, SI
|
|
SUBL 12(SP), SI
|
|
SHRL $0x05, SI
|
|
LEAL 4(DX)(SI*1), SI
|
|
CMPL SI, 8(SP)
|
|
JAE emit_remainder_encodeSnappyBlockAsm10B
|
|
MOVQ (BX)(DX*1), DI
|
|
MOVL SI, 20(SP)
|
|
MOVQ $0x9e3779b1, R9
|
|
MOVQ DI, R10
|
|
MOVQ DI, R11
|
|
SHRQ $0x08, R11
|
|
SHLQ $0x20, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x36, R10
|
|
SHLQ $0x20, R11
|
|
IMULQ R9, R11
|
|
SHRQ $0x36, R11
|
|
MOVL (AX)(R10*4), SI
|
|
MOVL (AX)(R11*4), R8
|
|
MOVL DX, (AX)(R10*4)
|
|
LEAL 1(DX), R10
|
|
MOVL R10, (AX)(R11*4)
|
|
MOVQ DI, R10
|
|
SHRQ $0x10, R10
|
|
SHLQ $0x20, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x36, R10
|
|
MOVL DX, R9
|
|
SUBL 16(SP), R9
|
|
MOVL 1(BX)(R9*1), R11
|
|
MOVQ DI, R9
|
|
SHRQ $0x08, R9
|
|
CMPL R9, R11
|
|
JNE no_repeat_found_encodeSnappyBlockAsm10B
|
|
LEAL 1(DX), DI
|
|
MOVL 12(SP), SI
|
|
MOVL DI, R8
|
|
SUBL 16(SP), R8
|
|
JZ repeat_extend_back_end_encodeSnappyBlockAsm10B
|
|
|
|
repeat_extend_back_loop_encodeSnappyBlockAsm10B:
|
|
CMPL DI, SI
|
|
JBE repeat_extend_back_end_encodeSnappyBlockAsm10B
|
|
MOVB -1(BX)(R8*1), R9
|
|
MOVB -1(BX)(DI*1), R10
|
|
CMPB R9, R10
|
|
JNE repeat_extend_back_end_encodeSnappyBlockAsm10B
|
|
LEAL -1(DI), DI
|
|
DECL R8
|
|
JNZ repeat_extend_back_loop_encodeSnappyBlockAsm10B
|
|
|
|
repeat_extend_back_end_encodeSnappyBlockAsm10B:
|
|
MOVL DI, SI
|
|
SUBL 12(SP), SI
|
|
LEAQ 3(CX)(SI*1), SI
|
|
CMPQ SI, (SP)
|
|
JB repeat_dst_size_check_encodeSnappyBlockAsm10B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
repeat_dst_size_check_encodeSnappyBlockAsm10B:
|
|
MOVL 12(SP), SI
|
|
CMPL SI, DI
|
|
JEQ emit_literal_done_repeat_emit_encodeSnappyBlockAsm10B
|
|
MOVL DI, R8
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(SI*1), R9
|
|
SUBL SI, R8
|
|
LEAL -1(R8), SI
|
|
CMPL SI, $0x3c
|
|
JB one_byte_repeat_emit_encodeSnappyBlockAsm10B
|
|
CMPL SI, $0x00000100
|
|
JB two_bytes_repeat_emit_encodeSnappyBlockAsm10B
|
|
JB three_bytes_repeat_emit_encodeSnappyBlockAsm10B
|
|
|
|
three_bytes_repeat_emit_encodeSnappyBlockAsm10B:
|
|
MOVB $0xf4, (CX)
|
|
MOVW SI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_repeat_emit_encodeSnappyBlockAsm10B
|
|
|
|
two_bytes_repeat_emit_encodeSnappyBlockAsm10B:
|
|
MOVB $0xf0, (CX)
|
|
MOVB SI, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL SI, $0x40
|
|
JB memmove_repeat_emit_encodeSnappyBlockAsm10B
|
|
JMP memmove_long_repeat_emit_encodeSnappyBlockAsm10B
|
|
|
|
one_byte_repeat_emit_encodeSnappyBlockAsm10B:
|
|
SHLB $0x02, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_repeat_emit_encodeSnappyBlockAsm10B:
|
|
LEAQ (CX)(R8*1), SI
|
|
|
|
// genMemMoveShort
|
|
CMPQ R8, $0x08
|
|
JBE emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_8
|
|
CMPQ R8, $0x10
|
|
JBE emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_8through16
|
|
CMPQ R8, $0x20
|
|
JBE emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_17through32
|
|
JMP emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_8:
|
|
MOVQ (R9), R10
|
|
MOVQ R10, (CX)
|
|
JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm10B
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_8through16:
|
|
MOVQ (R9), R10
|
|
MOVQ -8(R9)(R8*1), R9
|
|
MOVQ R10, (CX)
|
|
MOVQ R9, -8(CX)(R8*1)
|
|
JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm10B
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_17through32:
|
|
MOVOU (R9), X0
|
|
MOVOU -16(R9)(R8*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(R8*1)
|
|
JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm10B
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_33through64:
|
|
MOVOU (R9), X0
|
|
MOVOU 16(R9), X1
|
|
MOVOU -32(R9)(R8*1), X2
|
|
MOVOU -16(R9)(R8*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R8*1)
|
|
MOVOU X3, -16(CX)(R8*1)
|
|
|
|
memmove_end_copy_repeat_emit_encodeSnappyBlockAsm10B:
|
|
MOVQ SI, CX
|
|
JMP emit_literal_done_repeat_emit_encodeSnappyBlockAsm10B
|
|
|
|
memmove_long_repeat_emit_encodeSnappyBlockAsm10B:
|
|
LEAQ (CX)(R8*1), SI
|
|
|
|
// genMemMoveLong
|
|
MOVOU (R9), X0
|
|
MOVOU 16(R9), X1
|
|
MOVOU -32(R9)(R8*1), X2
|
|
MOVOU -16(R9)(R8*1), X3
|
|
MOVQ R8, R11
|
|
SHRQ $0x05, R11
|
|
MOVQ CX, R10
|
|
ANDL $0x0000001f, R10
|
|
MOVQ $0x00000040, R12
|
|
SUBQ R10, R12
|
|
DECQ R11
|
|
JA emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm10Blarge_forward_sse_loop_32
|
|
LEAQ -32(R9)(R12*1), R10
|
|
LEAQ -32(CX)(R12*1), R13
|
|
|
|
emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm10Blarge_big_loop_back:
|
|
MOVOU (R10), X4
|
|
MOVOU 16(R10), X5
|
|
MOVOA X4, (R13)
|
|
MOVOA X5, 16(R13)
|
|
ADDQ $0x20, R13
|
|
ADDQ $0x20, R10
|
|
ADDQ $0x20, R12
|
|
DECQ R11
|
|
JNA emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm10Blarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm10Blarge_forward_sse_loop_32:
|
|
MOVOU -32(R9)(R12*1), X4
|
|
MOVOU -16(R9)(R12*1), X5
|
|
MOVOA X4, -32(CX)(R12*1)
|
|
MOVOA X5, -16(CX)(R12*1)
|
|
ADDQ $0x20, R12
|
|
CMPQ R8, R12
|
|
JAE emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm10Blarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R8*1)
|
|
MOVOU X3, -16(CX)(R8*1)
|
|
MOVQ SI, CX
|
|
|
|
emit_literal_done_repeat_emit_encodeSnappyBlockAsm10B:
|
|
ADDL $0x05, DX
|
|
MOVL DX, SI
|
|
SUBL 16(SP), SI
|
|
MOVQ src_len+32(FP), R8
|
|
SUBL DX, R8
|
|
LEAQ (BX)(DX*1), R9
|
|
LEAQ (BX)(SI*1), SI
|
|
|
|
// matchLen
|
|
XORL R11, R11
|
|
|
|
matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm10B:
|
|
CMPL R8, $0x10
|
|
JB matchlen_match8_repeat_extend_encodeSnappyBlockAsm10B
|
|
MOVQ (R9)(R11*1), R10
|
|
MOVQ 8(R9)(R11*1), R12
|
|
XORQ (SI)(R11*1), R10
|
|
JNZ matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm10B
|
|
XORQ 8(SI)(R11*1), R12
|
|
JNZ matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm10B
|
|
LEAL -16(R8), R8
|
|
LEAL 16(R11), R11
|
|
JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm10B
|
|
|
|
matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm10B:
|
|
TZCNTQ R12, R12
|
|
SARQ $0x03, R12
|
|
LEAL 8(R11)(R12*1), R11
|
|
JMP repeat_extend_forward_end_encodeSnappyBlockAsm10B
|
|
|
|
matchlen_match8_repeat_extend_encodeSnappyBlockAsm10B:
|
|
CMPL R8, $0x08
|
|
JB matchlen_match4_repeat_extend_encodeSnappyBlockAsm10B
|
|
MOVQ (R9)(R11*1), R10
|
|
XORQ (SI)(R11*1), R10
|
|
JNZ matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm10B
|
|
LEAL -8(R8), R8
|
|
LEAL 8(R11), R11
|
|
JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm10B
|
|
|
|
matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm10B:
|
|
TZCNTQ R10, R10
|
|
SARQ $0x03, R10
|
|
LEAL (R11)(R10*1), R11
|
|
JMP repeat_extend_forward_end_encodeSnappyBlockAsm10B
|
|
|
|
matchlen_match4_repeat_extend_encodeSnappyBlockAsm10B:
|
|
CMPL R8, $0x04
|
|
JB matchlen_match2_repeat_extend_encodeSnappyBlockAsm10B
|
|
MOVL (R9)(R11*1), R10
|
|
CMPL (SI)(R11*1), R10
|
|
JNE matchlen_match2_repeat_extend_encodeSnappyBlockAsm10B
|
|
LEAL -4(R8), R8
|
|
LEAL 4(R11), R11
|
|
|
|
matchlen_match2_repeat_extend_encodeSnappyBlockAsm10B:
|
|
CMPL R8, $0x01
|
|
JE matchlen_match1_repeat_extend_encodeSnappyBlockAsm10B
|
|
JB repeat_extend_forward_end_encodeSnappyBlockAsm10B
|
|
MOVW (R9)(R11*1), R10
|
|
CMPW (SI)(R11*1), R10
|
|
JNE matchlen_match1_repeat_extend_encodeSnappyBlockAsm10B
|
|
LEAL 2(R11), R11
|
|
SUBL $0x02, R8
|
|
JZ repeat_extend_forward_end_encodeSnappyBlockAsm10B
|
|
|
|
matchlen_match1_repeat_extend_encodeSnappyBlockAsm10B:
|
|
MOVB (R9)(R11*1), R10
|
|
CMPB (SI)(R11*1), R10
|
|
JNE repeat_extend_forward_end_encodeSnappyBlockAsm10B
|
|
LEAL 1(R11), R11
|
|
|
|
repeat_extend_forward_end_encodeSnappyBlockAsm10B:
|
|
ADDL R11, DX
|
|
MOVL DX, SI
|
|
SUBL DI, SI
|
|
MOVL 16(SP), DI
|
|
|
|
// emitCopy
|
|
two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm10B:
|
|
CMPL SI, $0x40
|
|
JBE two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm10B
|
|
MOVB $0xee, (CX)
|
|
MOVW DI, 1(CX)
|
|
LEAL -60(SI), SI
|
|
ADDQ $0x03, CX
|
|
JMP two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm10B
|
|
|
|
two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm10B:
|
|
MOVL SI, R8
|
|
SHLL $0x02, R8
|
|
CMPL SI, $0x0c
|
|
JAE emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm10B
|
|
CMPL DI, $0x00000800
|
|
JAE emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm10B
|
|
LEAL -15(R8), R8
|
|
MOVB DI, 1(CX)
|
|
SHRL $0x08, DI
|
|
SHLL $0x05, DI
|
|
ORL DI, R8
|
|
MOVB R8, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeSnappyBlockAsm10B
|
|
|
|
emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm10B:
|
|
LEAL -2(R8), R8
|
|
MOVB R8, (CX)
|
|
MOVW DI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
|
|
repeat_end_emit_encodeSnappyBlockAsm10B:
|
|
MOVL DX, 12(SP)
|
|
JMP search_loop_encodeSnappyBlockAsm10B
|
|
|
|
no_repeat_found_encodeSnappyBlockAsm10B:
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ candidate_match_encodeSnappyBlockAsm10B
|
|
SHRQ $0x08, DI
|
|
MOVL (AX)(R10*4), SI
|
|
LEAL 2(DX), R9
|
|
CMPL (BX)(R8*1), DI
|
|
JEQ candidate2_match_encodeSnappyBlockAsm10B
|
|
MOVL R9, (AX)(R10*4)
|
|
SHRQ $0x08, DI
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ candidate3_match_encodeSnappyBlockAsm10B
|
|
MOVL 20(SP), DX
|
|
JMP search_loop_encodeSnappyBlockAsm10B
|
|
|
|
candidate3_match_encodeSnappyBlockAsm10B:
|
|
ADDL $0x02, DX
|
|
JMP candidate_match_encodeSnappyBlockAsm10B
|
|
|
|
candidate2_match_encodeSnappyBlockAsm10B:
|
|
MOVL R9, (AX)(R10*4)
|
|
INCL DX
|
|
MOVL R8, SI
|
|
|
|
candidate_match_encodeSnappyBlockAsm10B:
|
|
MOVL 12(SP), DI
|
|
TESTL SI, SI
|
|
JZ match_extend_back_end_encodeSnappyBlockAsm10B
|
|
|
|
match_extend_back_loop_encodeSnappyBlockAsm10B:
|
|
CMPL DX, DI
|
|
JBE match_extend_back_end_encodeSnappyBlockAsm10B
|
|
MOVB -1(BX)(SI*1), R8
|
|
MOVB -1(BX)(DX*1), R9
|
|
CMPB R8, R9
|
|
JNE match_extend_back_end_encodeSnappyBlockAsm10B
|
|
LEAL -1(DX), DX
|
|
DECL SI
|
|
JZ match_extend_back_end_encodeSnappyBlockAsm10B
|
|
JMP match_extend_back_loop_encodeSnappyBlockAsm10B
|
|
|
|
match_extend_back_end_encodeSnappyBlockAsm10B:
|
|
MOVL DX, DI
|
|
SUBL 12(SP), DI
|
|
LEAQ 3(CX)(DI*1), DI
|
|
CMPQ DI, (SP)
|
|
JB match_dst_size_check_encodeSnappyBlockAsm10B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeSnappyBlockAsm10B:
|
|
MOVL DX, DI
|
|
MOVL 12(SP), R8
|
|
CMPL R8, DI
|
|
JEQ emit_literal_done_match_emit_encodeSnappyBlockAsm10B
|
|
MOVL DI, R9
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(R8*1), DI
|
|
SUBL R8, R9
|
|
LEAL -1(R9), R8
|
|
CMPL R8, $0x3c
|
|
JB one_byte_match_emit_encodeSnappyBlockAsm10B
|
|
CMPL R8, $0x00000100
|
|
JB two_bytes_match_emit_encodeSnappyBlockAsm10B
|
|
JB three_bytes_match_emit_encodeSnappyBlockAsm10B
|
|
|
|
three_bytes_match_emit_encodeSnappyBlockAsm10B:
|
|
MOVB $0xf4, (CX)
|
|
MOVW R8, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_match_emit_encodeSnappyBlockAsm10B
|
|
|
|
two_bytes_match_emit_encodeSnappyBlockAsm10B:
|
|
MOVB $0xf0, (CX)
|
|
MOVB R8, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL R8, $0x40
|
|
JB memmove_match_emit_encodeSnappyBlockAsm10B
|
|
JMP memmove_long_match_emit_encodeSnappyBlockAsm10B
|
|
|
|
one_byte_match_emit_encodeSnappyBlockAsm10B:
|
|
SHLB $0x02, R8
|
|
MOVB R8, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_match_emit_encodeSnappyBlockAsm10B:
|
|
LEAQ (CX)(R9*1), R8
|
|
|
|
// genMemMoveShort
|
|
CMPQ R9, $0x08
|
|
JBE emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_8
|
|
CMPQ R9, $0x10
|
|
JBE emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_8through16
|
|
CMPQ R9, $0x20
|
|
JBE emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_8:
|
|
MOVQ (DI), R10
|
|
MOVQ R10, (CX)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm10B
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_8through16:
|
|
MOVQ (DI), R10
|
|
MOVQ -8(DI)(R9*1), DI
|
|
MOVQ R10, (CX)
|
|
MOVQ DI, -8(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm10B
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_17through32:
|
|
MOVOU (DI), X0
|
|
MOVOU -16(DI)(R9*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm10B
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_33through64:
|
|
MOVOU (DI), X0
|
|
MOVOU 16(DI), X1
|
|
MOVOU -32(DI)(R9*1), X2
|
|
MOVOU -16(DI)(R9*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
|
|
memmove_end_copy_match_emit_encodeSnappyBlockAsm10B:
|
|
MOVQ R8, CX
|
|
JMP emit_literal_done_match_emit_encodeSnappyBlockAsm10B
|
|
|
|
memmove_long_match_emit_encodeSnappyBlockAsm10B:
|
|
LEAQ (CX)(R9*1), R8
|
|
|
|
// genMemMoveLong
|
|
MOVOU (DI), X0
|
|
MOVOU 16(DI), X1
|
|
MOVOU -32(DI)(R9*1), X2
|
|
MOVOU -16(DI)(R9*1), X3
|
|
MOVQ R9, R11
|
|
SHRQ $0x05, R11
|
|
MOVQ CX, R10
|
|
ANDL $0x0000001f, R10
|
|
MOVQ $0x00000040, R12
|
|
SUBQ R10, R12
|
|
DECQ R11
|
|
JA emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm10Blarge_forward_sse_loop_32
|
|
LEAQ -32(DI)(R12*1), R10
|
|
LEAQ -32(CX)(R12*1), R13
|
|
|
|
emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm10Blarge_big_loop_back:
|
|
MOVOU (R10), X4
|
|
MOVOU 16(R10), X5
|
|
MOVOA X4, (R13)
|
|
MOVOA X5, 16(R13)
|
|
ADDQ $0x20, R13
|
|
ADDQ $0x20, R10
|
|
ADDQ $0x20, R12
|
|
DECQ R11
|
|
JNA emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm10Blarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm10Blarge_forward_sse_loop_32:
|
|
MOVOU -32(DI)(R12*1), X4
|
|
MOVOU -16(DI)(R12*1), X5
|
|
MOVOA X4, -32(CX)(R12*1)
|
|
MOVOA X5, -16(CX)(R12*1)
|
|
ADDQ $0x20, R12
|
|
CMPQ R9, R12
|
|
JAE emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm10Blarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
MOVQ R8, CX
|
|
|
|
emit_literal_done_match_emit_encodeSnappyBlockAsm10B:
|
|
match_nolit_loop_encodeSnappyBlockAsm10B:
|
|
MOVL DX, DI
|
|
SUBL SI, DI
|
|
MOVL DI, 16(SP)
|
|
ADDL $0x04, DX
|
|
ADDL $0x04, SI
|
|
MOVQ src_len+32(FP), DI
|
|
SUBL DX, DI
|
|
LEAQ (BX)(DX*1), R8
|
|
LEAQ (BX)(SI*1), SI
|
|
|
|
// matchLen
|
|
XORL R10, R10
|
|
|
|
matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm10B:
|
|
CMPL DI, $0x10
|
|
JB matchlen_match8_match_nolit_encodeSnappyBlockAsm10B
|
|
MOVQ (R8)(R10*1), R9
|
|
MOVQ 8(R8)(R10*1), R11
|
|
XORQ (SI)(R10*1), R9
|
|
JNZ matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm10B
|
|
XORQ 8(SI)(R10*1), R11
|
|
JNZ matchlen_bsf_16match_nolit_encodeSnappyBlockAsm10B
|
|
LEAL -16(DI), DI
|
|
LEAL 16(R10), R10
|
|
JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm10B
|
|
|
|
matchlen_bsf_16match_nolit_encodeSnappyBlockAsm10B:
|
|
TZCNTQ R11, R11
|
|
SARQ $0x03, R11
|
|
LEAL 8(R10)(R11*1), R10
|
|
JMP match_nolit_end_encodeSnappyBlockAsm10B
|
|
|
|
matchlen_match8_match_nolit_encodeSnappyBlockAsm10B:
|
|
CMPL DI, $0x08
|
|
JB matchlen_match4_match_nolit_encodeSnappyBlockAsm10B
|
|
MOVQ (R8)(R10*1), R9
|
|
XORQ (SI)(R10*1), R9
|
|
JNZ matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm10B
|
|
LEAL -8(DI), DI
|
|
LEAL 8(R10), R10
|
|
JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm10B
|
|
|
|
matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm10B:
|
|
TZCNTQ R9, R9
|
|
SARQ $0x03, R9
|
|
LEAL (R10)(R9*1), R10
|
|
JMP match_nolit_end_encodeSnappyBlockAsm10B
|
|
|
|
matchlen_match4_match_nolit_encodeSnappyBlockAsm10B:
|
|
CMPL DI, $0x04
|
|
JB matchlen_match2_match_nolit_encodeSnappyBlockAsm10B
|
|
MOVL (R8)(R10*1), R9
|
|
CMPL (SI)(R10*1), R9
|
|
JNE matchlen_match2_match_nolit_encodeSnappyBlockAsm10B
|
|
LEAL -4(DI), DI
|
|
LEAL 4(R10), R10
|
|
|
|
matchlen_match2_match_nolit_encodeSnappyBlockAsm10B:
|
|
CMPL DI, $0x01
|
|
JE matchlen_match1_match_nolit_encodeSnappyBlockAsm10B
|
|
JB match_nolit_end_encodeSnappyBlockAsm10B
|
|
MOVW (R8)(R10*1), R9
|
|
CMPW (SI)(R10*1), R9
|
|
JNE matchlen_match1_match_nolit_encodeSnappyBlockAsm10B
|
|
LEAL 2(R10), R10
|
|
SUBL $0x02, DI
|
|
JZ match_nolit_end_encodeSnappyBlockAsm10B
|
|
|
|
matchlen_match1_match_nolit_encodeSnappyBlockAsm10B:
|
|
MOVB (R8)(R10*1), R9
|
|
CMPB (SI)(R10*1), R9
|
|
JNE match_nolit_end_encodeSnappyBlockAsm10B
|
|
LEAL 1(R10), R10
|
|
|
|
match_nolit_end_encodeSnappyBlockAsm10B:
|
|
ADDL R10, DX
|
|
MOVL 16(SP), SI
|
|
ADDL $0x04, R10
|
|
MOVL DX, 12(SP)
|
|
|
|
// emitCopy
|
|
two_byte_offset_match_nolit_encodeSnappyBlockAsm10B:
|
|
CMPL R10, $0x40
|
|
JBE two_byte_offset_short_match_nolit_encodeSnappyBlockAsm10B
|
|
MOVB $0xee, (CX)
|
|
MOVW SI, 1(CX)
|
|
LEAL -60(R10), R10
|
|
ADDQ $0x03, CX
|
|
JMP two_byte_offset_match_nolit_encodeSnappyBlockAsm10B
|
|
|
|
two_byte_offset_short_match_nolit_encodeSnappyBlockAsm10B:
|
|
MOVL R10, DI
|
|
SHLL $0x02, DI
|
|
CMPL R10, $0x0c
|
|
JAE emit_copy_three_match_nolit_encodeSnappyBlockAsm10B
|
|
CMPL SI, $0x00000800
|
|
JAE emit_copy_three_match_nolit_encodeSnappyBlockAsm10B
|
|
LEAL -15(DI), DI
|
|
MOVB SI, 1(CX)
|
|
SHRL $0x08, SI
|
|
SHLL $0x05, SI
|
|
ORL SI, DI
|
|
MOVB DI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeSnappyBlockAsm10B
|
|
|
|
emit_copy_three_match_nolit_encodeSnappyBlockAsm10B:
|
|
LEAL -2(DI), DI
|
|
MOVB DI, (CX)
|
|
MOVW SI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
|
|
match_nolit_emitcopy_end_encodeSnappyBlockAsm10B:
|
|
CMPL DX, 8(SP)
|
|
JAE emit_remainder_encodeSnappyBlockAsm10B
|
|
MOVQ -2(BX)(DX*1), DI
|
|
CMPQ CX, (SP)
|
|
JB match_nolit_dst_ok_encodeSnappyBlockAsm10B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeSnappyBlockAsm10B:
|
|
MOVQ $0x9e3779b1, R9
|
|
MOVQ DI, R8
|
|
SHRQ $0x10, DI
|
|
MOVQ DI, SI
|
|
SHLQ $0x20, R8
|
|
IMULQ R9, R8
|
|
SHRQ $0x36, R8
|
|
SHLQ $0x20, SI
|
|
IMULQ R9, SI
|
|
SHRQ $0x36, SI
|
|
LEAL -2(DX), R9
|
|
LEAQ (AX)(SI*4), R10
|
|
MOVL (R10), SI
|
|
MOVL R9, (AX)(R8*4)
|
|
MOVL DX, (R10)
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ match_nolit_loop_encodeSnappyBlockAsm10B
|
|
INCL DX
|
|
JMP search_loop_encodeSnappyBlockAsm10B
|
|
|
|
emit_remainder_encodeSnappyBlockAsm10B:
|
|
MOVQ src_len+32(FP), AX
|
|
SUBL 12(SP), AX
|
|
LEAQ 3(CX)(AX*1), AX
|
|
CMPQ AX, (SP)
|
|
JB emit_remainder_ok_encodeSnappyBlockAsm10B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeSnappyBlockAsm10B:
|
|
MOVQ src_len+32(FP), AX
|
|
MOVL 12(SP), DX
|
|
CMPL DX, AX
|
|
JEQ emit_literal_done_emit_remainder_encodeSnappyBlockAsm10B
|
|
MOVL AX, SI
|
|
MOVL AX, 12(SP)
|
|
LEAQ (BX)(DX*1), AX
|
|
SUBL DX, SI
|
|
LEAL -1(SI), DX
|
|
CMPL DX, $0x3c
|
|
JB one_byte_emit_remainder_encodeSnappyBlockAsm10B
|
|
CMPL DX, $0x00000100
|
|
JB two_bytes_emit_remainder_encodeSnappyBlockAsm10B
|
|
JB three_bytes_emit_remainder_encodeSnappyBlockAsm10B
|
|
|
|
three_bytes_emit_remainder_encodeSnappyBlockAsm10B:
|
|
MOVB $0xf4, (CX)
|
|
MOVW DX, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_emit_remainder_encodeSnappyBlockAsm10B
|
|
|
|
two_bytes_emit_remainder_encodeSnappyBlockAsm10B:
|
|
MOVB $0xf0, (CX)
|
|
MOVB DL, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL DX, $0x40
|
|
JB memmove_emit_remainder_encodeSnappyBlockAsm10B
|
|
JMP memmove_long_emit_remainder_encodeSnappyBlockAsm10B
|
|
|
|
one_byte_emit_remainder_encodeSnappyBlockAsm10B:
|
|
SHLB $0x02, DL
|
|
MOVB DL, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_emit_remainder_encodeSnappyBlockAsm10B:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveShort
|
|
CMPQ BX, $0x03
|
|
JB emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_1or2
|
|
JE emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_3
|
|
CMPQ BX, $0x08
|
|
JB emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_4through7
|
|
CMPQ BX, $0x10
|
|
JBE emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_8through16
|
|
CMPQ BX, $0x20
|
|
JBE emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_1or2:
|
|
MOVB (AX), SI
|
|
MOVB -1(AX)(BX*1), AL
|
|
MOVB SI, (CX)
|
|
MOVB AL, -1(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_3:
|
|
MOVW (AX), SI
|
|
MOVB 2(AX), AL
|
|
MOVW SI, (CX)
|
|
MOVB AL, 2(CX)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_4through7:
|
|
MOVL (AX), SI
|
|
MOVL -4(AX)(BX*1), AX
|
|
MOVL SI, (CX)
|
|
MOVL AX, -4(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_8through16:
|
|
MOVQ (AX), SI
|
|
MOVQ -8(AX)(BX*1), AX
|
|
MOVQ SI, (CX)
|
|
MOVQ AX, -8(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_17through32:
|
|
MOVOU (AX), X0
|
|
MOVOU -16(AX)(BX*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_33through64:
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
|
|
memmove_end_copy_emit_remainder_encodeSnappyBlockAsm10B:
|
|
MOVQ DX, CX
|
|
JMP emit_literal_done_emit_remainder_encodeSnappyBlockAsm10B
|
|
|
|
memmove_long_emit_remainder_encodeSnappyBlockAsm10B:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveLong
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVQ BX, DI
|
|
SHRQ $0x05, DI
|
|
MOVQ CX, SI
|
|
ANDL $0x0000001f, SI
|
|
MOVQ $0x00000040, R8
|
|
SUBQ SI, R8
|
|
DECQ DI
|
|
JA emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm10Blarge_forward_sse_loop_32
|
|
LEAQ -32(AX)(R8*1), SI
|
|
LEAQ -32(CX)(R8*1), R9
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm10Blarge_big_loop_back:
|
|
MOVOU (SI), X4
|
|
MOVOU 16(SI), X5
|
|
MOVOA X4, (R9)
|
|
MOVOA X5, 16(R9)
|
|
ADDQ $0x20, R9
|
|
ADDQ $0x20, SI
|
|
ADDQ $0x20, R8
|
|
DECQ DI
|
|
JNA emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm10Blarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm10Blarge_forward_sse_loop_32:
|
|
MOVOU -32(AX)(R8*1), X4
|
|
MOVOU -16(AX)(R8*1), X5
|
|
MOVOA X4, -32(CX)(R8*1)
|
|
MOVOA X5, -16(CX)(R8*1)
|
|
ADDQ $0x20, R8
|
|
CMPQ BX, R8
|
|
JAE emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm10Blarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
MOVQ DX, CX
|
|
|
|
emit_literal_done_emit_remainder_encodeSnappyBlockAsm10B:
|
|
MOVQ dst_base+0(FP), AX
|
|
SUBQ AX, CX
|
|
MOVQ CX, ret+56(FP)
|
|
RET
|
|
|
|
// func encodeSnappyBlockAsm8B(dst []byte, src []byte, tmp *[1024]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeSnappyBlockAsm8B(SB), $24-64
|
|
MOVQ tmp+48(FP), AX
|
|
MOVQ dst_base+0(FP), CX
|
|
MOVQ $0x00000008, DX
|
|
MOVQ AX, BX
|
|
PXOR X0, X0
|
|
|
|
zero_loop_encodeSnappyBlockAsm8B:
|
|
MOVOU X0, (BX)
|
|
MOVOU X0, 16(BX)
|
|
MOVOU X0, 32(BX)
|
|
MOVOU X0, 48(BX)
|
|
MOVOU X0, 64(BX)
|
|
MOVOU X0, 80(BX)
|
|
MOVOU X0, 96(BX)
|
|
MOVOU X0, 112(BX)
|
|
ADDQ $0x80, BX
|
|
DECQ DX
|
|
JNZ zero_loop_encodeSnappyBlockAsm8B
|
|
MOVL $0x00000000, 12(SP)
|
|
MOVQ src_len+32(FP), DX
|
|
LEAQ -9(DX), BX
|
|
LEAQ -8(DX), SI
|
|
MOVL SI, 8(SP)
|
|
SHRQ $0x05, DX
|
|
SUBL DX, BX
|
|
LEAQ (CX)(BX*1), BX
|
|
MOVQ BX, (SP)
|
|
MOVL $0x00000001, DX
|
|
MOVL DX, 16(SP)
|
|
MOVQ src_base+24(FP), BX
|
|
|
|
search_loop_encodeSnappyBlockAsm8B:
|
|
MOVL DX, SI
|
|
SUBL 12(SP), SI
|
|
SHRL $0x04, SI
|
|
LEAL 4(DX)(SI*1), SI
|
|
CMPL SI, 8(SP)
|
|
JAE emit_remainder_encodeSnappyBlockAsm8B
|
|
MOVQ (BX)(DX*1), DI
|
|
MOVL SI, 20(SP)
|
|
MOVQ $0x9e3779b1, R9
|
|
MOVQ DI, R10
|
|
MOVQ DI, R11
|
|
SHRQ $0x08, R11
|
|
SHLQ $0x20, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x38, R10
|
|
SHLQ $0x20, R11
|
|
IMULQ R9, R11
|
|
SHRQ $0x38, R11
|
|
MOVL (AX)(R10*4), SI
|
|
MOVL (AX)(R11*4), R8
|
|
MOVL DX, (AX)(R10*4)
|
|
LEAL 1(DX), R10
|
|
MOVL R10, (AX)(R11*4)
|
|
MOVQ DI, R10
|
|
SHRQ $0x10, R10
|
|
SHLQ $0x20, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x38, R10
|
|
MOVL DX, R9
|
|
SUBL 16(SP), R9
|
|
MOVL 1(BX)(R9*1), R11
|
|
MOVQ DI, R9
|
|
SHRQ $0x08, R9
|
|
CMPL R9, R11
|
|
JNE no_repeat_found_encodeSnappyBlockAsm8B
|
|
LEAL 1(DX), DI
|
|
MOVL 12(SP), SI
|
|
MOVL DI, R8
|
|
SUBL 16(SP), R8
|
|
JZ repeat_extend_back_end_encodeSnappyBlockAsm8B
|
|
|
|
repeat_extend_back_loop_encodeSnappyBlockAsm8B:
|
|
CMPL DI, SI
|
|
JBE repeat_extend_back_end_encodeSnappyBlockAsm8B
|
|
MOVB -1(BX)(R8*1), R9
|
|
MOVB -1(BX)(DI*1), R10
|
|
CMPB R9, R10
|
|
JNE repeat_extend_back_end_encodeSnappyBlockAsm8B
|
|
LEAL -1(DI), DI
|
|
DECL R8
|
|
JNZ repeat_extend_back_loop_encodeSnappyBlockAsm8B
|
|
|
|
repeat_extend_back_end_encodeSnappyBlockAsm8B:
|
|
MOVL DI, SI
|
|
SUBL 12(SP), SI
|
|
LEAQ 3(CX)(SI*1), SI
|
|
CMPQ SI, (SP)
|
|
JB repeat_dst_size_check_encodeSnappyBlockAsm8B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
repeat_dst_size_check_encodeSnappyBlockAsm8B:
|
|
MOVL 12(SP), SI
|
|
CMPL SI, DI
|
|
JEQ emit_literal_done_repeat_emit_encodeSnappyBlockAsm8B
|
|
MOVL DI, R8
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(SI*1), R9
|
|
SUBL SI, R8
|
|
LEAL -1(R8), SI
|
|
CMPL SI, $0x3c
|
|
JB one_byte_repeat_emit_encodeSnappyBlockAsm8B
|
|
CMPL SI, $0x00000100
|
|
JB two_bytes_repeat_emit_encodeSnappyBlockAsm8B
|
|
JB three_bytes_repeat_emit_encodeSnappyBlockAsm8B
|
|
|
|
three_bytes_repeat_emit_encodeSnappyBlockAsm8B:
|
|
MOVB $0xf4, (CX)
|
|
MOVW SI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_repeat_emit_encodeSnappyBlockAsm8B
|
|
|
|
two_bytes_repeat_emit_encodeSnappyBlockAsm8B:
|
|
MOVB $0xf0, (CX)
|
|
MOVB SI, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL SI, $0x40
|
|
JB memmove_repeat_emit_encodeSnappyBlockAsm8B
|
|
JMP memmove_long_repeat_emit_encodeSnappyBlockAsm8B
|
|
|
|
one_byte_repeat_emit_encodeSnappyBlockAsm8B:
|
|
SHLB $0x02, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_repeat_emit_encodeSnappyBlockAsm8B:
|
|
LEAQ (CX)(R8*1), SI
|
|
|
|
// genMemMoveShort
|
|
CMPQ R8, $0x08
|
|
JBE emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_8
|
|
CMPQ R8, $0x10
|
|
JBE emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_8through16
|
|
CMPQ R8, $0x20
|
|
JBE emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_17through32
|
|
JMP emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_8:
|
|
MOVQ (R9), R10
|
|
MOVQ R10, (CX)
|
|
JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm8B
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_8through16:
|
|
MOVQ (R9), R10
|
|
MOVQ -8(R9)(R8*1), R9
|
|
MOVQ R10, (CX)
|
|
MOVQ R9, -8(CX)(R8*1)
|
|
JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm8B
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_17through32:
|
|
MOVOU (R9), X0
|
|
MOVOU -16(R9)(R8*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(R8*1)
|
|
JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm8B
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_33through64:
|
|
MOVOU (R9), X0
|
|
MOVOU 16(R9), X1
|
|
MOVOU -32(R9)(R8*1), X2
|
|
MOVOU -16(R9)(R8*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R8*1)
|
|
MOVOU X3, -16(CX)(R8*1)
|
|
|
|
memmove_end_copy_repeat_emit_encodeSnappyBlockAsm8B:
|
|
MOVQ SI, CX
|
|
JMP emit_literal_done_repeat_emit_encodeSnappyBlockAsm8B
|
|
|
|
memmove_long_repeat_emit_encodeSnappyBlockAsm8B:
|
|
LEAQ (CX)(R8*1), SI
|
|
|
|
// genMemMoveLong
|
|
MOVOU (R9), X0
|
|
MOVOU 16(R9), X1
|
|
MOVOU -32(R9)(R8*1), X2
|
|
MOVOU -16(R9)(R8*1), X3
|
|
MOVQ R8, R11
|
|
SHRQ $0x05, R11
|
|
MOVQ CX, R10
|
|
ANDL $0x0000001f, R10
|
|
MOVQ $0x00000040, R12
|
|
SUBQ R10, R12
|
|
DECQ R11
|
|
JA emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm8Blarge_forward_sse_loop_32
|
|
LEAQ -32(R9)(R12*1), R10
|
|
LEAQ -32(CX)(R12*1), R13
|
|
|
|
emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm8Blarge_big_loop_back:
|
|
MOVOU (R10), X4
|
|
MOVOU 16(R10), X5
|
|
MOVOA X4, (R13)
|
|
MOVOA X5, 16(R13)
|
|
ADDQ $0x20, R13
|
|
ADDQ $0x20, R10
|
|
ADDQ $0x20, R12
|
|
DECQ R11
|
|
JNA emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm8Blarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm8Blarge_forward_sse_loop_32:
|
|
MOVOU -32(R9)(R12*1), X4
|
|
MOVOU -16(R9)(R12*1), X5
|
|
MOVOA X4, -32(CX)(R12*1)
|
|
MOVOA X5, -16(CX)(R12*1)
|
|
ADDQ $0x20, R12
|
|
CMPQ R8, R12
|
|
JAE emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm8Blarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R8*1)
|
|
MOVOU X3, -16(CX)(R8*1)
|
|
MOVQ SI, CX
|
|
|
|
emit_literal_done_repeat_emit_encodeSnappyBlockAsm8B:
|
|
ADDL $0x05, DX
|
|
MOVL DX, SI
|
|
SUBL 16(SP), SI
|
|
MOVQ src_len+32(FP), R8
|
|
SUBL DX, R8
|
|
LEAQ (BX)(DX*1), R9
|
|
LEAQ (BX)(SI*1), SI
|
|
|
|
// matchLen
|
|
XORL R11, R11
|
|
|
|
matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm8B:
|
|
CMPL R8, $0x10
|
|
JB matchlen_match8_repeat_extend_encodeSnappyBlockAsm8B
|
|
MOVQ (R9)(R11*1), R10
|
|
MOVQ 8(R9)(R11*1), R12
|
|
XORQ (SI)(R11*1), R10
|
|
JNZ matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm8B
|
|
XORQ 8(SI)(R11*1), R12
|
|
JNZ matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm8B
|
|
LEAL -16(R8), R8
|
|
LEAL 16(R11), R11
|
|
JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm8B
|
|
|
|
matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm8B:
|
|
TZCNTQ R12, R12
|
|
SARQ $0x03, R12
|
|
LEAL 8(R11)(R12*1), R11
|
|
JMP repeat_extend_forward_end_encodeSnappyBlockAsm8B
|
|
|
|
matchlen_match8_repeat_extend_encodeSnappyBlockAsm8B:
|
|
CMPL R8, $0x08
|
|
JB matchlen_match4_repeat_extend_encodeSnappyBlockAsm8B
|
|
MOVQ (R9)(R11*1), R10
|
|
XORQ (SI)(R11*1), R10
|
|
JNZ matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm8B
|
|
LEAL -8(R8), R8
|
|
LEAL 8(R11), R11
|
|
JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm8B
|
|
|
|
matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm8B:
|
|
TZCNTQ R10, R10
|
|
SARQ $0x03, R10
|
|
LEAL (R11)(R10*1), R11
|
|
JMP repeat_extend_forward_end_encodeSnappyBlockAsm8B
|
|
|
|
matchlen_match4_repeat_extend_encodeSnappyBlockAsm8B:
|
|
CMPL R8, $0x04
|
|
JB matchlen_match2_repeat_extend_encodeSnappyBlockAsm8B
|
|
MOVL (R9)(R11*1), R10
|
|
CMPL (SI)(R11*1), R10
|
|
JNE matchlen_match2_repeat_extend_encodeSnappyBlockAsm8B
|
|
LEAL -4(R8), R8
|
|
LEAL 4(R11), R11
|
|
|
|
matchlen_match2_repeat_extend_encodeSnappyBlockAsm8B:
|
|
CMPL R8, $0x01
|
|
JE matchlen_match1_repeat_extend_encodeSnappyBlockAsm8B
|
|
JB repeat_extend_forward_end_encodeSnappyBlockAsm8B
|
|
MOVW (R9)(R11*1), R10
|
|
CMPW (SI)(R11*1), R10
|
|
JNE matchlen_match1_repeat_extend_encodeSnappyBlockAsm8B
|
|
LEAL 2(R11), R11
|
|
SUBL $0x02, R8
|
|
JZ repeat_extend_forward_end_encodeSnappyBlockAsm8B
|
|
|
|
matchlen_match1_repeat_extend_encodeSnappyBlockAsm8B:
|
|
MOVB (R9)(R11*1), R10
|
|
CMPB (SI)(R11*1), R10
|
|
JNE repeat_extend_forward_end_encodeSnappyBlockAsm8B
|
|
LEAL 1(R11), R11
|
|
|
|
repeat_extend_forward_end_encodeSnappyBlockAsm8B:
|
|
ADDL R11, DX
|
|
MOVL DX, SI
|
|
SUBL DI, SI
|
|
MOVL 16(SP), DI
|
|
|
|
// emitCopy
|
|
two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm8B:
|
|
CMPL SI, $0x40
|
|
JBE two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm8B
|
|
MOVB $0xee, (CX)
|
|
MOVW DI, 1(CX)
|
|
LEAL -60(SI), SI
|
|
ADDQ $0x03, CX
|
|
JMP two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm8B
|
|
|
|
two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm8B:
|
|
MOVL SI, R8
|
|
SHLL $0x02, R8
|
|
CMPL SI, $0x0c
|
|
JAE emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm8B
|
|
LEAL -15(R8), R8
|
|
MOVB DI, 1(CX)
|
|
SHRL $0x08, DI
|
|
SHLL $0x05, DI
|
|
ORL DI, R8
|
|
MOVB R8, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_encodeSnappyBlockAsm8B
|
|
|
|
emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm8B:
|
|
LEAL -2(R8), R8
|
|
MOVB R8, (CX)
|
|
MOVW DI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
|
|
repeat_end_emit_encodeSnappyBlockAsm8B:
|
|
MOVL DX, 12(SP)
|
|
JMP search_loop_encodeSnappyBlockAsm8B
|
|
|
|
no_repeat_found_encodeSnappyBlockAsm8B:
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ candidate_match_encodeSnappyBlockAsm8B
|
|
SHRQ $0x08, DI
|
|
MOVL (AX)(R10*4), SI
|
|
LEAL 2(DX), R9
|
|
CMPL (BX)(R8*1), DI
|
|
JEQ candidate2_match_encodeSnappyBlockAsm8B
|
|
MOVL R9, (AX)(R10*4)
|
|
SHRQ $0x08, DI
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ candidate3_match_encodeSnappyBlockAsm8B
|
|
MOVL 20(SP), DX
|
|
JMP search_loop_encodeSnappyBlockAsm8B
|
|
|
|
candidate3_match_encodeSnappyBlockAsm8B:
|
|
ADDL $0x02, DX
|
|
JMP candidate_match_encodeSnappyBlockAsm8B
|
|
|
|
candidate2_match_encodeSnappyBlockAsm8B:
|
|
MOVL R9, (AX)(R10*4)
|
|
INCL DX
|
|
MOVL R8, SI
|
|
|
|
candidate_match_encodeSnappyBlockAsm8B:
|
|
MOVL 12(SP), DI
|
|
TESTL SI, SI
|
|
JZ match_extend_back_end_encodeSnappyBlockAsm8B
|
|
|
|
match_extend_back_loop_encodeSnappyBlockAsm8B:
|
|
CMPL DX, DI
|
|
JBE match_extend_back_end_encodeSnappyBlockAsm8B
|
|
MOVB -1(BX)(SI*1), R8
|
|
MOVB -1(BX)(DX*1), R9
|
|
CMPB R8, R9
|
|
JNE match_extend_back_end_encodeSnappyBlockAsm8B
|
|
LEAL -1(DX), DX
|
|
DECL SI
|
|
JZ match_extend_back_end_encodeSnappyBlockAsm8B
|
|
JMP match_extend_back_loop_encodeSnappyBlockAsm8B
|
|
|
|
match_extend_back_end_encodeSnappyBlockAsm8B:
|
|
MOVL DX, DI
|
|
SUBL 12(SP), DI
|
|
LEAQ 3(CX)(DI*1), DI
|
|
CMPQ DI, (SP)
|
|
JB match_dst_size_check_encodeSnappyBlockAsm8B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeSnappyBlockAsm8B:
|
|
MOVL DX, DI
|
|
MOVL 12(SP), R8
|
|
CMPL R8, DI
|
|
JEQ emit_literal_done_match_emit_encodeSnappyBlockAsm8B
|
|
MOVL DI, R9
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(R8*1), DI
|
|
SUBL R8, R9
|
|
LEAL -1(R9), R8
|
|
CMPL R8, $0x3c
|
|
JB one_byte_match_emit_encodeSnappyBlockAsm8B
|
|
CMPL R8, $0x00000100
|
|
JB two_bytes_match_emit_encodeSnappyBlockAsm8B
|
|
JB three_bytes_match_emit_encodeSnappyBlockAsm8B
|
|
|
|
three_bytes_match_emit_encodeSnappyBlockAsm8B:
|
|
MOVB $0xf4, (CX)
|
|
MOVW R8, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_match_emit_encodeSnappyBlockAsm8B
|
|
|
|
two_bytes_match_emit_encodeSnappyBlockAsm8B:
|
|
MOVB $0xf0, (CX)
|
|
MOVB R8, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL R8, $0x40
|
|
JB memmove_match_emit_encodeSnappyBlockAsm8B
|
|
JMP memmove_long_match_emit_encodeSnappyBlockAsm8B
|
|
|
|
one_byte_match_emit_encodeSnappyBlockAsm8B:
|
|
SHLB $0x02, R8
|
|
MOVB R8, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_match_emit_encodeSnappyBlockAsm8B:
|
|
LEAQ (CX)(R9*1), R8
|
|
|
|
// genMemMoveShort
|
|
CMPQ R9, $0x08
|
|
JBE emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_8
|
|
CMPQ R9, $0x10
|
|
JBE emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_8through16
|
|
CMPQ R9, $0x20
|
|
JBE emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_8:
|
|
MOVQ (DI), R10
|
|
MOVQ R10, (CX)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm8B
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_8through16:
|
|
MOVQ (DI), R10
|
|
MOVQ -8(DI)(R9*1), DI
|
|
MOVQ R10, (CX)
|
|
MOVQ DI, -8(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm8B
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_17through32:
|
|
MOVOU (DI), X0
|
|
MOVOU -16(DI)(R9*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm8B
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_33through64:
|
|
MOVOU (DI), X0
|
|
MOVOU 16(DI), X1
|
|
MOVOU -32(DI)(R9*1), X2
|
|
MOVOU -16(DI)(R9*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
|
|
memmove_end_copy_match_emit_encodeSnappyBlockAsm8B:
|
|
MOVQ R8, CX
|
|
JMP emit_literal_done_match_emit_encodeSnappyBlockAsm8B
|
|
|
|
memmove_long_match_emit_encodeSnappyBlockAsm8B:
|
|
LEAQ (CX)(R9*1), R8
|
|
|
|
// genMemMoveLong
|
|
MOVOU (DI), X0
|
|
MOVOU 16(DI), X1
|
|
MOVOU -32(DI)(R9*1), X2
|
|
MOVOU -16(DI)(R9*1), X3
|
|
MOVQ R9, R11
|
|
SHRQ $0x05, R11
|
|
MOVQ CX, R10
|
|
ANDL $0x0000001f, R10
|
|
MOVQ $0x00000040, R12
|
|
SUBQ R10, R12
|
|
DECQ R11
|
|
JA emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm8Blarge_forward_sse_loop_32
|
|
LEAQ -32(DI)(R12*1), R10
|
|
LEAQ -32(CX)(R12*1), R13
|
|
|
|
emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm8Blarge_big_loop_back:
|
|
MOVOU (R10), X4
|
|
MOVOU 16(R10), X5
|
|
MOVOA X4, (R13)
|
|
MOVOA X5, 16(R13)
|
|
ADDQ $0x20, R13
|
|
ADDQ $0x20, R10
|
|
ADDQ $0x20, R12
|
|
DECQ R11
|
|
JNA emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm8Blarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm8Blarge_forward_sse_loop_32:
|
|
MOVOU -32(DI)(R12*1), X4
|
|
MOVOU -16(DI)(R12*1), X5
|
|
MOVOA X4, -32(CX)(R12*1)
|
|
MOVOA X5, -16(CX)(R12*1)
|
|
ADDQ $0x20, R12
|
|
CMPQ R9, R12
|
|
JAE emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm8Blarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
MOVQ R8, CX
|
|
|
|
emit_literal_done_match_emit_encodeSnappyBlockAsm8B:
|
|
match_nolit_loop_encodeSnappyBlockAsm8B:
|
|
MOVL DX, DI
|
|
SUBL SI, DI
|
|
MOVL DI, 16(SP)
|
|
ADDL $0x04, DX
|
|
ADDL $0x04, SI
|
|
MOVQ src_len+32(FP), DI
|
|
SUBL DX, DI
|
|
LEAQ (BX)(DX*1), R8
|
|
LEAQ (BX)(SI*1), SI
|
|
|
|
// matchLen
|
|
XORL R10, R10
|
|
|
|
matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm8B:
|
|
CMPL DI, $0x10
|
|
JB matchlen_match8_match_nolit_encodeSnappyBlockAsm8B
|
|
MOVQ (R8)(R10*1), R9
|
|
MOVQ 8(R8)(R10*1), R11
|
|
XORQ (SI)(R10*1), R9
|
|
JNZ matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm8B
|
|
XORQ 8(SI)(R10*1), R11
|
|
JNZ matchlen_bsf_16match_nolit_encodeSnappyBlockAsm8B
|
|
LEAL -16(DI), DI
|
|
LEAL 16(R10), R10
|
|
JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm8B
|
|
|
|
matchlen_bsf_16match_nolit_encodeSnappyBlockAsm8B:
|
|
TZCNTQ R11, R11
|
|
SARQ $0x03, R11
|
|
LEAL 8(R10)(R11*1), R10
|
|
JMP match_nolit_end_encodeSnappyBlockAsm8B
|
|
|
|
matchlen_match8_match_nolit_encodeSnappyBlockAsm8B:
|
|
CMPL DI, $0x08
|
|
JB matchlen_match4_match_nolit_encodeSnappyBlockAsm8B
|
|
MOVQ (R8)(R10*1), R9
|
|
XORQ (SI)(R10*1), R9
|
|
JNZ matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm8B
|
|
LEAL -8(DI), DI
|
|
LEAL 8(R10), R10
|
|
JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm8B
|
|
|
|
matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm8B:
|
|
TZCNTQ R9, R9
|
|
SARQ $0x03, R9
|
|
LEAL (R10)(R9*1), R10
|
|
JMP match_nolit_end_encodeSnappyBlockAsm8B
|
|
|
|
matchlen_match4_match_nolit_encodeSnappyBlockAsm8B:
|
|
CMPL DI, $0x04
|
|
JB matchlen_match2_match_nolit_encodeSnappyBlockAsm8B
|
|
MOVL (R8)(R10*1), R9
|
|
CMPL (SI)(R10*1), R9
|
|
JNE matchlen_match2_match_nolit_encodeSnappyBlockAsm8B
|
|
LEAL -4(DI), DI
|
|
LEAL 4(R10), R10
|
|
|
|
matchlen_match2_match_nolit_encodeSnappyBlockAsm8B:
|
|
CMPL DI, $0x01
|
|
JE matchlen_match1_match_nolit_encodeSnappyBlockAsm8B
|
|
JB match_nolit_end_encodeSnappyBlockAsm8B
|
|
MOVW (R8)(R10*1), R9
|
|
CMPW (SI)(R10*1), R9
|
|
JNE matchlen_match1_match_nolit_encodeSnappyBlockAsm8B
|
|
LEAL 2(R10), R10
|
|
SUBL $0x02, DI
|
|
JZ match_nolit_end_encodeSnappyBlockAsm8B
|
|
|
|
matchlen_match1_match_nolit_encodeSnappyBlockAsm8B:
|
|
MOVB (R8)(R10*1), R9
|
|
CMPB (SI)(R10*1), R9
|
|
JNE match_nolit_end_encodeSnappyBlockAsm8B
|
|
LEAL 1(R10), R10
|
|
|
|
match_nolit_end_encodeSnappyBlockAsm8B:
|
|
ADDL R10, DX
|
|
MOVL 16(SP), SI
|
|
ADDL $0x04, R10
|
|
MOVL DX, 12(SP)
|
|
|
|
// emitCopy
|
|
two_byte_offset_match_nolit_encodeSnappyBlockAsm8B:
|
|
CMPL R10, $0x40
|
|
JBE two_byte_offset_short_match_nolit_encodeSnappyBlockAsm8B
|
|
MOVB $0xee, (CX)
|
|
MOVW SI, 1(CX)
|
|
LEAL -60(R10), R10
|
|
ADDQ $0x03, CX
|
|
JMP two_byte_offset_match_nolit_encodeSnappyBlockAsm8B
|
|
|
|
two_byte_offset_short_match_nolit_encodeSnappyBlockAsm8B:
|
|
MOVL R10, DI
|
|
SHLL $0x02, DI
|
|
CMPL R10, $0x0c
|
|
JAE emit_copy_three_match_nolit_encodeSnappyBlockAsm8B
|
|
LEAL -15(DI), DI
|
|
MOVB SI, 1(CX)
|
|
SHRL $0x08, SI
|
|
SHLL $0x05, SI
|
|
ORL SI, DI
|
|
MOVB DI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeSnappyBlockAsm8B
|
|
|
|
emit_copy_three_match_nolit_encodeSnappyBlockAsm8B:
|
|
LEAL -2(DI), DI
|
|
MOVB DI, (CX)
|
|
MOVW SI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
|
|
match_nolit_emitcopy_end_encodeSnappyBlockAsm8B:
|
|
CMPL DX, 8(SP)
|
|
JAE emit_remainder_encodeSnappyBlockAsm8B
|
|
MOVQ -2(BX)(DX*1), DI
|
|
CMPQ CX, (SP)
|
|
JB match_nolit_dst_ok_encodeSnappyBlockAsm8B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeSnappyBlockAsm8B:
|
|
MOVQ $0x9e3779b1, R9
|
|
MOVQ DI, R8
|
|
SHRQ $0x10, DI
|
|
MOVQ DI, SI
|
|
SHLQ $0x20, R8
|
|
IMULQ R9, R8
|
|
SHRQ $0x38, R8
|
|
SHLQ $0x20, SI
|
|
IMULQ R9, SI
|
|
SHRQ $0x38, SI
|
|
LEAL -2(DX), R9
|
|
LEAQ (AX)(SI*4), R10
|
|
MOVL (R10), SI
|
|
MOVL R9, (AX)(R8*4)
|
|
MOVL DX, (R10)
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ match_nolit_loop_encodeSnappyBlockAsm8B
|
|
INCL DX
|
|
JMP search_loop_encodeSnappyBlockAsm8B
|
|
|
|
emit_remainder_encodeSnappyBlockAsm8B:
|
|
MOVQ src_len+32(FP), AX
|
|
SUBL 12(SP), AX
|
|
LEAQ 3(CX)(AX*1), AX
|
|
CMPQ AX, (SP)
|
|
JB emit_remainder_ok_encodeSnappyBlockAsm8B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeSnappyBlockAsm8B:
|
|
MOVQ src_len+32(FP), AX
|
|
MOVL 12(SP), DX
|
|
CMPL DX, AX
|
|
JEQ emit_literal_done_emit_remainder_encodeSnappyBlockAsm8B
|
|
MOVL AX, SI
|
|
MOVL AX, 12(SP)
|
|
LEAQ (BX)(DX*1), AX
|
|
SUBL DX, SI
|
|
LEAL -1(SI), DX
|
|
CMPL DX, $0x3c
|
|
JB one_byte_emit_remainder_encodeSnappyBlockAsm8B
|
|
CMPL DX, $0x00000100
|
|
JB two_bytes_emit_remainder_encodeSnappyBlockAsm8B
|
|
JB three_bytes_emit_remainder_encodeSnappyBlockAsm8B
|
|
|
|
three_bytes_emit_remainder_encodeSnappyBlockAsm8B:
|
|
MOVB $0xf4, (CX)
|
|
MOVW DX, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_emit_remainder_encodeSnappyBlockAsm8B
|
|
|
|
two_bytes_emit_remainder_encodeSnappyBlockAsm8B:
|
|
MOVB $0xf0, (CX)
|
|
MOVB DL, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL DX, $0x40
|
|
JB memmove_emit_remainder_encodeSnappyBlockAsm8B
|
|
JMP memmove_long_emit_remainder_encodeSnappyBlockAsm8B
|
|
|
|
one_byte_emit_remainder_encodeSnappyBlockAsm8B:
|
|
SHLB $0x02, DL
|
|
MOVB DL, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_emit_remainder_encodeSnappyBlockAsm8B:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveShort
|
|
CMPQ BX, $0x03
|
|
JB emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_1or2
|
|
JE emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_3
|
|
CMPQ BX, $0x08
|
|
JB emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_4through7
|
|
CMPQ BX, $0x10
|
|
JBE emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_8through16
|
|
CMPQ BX, $0x20
|
|
JBE emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_1or2:
|
|
MOVB (AX), SI
|
|
MOVB -1(AX)(BX*1), AL
|
|
MOVB SI, (CX)
|
|
MOVB AL, -1(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_3:
|
|
MOVW (AX), SI
|
|
MOVB 2(AX), AL
|
|
MOVW SI, (CX)
|
|
MOVB AL, 2(CX)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_4through7:
|
|
MOVL (AX), SI
|
|
MOVL -4(AX)(BX*1), AX
|
|
MOVL SI, (CX)
|
|
MOVL AX, -4(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_8through16:
|
|
MOVQ (AX), SI
|
|
MOVQ -8(AX)(BX*1), AX
|
|
MOVQ SI, (CX)
|
|
MOVQ AX, -8(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_17through32:
|
|
MOVOU (AX), X0
|
|
MOVOU -16(AX)(BX*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_33through64:
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
|
|
memmove_end_copy_emit_remainder_encodeSnappyBlockAsm8B:
|
|
MOVQ DX, CX
|
|
JMP emit_literal_done_emit_remainder_encodeSnappyBlockAsm8B
|
|
|
|
memmove_long_emit_remainder_encodeSnappyBlockAsm8B:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveLong
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVQ BX, DI
|
|
SHRQ $0x05, DI
|
|
MOVQ CX, SI
|
|
ANDL $0x0000001f, SI
|
|
MOVQ $0x00000040, R8
|
|
SUBQ SI, R8
|
|
DECQ DI
|
|
JA emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm8Blarge_forward_sse_loop_32
|
|
LEAQ -32(AX)(R8*1), SI
|
|
LEAQ -32(CX)(R8*1), R9
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm8Blarge_big_loop_back:
|
|
MOVOU (SI), X4
|
|
MOVOU 16(SI), X5
|
|
MOVOA X4, (R9)
|
|
MOVOA X5, 16(R9)
|
|
ADDQ $0x20, R9
|
|
ADDQ $0x20, SI
|
|
ADDQ $0x20, R8
|
|
DECQ DI
|
|
JNA emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm8Blarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm8Blarge_forward_sse_loop_32:
|
|
MOVOU -32(AX)(R8*1), X4
|
|
MOVOU -16(AX)(R8*1), X5
|
|
MOVOA X4, -32(CX)(R8*1)
|
|
MOVOA X5, -16(CX)(R8*1)
|
|
ADDQ $0x20, R8
|
|
CMPQ BX, R8
|
|
JAE emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm8Blarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
MOVQ DX, CX
|
|
|
|
emit_literal_done_emit_remainder_encodeSnappyBlockAsm8B:
|
|
MOVQ dst_base+0(FP), AX
|
|
SUBQ AX, CX
|
|
MOVQ CX, ret+56(FP)
|
|
RET
|
|
|
|
// func encodeSnappyBetterBlockAsm(dst []byte, src []byte, tmp *[589824]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeSnappyBetterBlockAsm(SB), $24-64
|
|
MOVQ tmp+48(FP), AX
|
|
MOVQ dst_base+0(FP), CX
|
|
MOVQ $0x00001200, DX
|
|
MOVQ AX, BX
|
|
PXOR X0, X0
|
|
|
|
zero_loop_encodeSnappyBetterBlockAsm:
|
|
MOVOU X0, (BX)
|
|
MOVOU X0, 16(BX)
|
|
MOVOU X0, 32(BX)
|
|
MOVOU X0, 48(BX)
|
|
MOVOU X0, 64(BX)
|
|
MOVOU X0, 80(BX)
|
|
MOVOU X0, 96(BX)
|
|
MOVOU X0, 112(BX)
|
|
ADDQ $0x80, BX
|
|
DECQ DX
|
|
JNZ zero_loop_encodeSnappyBetterBlockAsm
|
|
MOVL $0x00000000, 12(SP)
|
|
MOVQ src_len+32(FP), DX
|
|
LEAQ -9(DX), BX
|
|
LEAQ -8(DX), SI
|
|
MOVL SI, 8(SP)
|
|
SHRQ $0x05, DX
|
|
SUBL DX, BX
|
|
LEAQ (CX)(BX*1), BX
|
|
MOVQ BX, (SP)
|
|
MOVL $0x00000001, DX
|
|
MOVL $0x00000000, 16(SP)
|
|
MOVQ src_base+24(FP), BX
|
|
|
|
search_loop_encodeSnappyBetterBlockAsm:
|
|
MOVL DX, SI
|
|
SUBL 12(SP), SI
|
|
SHRL $0x07, SI
|
|
CMPL SI, $0x63
|
|
JBE check_maxskip_ok_encodeSnappyBetterBlockAsm
|
|
LEAL 100(DX), SI
|
|
JMP check_maxskip_cont_encodeSnappyBetterBlockAsm
|
|
|
|
check_maxskip_ok_encodeSnappyBetterBlockAsm:
|
|
LEAL 1(DX)(SI*1), SI
|
|
|
|
check_maxskip_cont_encodeSnappyBetterBlockAsm:
|
|
CMPL SI, 8(SP)
|
|
JAE emit_remainder_encodeSnappyBetterBlockAsm
|
|
MOVQ (BX)(DX*1), DI
|
|
MOVL SI, 20(SP)
|
|
MOVQ $0x00cf1bbcdcbfa563, R9
|
|
MOVQ $0x9e3779b1, SI
|
|
MOVQ DI, R10
|
|
MOVQ DI, R11
|
|
SHLQ $0x08, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x2f, R10
|
|
SHLQ $0x20, R11
|
|
IMULQ SI, R11
|
|
SHRQ $0x32, R11
|
|
MOVL (AX)(R10*4), SI
|
|
MOVL 524288(AX)(R11*4), R8
|
|
MOVL DX, (AX)(R10*4)
|
|
MOVL DX, 524288(AX)(R11*4)
|
|
MOVQ (BX)(SI*1), R10
|
|
MOVQ (BX)(R8*1), R11
|
|
CMPQ R10, DI
|
|
JEQ candidate_match_encodeSnappyBetterBlockAsm
|
|
CMPQ R11, DI
|
|
JNE no_short_found_encodeSnappyBetterBlockAsm
|
|
MOVL R8, SI
|
|
JMP candidate_match_encodeSnappyBetterBlockAsm
|
|
|
|
no_short_found_encodeSnappyBetterBlockAsm:
|
|
CMPL R10, DI
|
|
JEQ candidate_match_encodeSnappyBetterBlockAsm
|
|
CMPL R11, DI
|
|
JEQ candidateS_match_encodeSnappyBetterBlockAsm
|
|
MOVL 20(SP), DX
|
|
JMP search_loop_encodeSnappyBetterBlockAsm
|
|
|
|
candidateS_match_encodeSnappyBetterBlockAsm:
|
|
SHRQ $0x08, DI
|
|
MOVQ DI, R10
|
|
SHLQ $0x08, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x2f, R10
|
|
MOVL (AX)(R10*4), SI
|
|
INCL DX
|
|
MOVL DX, (AX)(R10*4)
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ candidate_match_encodeSnappyBetterBlockAsm
|
|
DECL DX
|
|
MOVL R8, SI
|
|
|
|
candidate_match_encodeSnappyBetterBlockAsm:
|
|
MOVL 12(SP), DI
|
|
TESTL SI, SI
|
|
JZ match_extend_back_end_encodeSnappyBetterBlockAsm
|
|
|
|
match_extend_back_loop_encodeSnappyBetterBlockAsm:
|
|
CMPL DX, DI
|
|
JBE match_extend_back_end_encodeSnappyBetterBlockAsm
|
|
MOVB -1(BX)(SI*1), R8
|
|
MOVB -1(BX)(DX*1), R9
|
|
CMPB R8, R9
|
|
JNE match_extend_back_end_encodeSnappyBetterBlockAsm
|
|
LEAL -1(DX), DX
|
|
DECL SI
|
|
JZ match_extend_back_end_encodeSnappyBetterBlockAsm
|
|
JMP match_extend_back_loop_encodeSnappyBetterBlockAsm
|
|
|
|
match_extend_back_end_encodeSnappyBetterBlockAsm:
|
|
MOVL DX, DI
|
|
SUBL 12(SP), DI
|
|
LEAQ 5(CX)(DI*1), DI
|
|
CMPQ DI, (SP)
|
|
JB match_dst_size_check_encodeSnappyBetterBlockAsm
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeSnappyBetterBlockAsm:
|
|
MOVL DX, DI
|
|
ADDL $0x04, DX
|
|
ADDL $0x04, SI
|
|
MOVQ src_len+32(FP), R8
|
|
SUBL DX, R8
|
|
LEAQ (BX)(DX*1), R9
|
|
LEAQ (BX)(SI*1), R10
|
|
|
|
// matchLen
|
|
XORL R12, R12
|
|
|
|
matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm:
|
|
CMPL R8, $0x10
|
|
JB matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm
|
|
MOVQ (R9)(R12*1), R11
|
|
MOVQ 8(R9)(R12*1), R13
|
|
XORQ (R10)(R12*1), R11
|
|
JNZ matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm
|
|
XORQ 8(R10)(R12*1), R13
|
|
JNZ matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm
|
|
LEAL -16(R8), R8
|
|
LEAL 16(R12), R12
|
|
JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm
|
|
|
|
matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm:
|
|
TZCNTQ R13, R13
|
|
SARQ $0x03, R13
|
|
LEAL 8(R12)(R13*1), R12
|
|
JMP match_nolit_end_encodeSnappyBetterBlockAsm
|
|
|
|
matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm:
|
|
CMPL R8, $0x08
|
|
JB matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm
|
|
MOVQ (R9)(R12*1), R11
|
|
XORQ (R10)(R12*1), R11
|
|
JNZ matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm
|
|
LEAL -8(R8), R8
|
|
LEAL 8(R12), R12
|
|
JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm
|
|
|
|
matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm:
|
|
TZCNTQ R11, R11
|
|
SARQ $0x03, R11
|
|
LEAL (R12)(R11*1), R12
|
|
JMP match_nolit_end_encodeSnappyBetterBlockAsm
|
|
|
|
matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm:
|
|
CMPL R8, $0x04
|
|
JB matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm
|
|
MOVL (R9)(R12*1), R11
|
|
CMPL (R10)(R12*1), R11
|
|
JNE matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm
|
|
LEAL -4(R8), R8
|
|
LEAL 4(R12), R12
|
|
|
|
matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm:
|
|
CMPL R8, $0x01
|
|
JE matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm
|
|
JB match_nolit_end_encodeSnappyBetterBlockAsm
|
|
MOVW (R9)(R12*1), R11
|
|
CMPW (R10)(R12*1), R11
|
|
JNE matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm
|
|
LEAL 2(R12), R12
|
|
SUBL $0x02, R8
|
|
JZ match_nolit_end_encodeSnappyBetterBlockAsm
|
|
|
|
matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm:
|
|
MOVB (R9)(R12*1), R11
|
|
CMPB (R10)(R12*1), R11
|
|
JNE match_nolit_end_encodeSnappyBetterBlockAsm
|
|
LEAL 1(R12), R12
|
|
|
|
match_nolit_end_encodeSnappyBetterBlockAsm:
|
|
MOVL DX, R8
|
|
SUBL SI, R8
|
|
|
|
// Check if repeat
|
|
CMPL R12, $0x01
|
|
JA match_length_ok_encodeSnappyBetterBlockAsm
|
|
CMPL R8, $0x0000ffff
|
|
JBE match_length_ok_encodeSnappyBetterBlockAsm
|
|
MOVL 20(SP), DX
|
|
INCL DX
|
|
JMP search_loop_encodeSnappyBetterBlockAsm
|
|
|
|
match_length_ok_encodeSnappyBetterBlockAsm:
|
|
MOVL R8, 16(SP)
|
|
MOVL 12(SP), SI
|
|
CMPL SI, DI
|
|
JEQ emit_literal_done_match_emit_encodeSnappyBetterBlockAsm
|
|
MOVL DI, R9
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(SI*1), R10
|
|
SUBL SI, R9
|
|
LEAL -1(R9), SI
|
|
CMPL SI, $0x3c
|
|
JB one_byte_match_emit_encodeSnappyBetterBlockAsm
|
|
CMPL SI, $0x00000100
|
|
JB two_bytes_match_emit_encodeSnappyBetterBlockAsm
|
|
CMPL SI, $0x00010000
|
|
JB three_bytes_match_emit_encodeSnappyBetterBlockAsm
|
|
CMPL SI, $0x01000000
|
|
JB four_bytes_match_emit_encodeSnappyBetterBlockAsm
|
|
MOVB $0xfc, (CX)
|
|
MOVL SI, 1(CX)
|
|
ADDQ $0x05, CX
|
|
JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm
|
|
|
|
four_bytes_match_emit_encodeSnappyBetterBlockAsm:
|
|
MOVL SI, R11
|
|
SHRL $0x10, R11
|
|
MOVB $0xf8, (CX)
|
|
MOVW SI, 1(CX)
|
|
MOVB R11, 3(CX)
|
|
ADDQ $0x04, CX
|
|
JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm
|
|
|
|
three_bytes_match_emit_encodeSnappyBetterBlockAsm:
|
|
MOVB $0xf4, (CX)
|
|
MOVW SI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm
|
|
|
|
two_bytes_match_emit_encodeSnappyBetterBlockAsm:
|
|
MOVB $0xf0, (CX)
|
|
MOVB SI, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL SI, $0x40
|
|
JB memmove_match_emit_encodeSnappyBetterBlockAsm
|
|
JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm
|
|
|
|
one_byte_match_emit_encodeSnappyBetterBlockAsm:
|
|
SHLB $0x02, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_match_emit_encodeSnappyBetterBlockAsm:
|
|
LEAQ (CX)(R9*1), SI
|
|
|
|
// genMemMoveShort
|
|
CMPQ R9, $0x08
|
|
JBE emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_8
|
|
CMPQ R9, $0x10
|
|
JBE emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_8through16
|
|
CMPQ R9, $0x20
|
|
JBE emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_8:
|
|
MOVQ (R10), R11
|
|
MOVQ R11, (CX)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_8through16:
|
|
MOVQ (R10), R11
|
|
MOVQ -8(R10)(R9*1), R10
|
|
MOVQ R11, (CX)
|
|
MOVQ R10, -8(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_17through32:
|
|
MOVOU (R10), X0
|
|
MOVOU -16(R10)(R9*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_33through64:
|
|
MOVOU (R10), X0
|
|
MOVOU 16(R10), X1
|
|
MOVOU -32(R10)(R9*1), X2
|
|
MOVOU -16(R10)(R9*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
|
|
memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm:
|
|
MOVQ SI, CX
|
|
JMP emit_literal_done_match_emit_encodeSnappyBetterBlockAsm
|
|
|
|
memmove_long_match_emit_encodeSnappyBetterBlockAsm:
|
|
LEAQ (CX)(R9*1), SI
|
|
|
|
// genMemMoveLong
|
|
MOVOU (R10), X0
|
|
MOVOU 16(R10), X1
|
|
MOVOU -32(R10)(R9*1), X2
|
|
MOVOU -16(R10)(R9*1), X3
|
|
MOVQ R9, R13
|
|
SHRQ $0x05, R13
|
|
MOVQ CX, R11
|
|
ANDL $0x0000001f, R11
|
|
MOVQ $0x00000040, R14
|
|
SUBQ R11, R14
|
|
DECQ R13
|
|
JA emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsmlarge_forward_sse_loop_32
|
|
LEAQ -32(R10)(R14*1), R11
|
|
LEAQ -32(CX)(R14*1), R15
|
|
|
|
emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsmlarge_big_loop_back:
|
|
MOVOU (R11), X4
|
|
MOVOU 16(R11), X5
|
|
MOVOA X4, (R15)
|
|
MOVOA X5, 16(R15)
|
|
ADDQ $0x20, R15
|
|
ADDQ $0x20, R11
|
|
ADDQ $0x20, R14
|
|
DECQ R13
|
|
JNA emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsmlarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsmlarge_forward_sse_loop_32:
|
|
MOVOU -32(R10)(R14*1), X4
|
|
MOVOU -16(R10)(R14*1), X5
|
|
MOVOA X4, -32(CX)(R14*1)
|
|
MOVOA X5, -16(CX)(R14*1)
|
|
ADDQ $0x20, R14
|
|
CMPQ R9, R14
|
|
JAE emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsmlarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
MOVQ SI, CX
|
|
|
|
emit_literal_done_match_emit_encodeSnappyBetterBlockAsm:
|
|
ADDL R12, DX
|
|
ADDL $0x04, R12
|
|
MOVL DX, 12(SP)
|
|
|
|
// emitCopy
|
|
CMPL R8, $0x00010000
|
|
JB two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm
|
|
|
|
four_bytes_loop_back_match_nolit_encodeSnappyBetterBlockAsm:
|
|
CMPL R12, $0x40
|
|
JBE four_bytes_remain_match_nolit_encodeSnappyBetterBlockAsm
|
|
MOVB $0xff, (CX)
|
|
MOVL R8, 1(CX)
|
|
LEAL -64(R12), R12
|
|
ADDQ $0x05, CX
|
|
CMPL R12, $0x04
|
|
JB four_bytes_remain_match_nolit_encodeSnappyBetterBlockAsm
|
|
JMP four_bytes_loop_back_match_nolit_encodeSnappyBetterBlockAsm
|
|
|
|
four_bytes_remain_match_nolit_encodeSnappyBetterBlockAsm:
|
|
TESTL R12, R12
|
|
JZ match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm
|
|
XORL SI, SI
|
|
LEAL -1(SI)(R12*4), R12
|
|
MOVB R12, (CX)
|
|
MOVL R8, 1(CX)
|
|
ADDQ $0x05, CX
|
|
JMP match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm
|
|
|
|
two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm:
|
|
CMPL R12, $0x40
|
|
JBE two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm
|
|
MOVB $0xee, (CX)
|
|
MOVW R8, 1(CX)
|
|
LEAL -60(R12), R12
|
|
ADDQ $0x03, CX
|
|
JMP two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm
|
|
|
|
two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm:
|
|
MOVL R12, SI
|
|
SHLL $0x02, SI
|
|
CMPL R12, $0x0c
|
|
JAE emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm
|
|
CMPL R8, $0x00000800
|
|
JAE emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm
|
|
LEAL -15(SI), SI
|
|
MOVB R8, 1(CX)
|
|
SHRL $0x08, R8
|
|
SHLL $0x05, R8
|
|
ORL R8, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm
|
|
|
|
emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm:
|
|
LEAL -2(SI), SI
|
|
MOVB SI, (CX)
|
|
MOVW R8, 1(CX)
|
|
ADDQ $0x03, CX
|
|
|
|
match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm:
|
|
CMPL DX, 8(SP)
|
|
JAE emit_remainder_encodeSnappyBetterBlockAsm
|
|
CMPQ CX, (SP)
|
|
JB match_nolit_dst_ok_encodeSnappyBetterBlockAsm
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeSnappyBetterBlockAsm:
|
|
MOVQ $0x00cf1bbcdcbfa563, SI
|
|
MOVQ $0x9e3779b1, R8
|
|
LEAQ 1(DI), DI
|
|
LEAQ -2(DX), R9
|
|
MOVQ (BX)(DI*1), R10
|
|
MOVQ 1(BX)(DI*1), R11
|
|
MOVQ (BX)(R9*1), R12
|
|
MOVQ 1(BX)(R9*1), R13
|
|
SHLQ $0x08, R10
|
|
IMULQ SI, R10
|
|
SHRQ $0x2f, R10
|
|
SHLQ $0x20, R11
|
|
IMULQ R8, R11
|
|
SHRQ $0x32, R11
|
|
SHLQ $0x08, R12
|
|
IMULQ SI, R12
|
|
SHRQ $0x2f, R12
|
|
SHLQ $0x20, R13
|
|
IMULQ R8, R13
|
|
SHRQ $0x32, R13
|
|
LEAQ 1(DI), R8
|
|
LEAQ 1(R9), R14
|
|
MOVL DI, (AX)(R10*4)
|
|
MOVL R9, (AX)(R12*4)
|
|
MOVL R8, 524288(AX)(R11*4)
|
|
MOVL R14, 524288(AX)(R13*4)
|
|
LEAQ 1(R9)(DI*1), R8
|
|
SHRQ $0x01, R8
|
|
ADDQ $0x01, DI
|
|
SUBQ $0x01, R9
|
|
|
|
index_loop_encodeSnappyBetterBlockAsm:
|
|
CMPQ R8, R9
|
|
JAE search_loop_encodeSnappyBetterBlockAsm
|
|
MOVQ (BX)(DI*1), R10
|
|
MOVQ (BX)(R8*1), R11
|
|
SHLQ $0x08, R10
|
|
IMULQ SI, R10
|
|
SHRQ $0x2f, R10
|
|
SHLQ $0x08, R11
|
|
IMULQ SI, R11
|
|
SHRQ $0x2f, R11
|
|
MOVL DI, (AX)(R10*4)
|
|
MOVL R8, (AX)(R11*4)
|
|
ADDQ $0x02, DI
|
|
ADDQ $0x02, R8
|
|
JMP index_loop_encodeSnappyBetterBlockAsm
|
|
|
|
emit_remainder_encodeSnappyBetterBlockAsm:
|
|
MOVQ src_len+32(FP), AX
|
|
SUBL 12(SP), AX
|
|
LEAQ 5(CX)(AX*1), AX
|
|
CMPQ AX, (SP)
|
|
JB emit_remainder_ok_encodeSnappyBetterBlockAsm
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeSnappyBetterBlockAsm:
|
|
MOVQ src_len+32(FP), AX
|
|
MOVL 12(SP), DX
|
|
CMPL DX, AX
|
|
JEQ emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm
|
|
MOVL AX, SI
|
|
MOVL AX, 12(SP)
|
|
LEAQ (BX)(DX*1), AX
|
|
SUBL DX, SI
|
|
LEAL -1(SI), DX
|
|
CMPL DX, $0x3c
|
|
JB one_byte_emit_remainder_encodeSnappyBetterBlockAsm
|
|
CMPL DX, $0x00000100
|
|
JB two_bytes_emit_remainder_encodeSnappyBetterBlockAsm
|
|
CMPL DX, $0x00010000
|
|
JB three_bytes_emit_remainder_encodeSnappyBetterBlockAsm
|
|
CMPL DX, $0x01000000
|
|
JB four_bytes_emit_remainder_encodeSnappyBetterBlockAsm
|
|
MOVB $0xfc, (CX)
|
|
MOVL DX, 1(CX)
|
|
ADDQ $0x05, CX
|
|
JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm
|
|
|
|
four_bytes_emit_remainder_encodeSnappyBetterBlockAsm:
|
|
MOVL DX, BX
|
|
SHRL $0x10, BX
|
|
MOVB $0xf8, (CX)
|
|
MOVW DX, 1(CX)
|
|
MOVB BL, 3(CX)
|
|
ADDQ $0x04, CX
|
|
JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm
|
|
|
|
three_bytes_emit_remainder_encodeSnappyBetterBlockAsm:
|
|
MOVB $0xf4, (CX)
|
|
MOVW DX, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm
|
|
|
|
two_bytes_emit_remainder_encodeSnappyBetterBlockAsm:
|
|
MOVB $0xf0, (CX)
|
|
MOVB DL, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL DX, $0x40
|
|
JB memmove_emit_remainder_encodeSnappyBetterBlockAsm
|
|
JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm
|
|
|
|
one_byte_emit_remainder_encodeSnappyBetterBlockAsm:
|
|
SHLB $0x02, DL
|
|
MOVB DL, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_emit_remainder_encodeSnappyBetterBlockAsm:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveShort
|
|
CMPQ BX, $0x03
|
|
JB emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_1or2
|
|
JE emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_3
|
|
CMPQ BX, $0x08
|
|
JB emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_4through7
|
|
CMPQ BX, $0x10
|
|
JBE emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_8through16
|
|
CMPQ BX, $0x20
|
|
JBE emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_1or2:
|
|
MOVB (AX), SI
|
|
MOVB -1(AX)(BX*1), AL
|
|
MOVB SI, (CX)
|
|
MOVB AL, -1(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_3:
|
|
MOVW (AX), SI
|
|
MOVB 2(AX), AL
|
|
MOVW SI, (CX)
|
|
MOVB AL, 2(CX)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_4through7:
|
|
MOVL (AX), SI
|
|
MOVL -4(AX)(BX*1), AX
|
|
MOVL SI, (CX)
|
|
MOVL AX, -4(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_8through16:
|
|
MOVQ (AX), SI
|
|
MOVQ -8(AX)(BX*1), AX
|
|
MOVQ SI, (CX)
|
|
MOVQ AX, -8(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_17through32:
|
|
MOVOU (AX), X0
|
|
MOVOU -16(AX)(BX*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_33through64:
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
|
|
memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm:
|
|
MOVQ DX, CX
|
|
JMP emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm
|
|
|
|
memmove_long_emit_remainder_encodeSnappyBetterBlockAsm:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveLong
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVQ BX, DI
|
|
SHRQ $0x05, DI
|
|
MOVQ CX, SI
|
|
ANDL $0x0000001f, SI
|
|
MOVQ $0x00000040, R8
|
|
SUBQ SI, R8
|
|
DECQ DI
|
|
JA emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsmlarge_forward_sse_loop_32
|
|
LEAQ -32(AX)(R8*1), SI
|
|
LEAQ -32(CX)(R8*1), R9
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsmlarge_big_loop_back:
|
|
MOVOU (SI), X4
|
|
MOVOU 16(SI), X5
|
|
MOVOA X4, (R9)
|
|
MOVOA X5, 16(R9)
|
|
ADDQ $0x20, R9
|
|
ADDQ $0x20, SI
|
|
ADDQ $0x20, R8
|
|
DECQ DI
|
|
JNA emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsmlarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsmlarge_forward_sse_loop_32:
|
|
MOVOU -32(AX)(R8*1), X4
|
|
MOVOU -16(AX)(R8*1), X5
|
|
MOVOA X4, -32(CX)(R8*1)
|
|
MOVOA X5, -16(CX)(R8*1)
|
|
ADDQ $0x20, R8
|
|
CMPQ BX, R8
|
|
JAE emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsmlarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
MOVQ DX, CX
|
|
|
|
emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm:
|
|
MOVQ dst_base+0(FP), AX
|
|
SUBQ AX, CX
|
|
MOVQ CX, ret+56(FP)
|
|
RET
|
|
|
|
// func encodeSnappyBetterBlockAsm64K(dst []byte, src []byte, tmp *[294912]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeSnappyBetterBlockAsm64K(SB), $24-64
|
|
MOVQ tmp+48(FP), AX
|
|
MOVQ dst_base+0(FP), CX
|
|
MOVQ $0x00000900, DX
|
|
MOVQ AX, BX
|
|
PXOR X0, X0
|
|
|
|
zero_loop_encodeSnappyBetterBlockAsm64K:
|
|
MOVOU X0, (BX)
|
|
MOVOU X0, 16(BX)
|
|
MOVOU X0, 32(BX)
|
|
MOVOU X0, 48(BX)
|
|
MOVOU X0, 64(BX)
|
|
MOVOU X0, 80(BX)
|
|
MOVOU X0, 96(BX)
|
|
MOVOU X0, 112(BX)
|
|
ADDQ $0x80, BX
|
|
DECQ DX
|
|
JNZ zero_loop_encodeSnappyBetterBlockAsm64K
|
|
MOVL $0x00000000, 12(SP)
|
|
MOVQ src_len+32(FP), DX
|
|
LEAQ -9(DX), BX
|
|
LEAQ -8(DX), SI
|
|
MOVL SI, 8(SP)
|
|
SHRQ $0x05, DX
|
|
SUBL DX, BX
|
|
LEAQ (CX)(BX*1), BX
|
|
MOVQ BX, (SP)
|
|
MOVL $0x00000001, DX
|
|
MOVL $0x00000000, 16(SP)
|
|
MOVQ src_base+24(FP), BX
|
|
|
|
search_loop_encodeSnappyBetterBlockAsm64K:
|
|
MOVL DX, SI
|
|
SUBL 12(SP), SI
|
|
SHRL $0x07, SI
|
|
LEAL 1(DX)(SI*1), SI
|
|
CMPL SI, 8(SP)
|
|
JAE emit_remainder_encodeSnappyBetterBlockAsm64K
|
|
MOVQ (BX)(DX*1), DI
|
|
MOVL SI, 20(SP)
|
|
MOVQ $0x00cf1bbcdcbfa563, R9
|
|
MOVQ $0x9e3779b1, SI
|
|
MOVQ DI, R10
|
|
MOVQ DI, R11
|
|
SHLQ $0x08, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x30, R10
|
|
SHLQ $0x20, R11
|
|
IMULQ SI, R11
|
|
SHRQ $0x33, R11
|
|
MOVL (AX)(R10*4), SI
|
|
MOVL 262144(AX)(R11*4), R8
|
|
MOVL DX, (AX)(R10*4)
|
|
MOVL DX, 262144(AX)(R11*4)
|
|
MOVQ (BX)(SI*1), R10
|
|
MOVQ (BX)(R8*1), R11
|
|
CMPQ R10, DI
|
|
JEQ candidate_match_encodeSnappyBetterBlockAsm64K
|
|
CMPQ R11, DI
|
|
JNE no_short_found_encodeSnappyBetterBlockAsm64K
|
|
MOVL R8, SI
|
|
JMP candidate_match_encodeSnappyBetterBlockAsm64K
|
|
|
|
no_short_found_encodeSnappyBetterBlockAsm64K:
|
|
CMPL R10, DI
|
|
JEQ candidate_match_encodeSnappyBetterBlockAsm64K
|
|
CMPL R11, DI
|
|
JEQ candidateS_match_encodeSnappyBetterBlockAsm64K
|
|
MOVL 20(SP), DX
|
|
JMP search_loop_encodeSnappyBetterBlockAsm64K
|
|
|
|
candidateS_match_encodeSnappyBetterBlockAsm64K:
|
|
SHRQ $0x08, DI
|
|
MOVQ DI, R10
|
|
SHLQ $0x08, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x30, R10
|
|
MOVL (AX)(R10*4), SI
|
|
INCL DX
|
|
MOVL DX, (AX)(R10*4)
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ candidate_match_encodeSnappyBetterBlockAsm64K
|
|
DECL DX
|
|
MOVL R8, SI
|
|
|
|
candidate_match_encodeSnappyBetterBlockAsm64K:
|
|
MOVL 12(SP), DI
|
|
TESTL SI, SI
|
|
JZ match_extend_back_end_encodeSnappyBetterBlockAsm64K
|
|
|
|
match_extend_back_loop_encodeSnappyBetterBlockAsm64K:
|
|
CMPL DX, DI
|
|
JBE match_extend_back_end_encodeSnappyBetterBlockAsm64K
|
|
MOVB -1(BX)(SI*1), R8
|
|
MOVB -1(BX)(DX*1), R9
|
|
CMPB R8, R9
|
|
JNE match_extend_back_end_encodeSnappyBetterBlockAsm64K
|
|
LEAL -1(DX), DX
|
|
DECL SI
|
|
JZ match_extend_back_end_encodeSnappyBetterBlockAsm64K
|
|
JMP match_extend_back_loop_encodeSnappyBetterBlockAsm64K
|
|
|
|
match_extend_back_end_encodeSnappyBetterBlockAsm64K:
|
|
MOVL DX, DI
|
|
SUBL 12(SP), DI
|
|
LEAQ 3(CX)(DI*1), DI
|
|
CMPQ DI, (SP)
|
|
JB match_dst_size_check_encodeSnappyBetterBlockAsm64K
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeSnappyBetterBlockAsm64K:
|
|
MOVL DX, DI
|
|
ADDL $0x04, DX
|
|
ADDL $0x04, SI
|
|
MOVQ src_len+32(FP), R8
|
|
SUBL DX, R8
|
|
LEAQ (BX)(DX*1), R9
|
|
LEAQ (BX)(SI*1), R10
|
|
|
|
// matchLen
|
|
XORL R12, R12
|
|
|
|
matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm64K:
|
|
CMPL R8, $0x10
|
|
JB matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm64K
|
|
MOVQ (R9)(R12*1), R11
|
|
MOVQ 8(R9)(R12*1), R13
|
|
XORQ (R10)(R12*1), R11
|
|
JNZ matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm64K
|
|
XORQ 8(R10)(R12*1), R13
|
|
JNZ matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm64K
|
|
LEAL -16(R8), R8
|
|
LEAL 16(R12), R12
|
|
JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm64K
|
|
|
|
matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm64K:
|
|
TZCNTQ R13, R13
|
|
SARQ $0x03, R13
|
|
LEAL 8(R12)(R13*1), R12
|
|
JMP match_nolit_end_encodeSnappyBetterBlockAsm64K
|
|
|
|
matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm64K:
|
|
CMPL R8, $0x08
|
|
JB matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm64K
|
|
MOVQ (R9)(R12*1), R11
|
|
XORQ (R10)(R12*1), R11
|
|
JNZ matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm64K
|
|
LEAL -8(R8), R8
|
|
LEAL 8(R12), R12
|
|
JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm64K
|
|
|
|
matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm64K:
|
|
TZCNTQ R11, R11
|
|
SARQ $0x03, R11
|
|
LEAL (R12)(R11*1), R12
|
|
JMP match_nolit_end_encodeSnappyBetterBlockAsm64K
|
|
|
|
matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm64K:
|
|
CMPL R8, $0x04
|
|
JB matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm64K
|
|
MOVL (R9)(R12*1), R11
|
|
CMPL (R10)(R12*1), R11
|
|
JNE matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm64K
|
|
LEAL -4(R8), R8
|
|
LEAL 4(R12), R12
|
|
|
|
matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm64K:
|
|
CMPL R8, $0x01
|
|
JE matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm64K
|
|
JB match_nolit_end_encodeSnappyBetterBlockAsm64K
|
|
MOVW (R9)(R12*1), R11
|
|
CMPW (R10)(R12*1), R11
|
|
JNE matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm64K
|
|
LEAL 2(R12), R12
|
|
SUBL $0x02, R8
|
|
JZ match_nolit_end_encodeSnappyBetterBlockAsm64K
|
|
|
|
matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm64K:
|
|
MOVB (R9)(R12*1), R11
|
|
CMPB (R10)(R12*1), R11
|
|
JNE match_nolit_end_encodeSnappyBetterBlockAsm64K
|
|
LEAL 1(R12), R12
|
|
|
|
match_nolit_end_encodeSnappyBetterBlockAsm64K:
|
|
MOVL DX, R8
|
|
SUBL SI, R8
|
|
|
|
// Check if repeat
|
|
MOVL R8, 16(SP)
|
|
MOVL 12(SP), SI
|
|
CMPL SI, DI
|
|
JEQ emit_literal_done_match_emit_encodeSnappyBetterBlockAsm64K
|
|
MOVL DI, R9
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(SI*1), R10
|
|
SUBL SI, R9
|
|
LEAL -1(R9), SI
|
|
CMPL SI, $0x3c
|
|
JB one_byte_match_emit_encodeSnappyBetterBlockAsm64K
|
|
CMPL SI, $0x00000100
|
|
JB two_bytes_match_emit_encodeSnappyBetterBlockAsm64K
|
|
JB three_bytes_match_emit_encodeSnappyBetterBlockAsm64K
|
|
|
|
three_bytes_match_emit_encodeSnappyBetterBlockAsm64K:
|
|
MOVB $0xf4, (CX)
|
|
MOVW SI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm64K
|
|
|
|
two_bytes_match_emit_encodeSnappyBetterBlockAsm64K:
|
|
MOVB $0xf0, (CX)
|
|
MOVB SI, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL SI, $0x40
|
|
JB memmove_match_emit_encodeSnappyBetterBlockAsm64K
|
|
JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm64K
|
|
|
|
one_byte_match_emit_encodeSnappyBetterBlockAsm64K:
|
|
SHLB $0x02, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_match_emit_encodeSnappyBetterBlockAsm64K:
|
|
LEAQ (CX)(R9*1), SI
|
|
|
|
// genMemMoveShort
|
|
CMPQ R9, $0x08
|
|
JBE emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_8
|
|
CMPQ R9, $0x10
|
|
JBE emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_8through16
|
|
CMPQ R9, $0x20
|
|
JBE emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_8:
|
|
MOVQ (R10), R11
|
|
MOVQ R11, (CX)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm64K
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_8through16:
|
|
MOVQ (R10), R11
|
|
MOVQ -8(R10)(R9*1), R10
|
|
MOVQ R11, (CX)
|
|
MOVQ R10, -8(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm64K
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_17through32:
|
|
MOVOU (R10), X0
|
|
MOVOU -16(R10)(R9*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm64K
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_33through64:
|
|
MOVOU (R10), X0
|
|
MOVOU 16(R10), X1
|
|
MOVOU -32(R10)(R9*1), X2
|
|
MOVOU -16(R10)(R9*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
|
|
memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm64K:
|
|
MOVQ SI, CX
|
|
JMP emit_literal_done_match_emit_encodeSnappyBetterBlockAsm64K
|
|
|
|
memmove_long_match_emit_encodeSnappyBetterBlockAsm64K:
|
|
LEAQ (CX)(R9*1), SI
|
|
|
|
// genMemMoveLong
|
|
MOVOU (R10), X0
|
|
MOVOU 16(R10), X1
|
|
MOVOU -32(R10)(R9*1), X2
|
|
MOVOU -16(R10)(R9*1), X3
|
|
MOVQ R9, R13
|
|
SHRQ $0x05, R13
|
|
MOVQ CX, R11
|
|
ANDL $0x0000001f, R11
|
|
MOVQ $0x00000040, R14
|
|
SUBQ R11, R14
|
|
DECQ R13
|
|
JA emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm64Klarge_forward_sse_loop_32
|
|
LEAQ -32(R10)(R14*1), R11
|
|
LEAQ -32(CX)(R14*1), R15
|
|
|
|
emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm64Klarge_big_loop_back:
|
|
MOVOU (R11), X4
|
|
MOVOU 16(R11), X5
|
|
MOVOA X4, (R15)
|
|
MOVOA X5, 16(R15)
|
|
ADDQ $0x20, R15
|
|
ADDQ $0x20, R11
|
|
ADDQ $0x20, R14
|
|
DECQ R13
|
|
JNA emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm64Klarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm64Klarge_forward_sse_loop_32:
|
|
MOVOU -32(R10)(R14*1), X4
|
|
MOVOU -16(R10)(R14*1), X5
|
|
MOVOA X4, -32(CX)(R14*1)
|
|
MOVOA X5, -16(CX)(R14*1)
|
|
ADDQ $0x20, R14
|
|
CMPQ R9, R14
|
|
JAE emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm64Klarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
MOVQ SI, CX
|
|
|
|
emit_literal_done_match_emit_encodeSnappyBetterBlockAsm64K:
|
|
ADDL R12, DX
|
|
ADDL $0x04, R12
|
|
MOVL DX, 12(SP)
|
|
|
|
// emitCopy
|
|
two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm64K:
|
|
CMPL R12, $0x40
|
|
JBE two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm64K
|
|
MOVB $0xee, (CX)
|
|
MOVW R8, 1(CX)
|
|
LEAL -60(R12), R12
|
|
ADDQ $0x03, CX
|
|
JMP two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm64K
|
|
|
|
two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm64K:
|
|
MOVL R12, SI
|
|
SHLL $0x02, SI
|
|
CMPL R12, $0x0c
|
|
JAE emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm64K
|
|
CMPL R8, $0x00000800
|
|
JAE emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm64K
|
|
LEAL -15(SI), SI
|
|
MOVB R8, 1(CX)
|
|
SHRL $0x08, R8
|
|
SHLL $0x05, R8
|
|
ORL R8, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm64K
|
|
|
|
emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm64K:
|
|
LEAL -2(SI), SI
|
|
MOVB SI, (CX)
|
|
MOVW R8, 1(CX)
|
|
ADDQ $0x03, CX
|
|
|
|
match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm64K:
|
|
CMPL DX, 8(SP)
|
|
JAE emit_remainder_encodeSnappyBetterBlockAsm64K
|
|
CMPQ CX, (SP)
|
|
JB match_nolit_dst_ok_encodeSnappyBetterBlockAsm64K
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeSnappyBetterBlockAsm64K:
|
|
MOVQ $0x00cf1bbcdcbfa563, SI
|
|
MOVQ $0x9e3779b1, R8
|
|
LEAQ 1(DI), DI
|
|
LEAQ -2(DX), R9
|
|
MOVQ (BX)(DI*1), R10
|
|
MOVQ 1(BX)(DI*1), R11
|
|
MOVQ (BX)(R9*1), R12
|
|
MOVQ 1(BX)(R9*1), R13
|
|
SHLQ $0x08, R10
|
|
IMULQ SI, R10
|
|
SHRQ $0x30, R10
|
|
SHLQ $0x20, R11
|
|
IMULQ R8, R11
|
|
SHRQ $0x33, R11
|
|
SHLQ $0x08, R12
|
|
IMULQ SI, R12
|
|
SHRQ $0x30, R12
|
|
SHLQ $0x20, R13
|
|
IMULQ R8, R13
|
|
SHRQ $0x33, R13
|
|
LEAQ 1(DI), R8
|
|
LEAQ 1(R9), R14
|
|
MOVL DI, (AX)(R10*4)
|
|
MOVL R9, (AX)(R12*4)
|
|
MOVL R8, 262144(AX)(R11*4)
|
|
MOVL R14, 262144(AX)(R13*4)
|
|
LEAQ 1(R9)(DI*1), R8
|
|
SHRQ $0x01, R8
|
|
ADDQ $0x01, DI
|
|
SUBQ $0x01, R9
|
|
|
|
index_loop_encodeSnappyBetterBlockAsm64K:
|
|
CMPQ R8, R9
|
|
JAE search_loop_encodeSnappyBetterBlockAsm64K
|
|
MOVQ (BX)(DI*1), R10
|
|
MOVQ (BX)(R8*1), R11
|
|
SHLQ $0x08, R10
|
|
IMULQ SI, R10
|
|
SHRQ $0x30, R10
|
|
SHLQ $0x08, R11
|
|
IMULQ SI, R11
|
|
SHRQ $0x30, R11
|
|
MOVL DI, (AX)(R10*4)
|
|
MOVL R8, (AX)(R11*4)
|
|
ADDQ $0x02, DI
|
|
ADDQ $0x02, R8
|
|
JMP index_loop_encodeSnappyBetterBlockAsm64K
|
|
|
|
emit_remainder_encodeSnappyBetterBlockAsm64K:
|
|
MOVQ src_len+32(FP), AX
|
|
SUBL 12(SP), AX
|
|
LEAQ 3(CX)(AX*1), AX
|
|
CMPQ AX, (SP)
|
|
JB emit_remainder_ok_encodeSnappyBetterBlockAsm64K
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeSnappyBetterBlockAsm64K:
|
|
MOVQ src_len+32(FP), AX
|
|
MOVL 12(SP), DX
|
|
CMPL DX, AX
|
|
JEQ emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm64K
|
|
MOVL AX, SI
|
|
MOVL AX, 12(SP)
|
|
LEAQ (BX)(DX*1), AX
|
|
SUBL DX, SI
|
|
LEAL -1(SI), DX
|
|
CMPL DX, $0x3c
|
|
JB one_byte_emit_remainder_encodeSnappyBetterBlockAsm64K
|
|
CMPL DX, $0x00000100
|
|
JB two_bytes_emit_remainder_encodeSnappyBetterBlockAsm64K
|
|
JB three_bytes_emit_remainder_encodeSnappyBetterBlockAsm64K
|
|
|
|
three_bytes_emit_remainder_encodeSnappyBetterBlockAsm64K:
|
|
MOVB $0xf4, (CX)
|
|
MOVW DX, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64K
|
|
|
|
two_bytes_emit_remainder_encodeSnappyBetterBlockAsm64K:
|
|
MOVB $0xf0, (CX)
|
|
MOVB DL, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL DX, $0x40
|
|
JB memmove_emit_remainder_encodeSnappyBetterBlockAsm64K
|
|
JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64K
|
|
|
|
one_byte_emit_remainder_encodeSnappyBetterBlockAsm64K:
|
|
SHLB $0x02, DL
|
|
MOVB DL, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_emit_remainder_encodeSnappyBetterBlockAsm64K:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveShort
|
|
CMPQ BX, $0x03
|
|
JB emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_1or2
|
|
JE emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_3
|
|
CMPQ BX, $0x08
|
|
JB emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_4through7
|
|
CMPQ BX, $0x10
|
|
JBE emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_8through16
|
|
CMPQ BX, $0x20
|
|
JBE emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_1or2:
|
|
MOVB (AX), SI
|
|
MOVB -1(AX)(BX*1), AL
|
|
MOVB SI, (CX)
|
|
MOVB AL, -1(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm64K
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_3:
|
|
MOVW (AX), SI
|
|
MOVB 2(AX), AL
|
|
MOVW SI, (CX)
|
|
MOVB AL, 2(CX)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm64K
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_4through7:
|
|
MOVL (AX), SI
|
|
MOVL -4(AX)(BX*1), AX
|
|
MOVL SI, (CX)
|
|
MOVL AX, -4(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm64K
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_8through16:
|
|
MOVQ (AX), SI
|
|
MOVQ -8(AX)(BX*1), AX
|
|
MOVQ SI, (CX)
|
|
MOVQ AX, -8(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm64K
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_17through32:
|
|
MOVOU (AX), X0
|
|
MOVOU -16(AX)(BX*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm64K
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_33through64:
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
|
|
memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm64K:
|
|
MOVQ DX, CX
|
|
JMP emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm64K
|
|
|
|
memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64K:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveLong
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVQ BX, DI
|
|
SHRQ $0x05, DI
|
|
MOVQ CX, SI
|
|
ANDL $0x0000001f, SI
|
|
MOVQ $0x00000040, R8
|
|
SUBQ SI, R8
|
|
DECQ DI
|
|
JA emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64Klarge_forward_sse_loop_32
|
|
LEAQ -32(AX)(R8*1), SI
|
|
LEAQ -32(CX)(R8*1), R9
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64Klarge_big_loop_back:
|
|
MOVOU (SI), X4
|
|
MOVOU 16(SI), X5
|
|
MOVOA X4, (R9)
|
|
MOVOA X5, 16(R9)
|
|
ADDQ $0x20, R9
|
|
ADDQ $0x20, SI
|
|
ADDQ $0x20, R8
|
|
DECQ DI
|
|
JNA emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64Klarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64Klarge_forward_sse_loop_32:
|
|
MOVOU -32(AX)(R8*1), X4
|
|
MOVOU -16(AX)(R8*1), X5
|
|
MOVOA X4, -32(CX)(R8*1)
|
|
MOVOA X5, -16(CX)(R8*1)
|
|
ADDQ $0x20, R8
|
|
CMPQ BX, R8
|
|
JAE emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64Klarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
MOVQ DX, CX
|
|
|
|
emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm64K:
|
|
MOVQ dst_base+0(FP), AX
|
|
SUBQ AX, CX
|
|
MOVQ CX, ret+56(FP)
|
|
RET
|
|
|
|
// func encodeSnappyBetterBlockAsm12B(dst []byte, src []byte, tmp *[81920]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeSnappyBetterBlockAsm12B(SB), $24-64
|
|
MOVQ tmp+48(FP), AX
|
|
MOVQ dst_base+0(FP), CX
|
|
MOVQ $0x00000280, DX
|
|
MOVQ AX, BX
|
|
PXOR X0, X0
|
|
|
|
zero_loop_encodeSnappyBetterBlockAsm12B:
|
|
MOVOU X0, (BX)
|
|
MOVOU X0, 16(BX)
|
|
MOVOU X0, 32(BX)
|
|
MOVOU X0, 48(BX)
|
|
MOVOU X0, 64(BX)
|
|
MOVOU X0, 80(BX)
|
|
MOVOU X0, 96(BX)
|
|
MOVOU X0, 112(BX)
|
|
ADDQ $0x80, BX
|
|
DECQ DX
|
|
JNZ zero_loop_encodeSnappyBetterBlockAsm12B
|
|
MOVL $0x00000000, 12(SP)
|
|
MOVQ src_len+32(FP), DX
|
|
LEAQ -9(DX), BX
|
|
LEAQ -8(DX), SI
|
|
MOVL SI, 8(SP)
|
|
SHRQ $0x05, DX
|
|
SUBL DX, BX
|
|
LEAQ (CX)(BX*1), BX
|
|
MOVQ BX, (SP)
|
|
MOVL $0x00000001, DX
|
|
MOVL $0x00000000, 16(SP)
|
|
MOVQ src_base+24(FP), BX
|
|
|
|
search_loop_encodeSnappyBetterBlockAsm12B:
|
|
MOVL DX, SI
|
|
SUBL 12(SP), SI
|
|
SHRL $0x06, SI
|
|
LEAL 1(DX)(SI*1), SI
|
|
CMPL SI, 8(SP)
|
|
JAE emit_remainder_encodeSnappyBetterBlockAsm12B
|
|
MOVQ (BX)(DX*1), DI
|
|
MOVL SI, 20(SP)
|
|
MOVQ $0x0000cf1bbcdcbf9b, R9
|
|
MOVQ $0x9e3779b1, SI
|
|
MOVQ DI, R10
|
|
MOVQ DI, R11
|
|
SHLQ $0x10, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x32, R10
|
|
SHLQ $0x20, R11
|
|
IMULQ SI, R11
|
|
SHRQ $0x34, R11
|
|
MOVL (AX)(R10*4), SI
|
|
MOVL 65536(AX)(R11*4), R8
|
|
MOVL DX, (AX)(R10*4)
|
|
MOVL DX, 65536(AX)(R11*4)
|
|
MOVQ (BX)(SI*1), R10
|
|
MOVQ (BX)(R8*1), R11
|
|
CMPQ R10, DI
|
|
JEQ candidate_match_encodeSnappyBetterBlockAsm12B
|
|
CMPQ R11, DI
|
|
JNE no_short_found_encodeSnappyBetterBlockAsm12B
|
|
MOVL R8, SI
|
|
JMP candidate_match_encodeSnappyBetterBlockAsm12B
|
|
|
|
no_short_found_encodeSnappyBetterBlockAsm12B:
|
|
CMPL R10, DI
|
|
JEQ candidate_match_encodeSnappyBetterBlockAsm12B
|
|
CMPL R11, DI
|
|
JEQ candidateS_match_encodeSnappyBetterBlockAsm12B
|
|
MOVL 20(SP), DX
|
|
JMP search_loop_encodeSnappyBetterBlockAsm12B
|
|
|
|
candidateS_match_encodeSnappyBetterBlockAsm12B:
|
|
SHRQ $0x08, DI
|
|
MOVQ DI, R10
|
|
SHLQ $0x10, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x32, R10
|
|
MOVL (AX)(R10*4), SI
|
|
INCL DX
|
|
MOVL DX, (AX)(R10*4)
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ candidate_match_encodeSnappyBetterBlockAsm12B
|
|
DECL DX
|
|
MOVL R8, SI
|
|
|
|
candidate_match_encodeSnappyBetterBlockAsm12B:
|
|
MOVL 12(SP), DI
|
|
TESTL SI, SI
|
|
JZ match_extend_back_end_encodeSnappyBetterBlockAsm12B
|
|
|
|
match_extend_back_loop_encodeSnappyBetterBlockAsm12B:
|
|
CMPL DX, DI
|
|
JBE match_extend_back_end_encodeSnappyBetterBlockAsm12B
|
|
MOVB -1(BX)(SI*1), R8
|
|
MOVB -1(BX)(DX*1), R9
|
|
CMPB R8, R9
|
|
JNE match_extend_back_end_encodeSnappyBetterBlockAsm12B
|
|
LEAL -1(DX), DX
|
|
DECL SI
|
|
JZ match_extend_back_end_encodeSnappyBetterBlockAsm12B
|
|
JMP match_extend_back_loop_encodeSnappyBetterBlockAsm12B
|
|
|
|
match_extend_back_end_encodeSnappyBetterBlockAsm12B:
|
|
MOVL DX, DI
|
|
SUBL 12(SP), DI
|
|
LEAQ 3(CX)(DI*1), DI
|
|
CMPQ DI, (SP)
|
|
JB match_dst_size_check_encodeSnappyBetterBlockAsm12B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeSnappyBetterBlockAsm12B:
|
|
MOVL DX, DI
|
|
ADDL $0x04, DX
|
|
ADDL $0x04, SI
|
|
MOVQ src_len+32(FP), R8
|
|
SUBL DX, R8
|
|
LEAQ (BX)(DX*1), R9
|
|
LEAQ (BX)(SI*1), R10
|
|
|
|
// matchLen
|
|
XORL R12, R12
|
|
|
|
matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm12B:
|
|
CMPL R8, $0x10
|
|
JB matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm12B
|
|
MOVQ (R9)(R12*1), R11
|
|
MOVQ 8(R9)(R12*1), R13
|
|
XORQ (R10)(R12*1), R11
|
|
JNZ matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm12B
|
|
XORQ 8(R10)(R12*1), R13
|
|
JNZ matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm12B
|
|
LEAL -16(R8), R8
|
|
LEAL 16(R12), R12
|
|
JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm12B
|
|
|
|
matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm12B:
|
|
TZCNTQ R13, R13
|
|
SARQ $0x03, R13
|
|
LEAL 8(R12)(R13*1), R12
|
|
JMP match_nolit_end_encodeSnappyBetterBlockAsm12B
|
|
|
|
matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm12B:
|
|
CMPL R8, $0x08
|
|
JB matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm12B
|
|
MOVQ (R9)(R12*1), R11
|
|
XORQ (R10)(R12*1), R11
|
|
JNZ matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm12B
|
|
LEAL -8(R8), R8
|
|
LEAL 8(R12), R12
|
|
JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm12B
|
|
|
|
matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm12B:
|
|
TZCNTQ R11, R11
|
|
SARQ $0x03, R11
|
|
LEAL (R12)(R11*1), R12
|
|
JMP match_nolit_end_encodeSnappyBetterBlockAsm12B
|
|
|
|
matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm12B:
|
|
CMPL R8, $0x04
|
|
JB matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm12B
|
|
MOVL (R9)(R12*1), R11
|
|
CMPL (R10)(R12*1), R11
|
|
JNE matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm12B
|
|
LEAL -4(R8), R8
|
|
LEAL 4(R12), R12
|
|
|
|
matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm12B:
|
|
CMPL R8, $0x01
|
|
JE matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm12B
|
|
JB match_nolit_end_encodeSnappyBetterBlockAsm12B
|
|
MOVW (R9)(R12*1), R11
|
|
CMPW (R10)(R12*1), R11
|
|
JNE matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm12B
|
|
LEAL 2(R12), R12
|
|
SUBL $0x02, R8
|
|
JZ match_nolit_end_encodeSnappyBetterBlockAsm12B
|
|
|
|
matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm12B:
|
|
MOVB (R9)(R12*1), R11
|
|
CMPB (R10)(R12*1), R11
|
|
JNE match_nolit_end_encodeSnappyBetterBlockAsm12B
|
|
LEAL 1(R12), R12
|
|
|
|
match_nolit_end_encodeSnappyBetterBlockAsm12B:
|
|
MOVL DX, R8
|
|
SUBL SI, R8
|
|
|
|
// Check if repeat
|
|
MOVL R8, 16(SP)
|
|
MOVL 12(SP), SI
|
|
CMPL SI, DI
|
|
JEQ emit_literal_done_match_emit_encodeSnappyBetterBlockAsm12B
|
|
MOVL DI, R9
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(SI*1), R10
|
|
SUBL SI, R9
|
|
LEAL -1(R9), SI
|
|
CMPL SI, $0x3c
|
|
JB one_byte_match_emit_encodeSnappyBetterBlockAsm12B
|
|
CMPL SI, $0x00000100
|
|
JB two_bytes_match_emit_encodeSnappyBetterBlockAsm12B
|
|
JB three_bytes_match_emit_encodeSnappyBetterBlockAsm12B
|
|
|
|
three_bytes_match_emit_encodeSnappyBetterBlockAsm12B:
|
|
MOVB $0xf4, (CX)
|
|
MOVW SI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm12B
|
|
|
|
two_bytes_match_emit_encodeSnappyBetterBlockAsm12B:
|
|
MOVB $0xf0, (CX)
|
|
MOVB SI, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL SI, $0x40
|
|
JB memmove_match_emit_encodeSnappyBetterBlockAsm12B
|
|
JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm12B
|
|
|
|
one_byte_match_emit_encodeSnappyBetterBlockAsm12B:
|
|
SHLB $0x02, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_match_emit_encodeSnappyBetterBlockAsm12B:
|
|
LEAQ (CX)(R9*1), SI
|
|
|
|
// genMemMoveShort
|
|
CMPQ R9, $0x08
|
|
JBE emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_8
|
|
CMPQ R9, $0x10
|
|
JBE emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_8through16
|
|
CMPQ R9, $0x20
|
|
JBE emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_8:
|
|
MOVQ (R10), R11
|
|
MOVQ R11, (CX)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_8through16:
|
|
MOVQ (R10), R11
|
|
MOVQ -8(R10)(R9*1), R10
|
|
MOVQ R11, (CX)
|
|
MOVQ R10, -8(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_17through32:
|
|
MOVOU (R10), X0
|
|
MOVOU -16(R10)(R9*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_33through64:
|
|
MOVOU (R10), X0
|
|
MOVOU 16(R10), X1
|
|
MOVOU -32(R10)(R9*1), X2
|
|
MOVOU -16(R10)(R9*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
|
|
memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm12B:
|
|
MOVQ SI, CX
|
|
JMP emit_literal_done_match_emit_encodeSnappyBetterBlockAsm12B
|
|
|
|
memmove_long_match_emit_encodeSnappyBetterBlockAsm12B:
|
|
LEAQ (CX)(R9*1), SI
|
|
|
|
// genMemMoveLong
|
|
MOVOU (R10), X0
|
|
MOVOU 16(R10), X1
|
|
MOVOU -32(R10)(R9*1), X2
|
|
MOVOU -16(R10)(R9*1), X3
|
|
MOVQ R9, R13
|
|
SHRQ $0x05, R13
|
|
MOVQ CX, R11
|
|
ANDL $0x0000001f, R11
|
|
MOVQ $0x00000040, R14
|
|
SUBQ R11, R14
|
|
DECQ R13
|
|
JA emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm12Blarge_forward_sse_loop_32
|
|
LEAQ -32(R10)(R14*1), R11
|
|
LEAQ -32(CX)(R14*1), R15
|
|
|
|
emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm12Blarge_big_loop_back:
|
|
MOVOU (R11), X4
|
|
MOVOU 16(R11), X5
|
|
MOVOA X4, (R15)
|
|
MOVOA X5, 16(R15)
|
|
ADDQ $0x20, R15
|
|
ADDQ $0x20, R11
|
|
ADDQ $0x20, R14
|
|
DECQ R13
|
|
JNA emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm12Blarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm12Blarge_forward_sse_loop_32:
|
|
MOVOU -32(R10)(R14*1), X4
|
|
MOVOU -16(R10)(R14*1), X5
|
|
MOVOA X4, -32(CX)(R14*1)
|
|
MOVOA X5, -16(CX)(R14*1)
|
|
ADDQ $0x20, R14
|
|
CMPQ R9, R14
|
|
JAE emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm12Blarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
MOVQ SI, CX
|
|
|
|
emit_literal_done_match_emit_encodeSnappyBetterBlockAsm12B:
|
|
ADDL R12, DX
|
|
ADDL $0x04, R12
|
|
MOVL DX, 12(SP)
|
|
|
|
// emitCopy
|
|
two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm12B:
|
|
CMPL R12, $0x40
|
|
JBE two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm12B
|
|
MOVB $0xee, (CX)
|
|
MOVW R8, 1(CX)
|
|
LEAL -60(R12), R12
|
|
ADDQ $0x03, CX
|
|
JMP two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm12B
|
|
|
|
two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm12B:
|
|
MOVL R12, SI
|
|
SHLL $0x02, SI
|
|
CMPL R12, $0x0c
|
|
JAE emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm12B
|
|
CMPL R8, $0x00000800
|
|
JAE emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm12B
|
|
LEAL -15(SI), SI
|
|
MOVB R8, 1(CX)
|
|
SHRL $0x08, R8
|
|
SHLL $0x05, R8
|
|
ORL R8, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm12B
|
|
|
|
emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm12B:
|
|
LEAL -2(SI), SI
|
|
MOVB SI, (CX)
|
|
MOVW R8, 1(CX)
|
|
ADDQ $0x03, CX
|
|
|
|
match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm12B:
|
|
CMPL DX, 8(SP)
|
|
JAE emit_remainder_encodeSnappyBetterBlockAsm12B
|
|
CMPQ CX, (SP)
|
|
JB match_nolit_dst_ok_encodeSnappyBetterBlockAsm12B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeSnappyBetterBlockAsm12B:
|
|
MOVQ $0x0000cf1bbcdcbf9b, SI
|
|
MOVQ $0x9e3779b1, R8
|
|
LEAQ 1(DI), DI
|
|
LEAQ -2(DX), R9
|
|
MOVQ (BX)(DI*1), R10
|
|
MOVQ 1(BX)(DI*1), R11
|
|
MOVQ (BX)(R9*1), R12
|
|
MOVQ 1(BX)(R9*1), R13
|
|
SHLQ $0x10, R10
|
|
IMULQ SI, R10
|
|
SHRQ $0x32, R10
|
|
SHLQ $0x20, R11
|
|
IMULQ R8, R11
|
|
SHRQ $0x34, R11
|
|
SHLQ $0x10, R12
|
|
IMULQ SI, R12
|
|
SHRQ $0x32, R12
|
|
SHLQ $0x20, R13
|
|
IMULQ R8, R13
|
|
SHRQ $0x34, R13
|
|
LEAQ 1(DI), R8
|
|
LEAQ 1(R9), R14
|
|
MOVL DI, (AX)(R10*4)
|
|
MOVL R9, (AX)(R12*4)
|
|
MOVL R8, 65536(AX)(R11*4)
|
|
MOVL R14, 65536(AX)(R13*4)
|
|
LEAQ 1(R9)(DI*1), R8
|
|
SHRQ $0x01, R8
|
|
ADDQ $0x01, DI
|
|
SUBQ $0x01, R9
|
|
|
|
index_loop_encodeSnappyBetterBlockAsm12B:
|
|
CMPQ R8, R9
|
|
JAE search_loop_encodeSnappyBetterBlockAsm12B
|
|
MOVQ (BX)(DI*1), R10
|
|
MOVQ (BX)(R8*1), R11
|
|
SHLQ $0x10, R10
|
|
IMULQ SI, R10
|
|
SHRQ $0x32, R10
|
|
SHLQ $0x10, R11
|
|
IMULQ SI, R11
|
|
SHRQ $0x32, R11
|
|
MOVL DI, (AX)(R10*4)
|
|
MOVL R8, (AX)(R11*4)
|
|
ADDQ $0x02, DI
|
|
ADDQ $0x02, R8
|
|
JMP index_loop_encodeSnappyBetterBlockAsm12B
|
|
|
|
emit_remainder_encodeSnappyBetterBlockAsm12B:
|
|
MOVQ src_len+32(FP), AX
|
|
SUBL 12(SP), AX
|
|
LEAQ 3(CX)(AX*1), AX
|
|
CMPQ AX, (SP)
|
|
JB emit_remainder_ok_encodeSnappyBetterBlockAsm12B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeSnappyBetterBlockAsm12B:
|
|
MOVQ src_len+32(FP), AX
|
|
MOVL 12(SP), DX
|
|
CMPL DX, AX
|
|
JEQ emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm12B
|
|
MOVL AX, SI
|
|
MOVL AX, 12(SP)
|
|
LEAQ (BX)(DX*1), AX
|
|
SUBL DX, SI
|
|
LEAL -1(SI), DX
|
|
CMPL DX, $0x3c
|
|
JB one_byte_emit_remainder_encodeSnappyBetterBlockAsm12B
|
|
CMPL DX, $0x00000100
|
|
JB two_bytes_emit_remainder_encodeSnappyBetterBlockAsm12B
|
|
JB three_bytes_emit_remainder_encodeSnappyBetterBlockAsm12B
|
|
|
|
three_bytes_emit_remainder_encodeSnappyBetterBlockAsm12B:
|
|
MOVB $0xf4, (CX)
|
|
MOVW DX, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12B
|
|
|
|
two_bytes_emit_remainder_encodeSnappyBetterBlockAsm12B:
|
|
MOVB $0xf0, (CX)
|
|
MOVB DL, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL DX, $0x40
|
|
JB memmove_emit_remainder_encodeSnappyBetterBlockAsm12B
|
|
JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12B
|
|
|
|
one_byte_emit_remainder_encodeSnappyBetterBlockAsm12B:
|
|
SHLB $0x02, DL
|
|
MOVB DL, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_emit_remainder_encodeSnappyBetterBlockAsm12B:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveShort
|
|
CMPQ BX, $0x03
|
|
JB emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_1or2
|
|
JE emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_3
|
|
CMPQ BX, $0x08
|
|
JB emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_4through7
|
|
CMPQ BX, $0x10
|
|
JBE emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_8through16
|
|
CMPQ BX, $0x20
|
|
JBE emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_1or2:
|
|
MOVB (AX), SI
|
|
MOVB -1(AX)(BX*1), AL
|
|
MOVB SI, (CX)
|
|
MOVB AL, -1(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_3:
|
|
MOVW (AX), SI
|
|
MOVB 2(AX), AL
|
|
MOVW SI, (CX)
|
|
MOVB AL, 2(CX)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_4through7:
|
|
MOVL (AX), SI
|
|
MOVL -4(AX)(BX*1), AX
|
|
MOVL SI, (CX)
|
|
MOVL AX, -4(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_8through16:
|
|
MOVQ (AX), SI
|
|
MOVQ -8(AX)(BX*1), AX
|
|
MOVQ SI, (CX)
|
|
MOVQ AX, -8(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_17through32:
|
|
MOVOU (AX), X0
|
|
MOVOU -16(AX)(BX*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_33through64:
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
|
|
memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm12B:
|
|
MOVQ DX, CX
|
|
JMP emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm12B
|
|
|
|
memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12B:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveLong
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVQ BX, DI
|
|
SHRQ $0x05, DI
|
|
MOVQ CX, SI
|
|
ANDL $0x0000001f, SI
|
|
MOVQ $0x00000040, R8
|
|
SUBQ SI, R8
|
|
DECQ DI
|
|
JA emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12Blarge_forward_sse_loop_32
|
|
LEAQ -32(AX)(R8*1), SI
|
|
LEAQ -32(CX)(R8*1), R9
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12Blarge_big_loop_back:
|
|
MOVOU (SI), X4
|
|
MOVOU 16(SI), X5
|
|
MOVOA X4, (R9)
|
|
MOVOA X5, 16(R9)
|
|
ADDQ $0x20, R9
|
|
ADDQ $0x20, SI
|
|
ADDQ $0x20, R8
|
|
DECQ DI
|
|
JNA emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12Blarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12Blarge_forward_sse_loop_32:
|
|
MOVOU -32(AX)(R8*1), X4
|
|
MOVOU -16(AX)(R8*1), X5
|
|
MOVOA X4, -32(CX)(R8*1)
|
|
MOVOA X5, -16(CX)(R8*1)
|
|
ADDQ $0x20, R8
|
|
CMPQ BX, R8
|
|
JAE emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12Blarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
MOVQ DX, CX
|
|
|
|
emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm12B:
|
|
MOVQ dst_base+0(FP), AX
|
|
SUBQ AX, CX
|
|
MOVQ CX, ret+56(FP)
|
|
RET
|
|
|
|
// func encodeSnappyBetterBlockAsm10B(dst []byte, src []byte, tmp *[20480]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeSnappyBetterBlockAsm10B(SB), $24-64
|
|
MOVQ tmp+48(FP), AX
|
|
MOVQ dst_base+0(FP), CX
|
|
MOVQ $0x000000a0, DX
|
|
MOVQ AX, BX
|
|
PXOR X0, X0
|
|
|
|
zero_loop_encodeSnappyBetterBlockAsm10B:
|
|
MOVOU X0, (BX)
|
|
MOVOU X0, 16(BX)
|
|
MOVOU X0, 32(BX)
|
|
MOVOU X0, 48(BX)
|
|
MOVOU X0, 64(BX)
|
|
MOVOU X0, 80(BX)
|
|
MOVOU X0, 96(BX)
|
|
MOVOU X0, 112(BX)
|
|
ADDQ $0x80, BX
|
|
DECQ DX
|
|
JNZ zero_loop_encodeSnappyBetterBlockAsm10B
|
|
MOVL $0x00000000, 12(SP)
|
|
MOVQ src_len+32(FP), DX
|
|
LEAQ -9(DX), BX
|
|
LEAQ -8(DX), SI
|
|
MOVL SI, 8(SP)
|
|
SHRQ $0x05, DX
|
|
SUBL DX, BX
|
|
LEAQ (CX)(BX*1), BX
|
|
MOVQ BX, (SP)
|
|
MOVL $0x00000001, DX
|
|
MOVL $0x00000000, 16(SP)
|
|
MOVQ src_base+24(FP), BX
|
|
|
|
search_loop_encodeSnappyBetterBlockAsm10B:
|
|
MOVL DX, SI
|
|
SUBL 12(SP), SI
|
|
SHRL $0x05, SI
|
|
LEAL 1(DX)(SI*1), SI
|
|
CMPL SI, 8(SP)
|
|
JAE emit_remainder_encodeSnappyBetterBlockAsm10B
|
|
MOVQ (BX)(DX*1), DI
|
|
MOVL SI, 20(SP)
|
|
MOVQ $0x0000cf1bbcdcbf9b, R9
|
|
MOVQ $0x9e3779b1, SI
|
|
MOVQ DI, R10
|
|
MOVQ DI, R11
|
|
SHLQ $0x10, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x34, R10
|
|
SHLQ $0x20, R11
|
|
IMULQ SI, R11
|
|
SHRQ $0x36, R11
|
|
MOVL (AX)(R10*4), SI
|
|
MOVL 16384(AX)(R11*4), R8
|
|
MOVL DX, (AX)(R10*4)
|
|
MOVL DX, 16384(AX)(R11*4)
|
|
MOVQ (BX)(SI*1), R10
|
|
MOVQ (BX)(R8*1), R11
|
|
CMPQ R10, DI
|
|
JEQ candidate_match_encodeSnappyBetterBlockAsm10B
|
|
CMPQ R11, DI
|
|
JNE no_short_found_encodeSnappyBetterBlockAsm10B
|
|
MOVL R8, SI
|
|
JMP candidate_match_encodeSnappyBetterBlockAsm10B
|
|
|
|
no_short_found_encodeSnappyBetterBlockAsm10B:
|
|
CMPL R10, DI
|
|
JEQ candidate_match_encodeSnappyBetterBlockAsm10B
|
|
CMPL R11, DI
|
|
JEQ candidateS_match_encodeSnappyBetterBlockAsm10B
|
|
MOVL 20(SP), DX
|
|
JMP search_loop_encodeSnappyBetterBlockAsm10B
|
|
|
|
candidateS_match_encodeSnappyBetterBlockAsm10B:
|
|
SHRQ $0x08, DI
|
|
MOVQ DI, R10
|
|
SHLQ $0x10, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x34, R10
|
|
MOVL (AX)(R10*4), SI
|
|
INCL DX
|
|
MOVL DX, (AX)(R10*4)
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ candidate_match_encodeSnappyBetterBlockAsm10B
|
|
DECL DX
|
|
MOVL R8, SI
|
|
|
|
candidate_match_encodeSnappyBetterBlockAsm10B:
|
|
MOVL 12(SP), DI
|
|
TESTL SI, SI
|
|
JZ match_extend_back_end_encodeSnappyBetterBlockAsm10B
|
|
|
|
match_extend_back_loop_encodeSnappyBetterBlockAsm10B:
|
|
CMPL DX, DI
|
|
JBE match_extend_back_end_encodeSnappyBetterBlockAsm10B
|
|
MOVB -1(BX)(SI*1), R8
|
|
MOVB -1(BX)(DX*1), R9
|
|
CMPB R8, R9
|
|
JNE match_extend_back_end_encodeSnappyBetterBlockAsm10B
|
|
LEAL -1(DX), DX
|
|
DECL SI
|
|
JZ match_extend_back_end_encodeSnappyBetterBlockAsm10B
|
|
JMP match_extend_back_loop_encodeSnappyBetterBlockAsm10B
|
|
|
|
match_extend_back_end_encodeSnappyBetterBlockAsm10B:
|
|
MOVL DX, DI
|
|
SUBL 12(SP), DI
|
|
LEAQ 3(CX)(DI*1), DI
|
|
CMPQ DI, (SP)
|
|
JB match_dst_size_check_encodeSnappyBetterBlockAsm10B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeSnappyBetterBlockAsm10B:
|
|
MOVL DX, DI
|
|
ADDL $0x04, DX
|
|
ADDL $0x04, SI
|
|
MOVQ src_len+32(FP), R8
|
|
SUBL DX, R8
|
|
LEAQ (BX)(DX*1), R9
|
|
LEAQ (BX)(SI*1), R10
|
|
|
|
// matchLen
|
|
XORL R12, R12
|
|
|
|
matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm10B:
|
|
CMPL R8, $0x10
|
|
JB matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm10B
|
|
MOVQ (R9)(R12*1), R11
|
|
MOVQ 8(R9)(R12*1), R13
|
|
XORQ (R10)(R12*1), R11
|
|
JNZ matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm10B
|
|
XORQ 8(R10)(R12*1), R13
|
|
JNZ matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm10B
|
|
LEAL -16(R8), R8
|
|
LEAL 16(R12), R12
|
|
JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm10B
|
|
|
|
matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm10B:
|
|
TZCNTQ R13, R13
|
|
SARQ $0x03, R13
|
|
LEAL 8(R12)(R13*1), R12
|
|
JMP match_nolit_end_encodeSnappyBetterBlockAsm10B
|
|
|
|
matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm10B:
|
|
CMPL R8, $0x08
|
|
JB matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm10B
|
|
MOVQ (R9)(R12*1), R11
|
|
XORQ (R10)(R12*1), R11
|
|
JNZ matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm10B
|
|
LEAL -8(R8), R8
|
|
LEAL 8(R12), R12
|
|
JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm10B
|
|
|
|
matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm10B:
|
|
TZCNTQ R11, R11
|
|
SARQ $0x03, R11
|
|
LEAL (R12)(R11*1), R12
|
|
JMP match_nolit_end_encodeSnappyBetterBlockAsm10B
|
|
|
|
matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm10B:
|
|
CMPL R8, $0x04
|
|
JB matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm10B
|
|
MOVL (R9)(R12*1), R11
|
|
CMPL (R10)(R12*1), R11
|
|
JNE matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm10B
|
|
LEAL -4(R8), R8
|
|
LEAL 4(R12), R12
|
|
|
|
matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm10B:
|
|
CMPL R8, $0x01
|
|
JE matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm10B
|
|
JB match_nolit_end_encodeSnappyBetterBlockAsm10B
|
|
MOVW (R9)(R12*1), R11
|
|
CMPW (R10)(R12*1), R11
|
|
JNE matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm10B
|
|
LEAL 2(R12), R12
|
|
SUBL $0x02, R8
|
|
JZ match_nolit_end_encodeSnappyBetterBlockAsm10B
|
|
|
|
matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm10B:
|
|
MOVB (R9)(R12*1), R11
|
|
CMPB (R10)(R12*1), R11
|
|
JNE match_nolit_end_encodeSnappyBetterBlockAsm10B
|
|
LEAL 1(R12), R12
|
|
|
|
match_nolit_end_encodeSnappyBetterBlockAsm10B:
|
|
MOVL DX, R8
|
|
SUBL SI, R8
|
|
|
|
// Check if repeat
|
|
MOVL R8, 16(SP)
|
|
MOVL 12(SP), SI
|
|
CMPL SI, DI
|
|
JEQ emit_literal_done_match_emit_encodeSnappyBetterBlockAsm10B
|
|
MOVL DI, R9
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(SI*1), R10
|
|
SUBL SI, R9
|
|
LEAL -1(R9), SI
|
|
CMPL SI, $0x3c
|
|
JB one_byte_match_emit_encodeSnappyBetterBlockAsm10B
|
|
CMPL SI, $0x00000100
|
|
JB two_bytes_match_emit_encodeSnappyBetterBlockAsm10B
|
|
JB three_bytes_match_emit_encodeSnappyBetterBlockAsm10B
|
|
|
|
three_bytes_match_emit_encodeSnappyBetterBlockAsm10B:
|
|
MOVB $0xf4, (CX)
|
|
MOVW SI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm10B
|
|
|
|
two_bytes_match_emit_encodeSnappyBetterBlockAsm10B:
|
|
MOVB $0xf0, (CX)
|
|
MOVB SI, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL SI, $0x40
|
|
JB memmove_match_emit_encodeSnappyBetterBlockAsm10B
|
|
JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm10B
|
|
|
|
one_byte_match_emit_encodeSnappyBetterBlockAsm10B:
|
|
SHLB $0x02, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_match_emit_encodeSnappyBetterBlockAsm10B:
|
|
LEAQ (CX)(R9*1), SI
|
|
|
|
// genMemMoveShort
|
|
CMPQ R9, $0x08
|
|
JBE emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_8
|
|
CMPQ R9, $0x10
|
|
JBE emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_8through16
|
|
CMPQ R9, $0x20
|
|
JBE emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_8:
|
|
MOVQ (R10), R11
|
|
MOVQ R11, (CX)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_8through16:
|
|
MOVQ (R10), R11
|
|
MOVQ -8(R10)(R9*1), R10
|
|
MOVQ R11, (CX)
|
|
MOVQ R10, -8(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_17through32:
|
|
MOVOU (R10), X0
|
|
MOVOU -16(R10)(R9*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_33through64:
|
|
MOVOU (R10), X0
|
|
MOVOU 16(R10), X1
|
|
MOVOU -32(R10)(R9*1), X2
|
|
MOVOU -16(R10)(R9*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
|
|
memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm10B:
|
|
MOVQ SI, CX
|
|
JMP emit_literal_done_match_emit_encodeSnappyBetterBlockAsm10B
|
|
|
|
memmove_long_match_emit_encodeSnappyBetterBlockAsm10B:
|
|
LEAQ (CX)(R9*1), SI
|
|
|
|
// genMemMoveLong
|
|
MOVOU (R10), X0
|
|
MOVOU 16(R10), X1
|
|
MOVOU -32(R10)(R9*1), X2
|
|
MOVOU -16(R10)(R9*1), X3
|
|
MOVQ R9, R13
|
|
SHRQ $0x05, R13
|
|
MOVQ CX, R11
|
|
ANDL $0x0000001f, R11
|
|
MOVQ $0x00000040, R14
|
|
SUBQ R11, R14
|
|
DECQ R13
|
|
JA emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm10Blarge_forward_sse_loop_32
|
|
LEAQ -32(R10)(R14*1), R11
|
|
LEAQ -32(CX)(R14*1), R15
|
|
|
|
emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm10Blarge_big_loop_back:
|
|
MOVOU (R11), X4
|
|
MOVOU 16(R11), X5
|
|
MOVOA X4, (R15)
|
|
MOVOA X5, 16(R15)
|
|
ADDQ $0x20, R15
|
|
ADDQ $0x20, R11
|
|
ADDQ $0x20, R14
|
|
DECQ R13
|
|
JNA emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm10Blarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm10Blarge_forward_sse_loop_32:
|
|
MOVOU -32(R10)(R14*1), X4
|
|
MOVOU -16(R10)(R14*1), X5
|
|
MOVOA X4, -32(CX)(R14*1)
|
|
MOVOA X5, -16(CX)(R14*1)
|
|
ADDQ $0x20, R14
|
|
CMPQ R9, R14
|
|
JAE emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm10Blarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
MOVQ SI, CX
|
|
|
|
emit_literal_done_match_emit_encodeSnappyBetterBlockAsm10B:
|
|
ADDL R12, DX
|
|
ADDL $0x04, R12
|
|
MOVL DX, 12(SP)
|
|
|
|
// emitCopy
|
|
two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm10B:
|
|
CMPL R12, $0x40
|
|
JBE two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm10B
|
|
MOVB $0xee, (CX)
|
|
MOVW R8, 1(CX)
|
|
LEAL -60(R12), R12
|
|
ADDQ $0x03, CX
|
|
JMP two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm10B
|
|
|
|
two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm10B:
|
|
MOVL R12, SI
|
|
SHLL $0x02, SI
|
|
CMPL R12, $0x0c
|
|
JAE emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm10B
|
|
CMPL R8, $0x00000800
|
|
JAE emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm10B
|
|
LEAL -15(SI), SI
|
|
MOVB R8, 1(CX)
|
|
SHRL $0x08, R8
|
|
SHLL $0x05, R8
|
|
ORL R8, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm10B
|
|
|
|
emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm10B:
|
|
LEAL -2(SI), SI
|
|
MOVB SI, (CX)
|
|
MOVW R8, 1(CX)
|
|
ADDQ $0x03, CX
|
|
|
|
match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm10B:
|
|
CMPL DX, 8(SP)
|
|
JAE emit_remainder_encodeSnappyBetterBlockAsm10B
|
|
CMPQ CX, (SP)
|
|
JB match_nolit_dst_ok_encodeSnappyBetterBlockAsm10B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeSnappyBetterBlockAsm10B:
|
|
MOVQ $0x0000cf1bbcdcbf9b, SI
|
|
MOVQ $0x9e3779b1, R8
|
|
LEAQ 1(DI), DI
|
|
LEAQ -2(DX), R9
|
|
MOVQ (BX)(DI*1), R10
|
|
MOVQ 1(BX)(DI*1), R11
|
|
MOVQ (BX)(R9*1), R12
|
|
MOVQ 1(BX)(R9*1), R13
|
|
SHLQ $0x10, R10
|
|
IMULQ SI, R10
|
|
SHRQ $0x34, R10
|
|
SHLQ $0x20, R11
|
|
IMULQ R8, R11
|
|
SHRQ $0x36, R11
|
|
SHLQ $0x10, R12
|
|
IMULQ SI, R12
|
|
SHRQ $0x34, R12
|
|
SHLQ $0x20, R13
|
|
IMULQ R8, R13
|
|
SHRQ $0x36, R13
|
|
LEAQ 1(DI), R8
|
|
LEAQ 1(R9), R14
|
|
MOVL DI, (AX)(R10*4)
|
|
MOVL R9, (AX)(R12*4)
|
|
MOVL R8, 16384(AX)(R11*4)
|
|
MOVL R14, 16384(AX)(R13*4)
|
|
LEAQ 1(R9)(DI*1), R8
|
|
SHRQ $0x01, R8
|
|
ADDQ $0x01, DI
|
|
SUBQ $0x01, R9
|
|
|
|
index_loop_encodeSnappyBetterBlockAsm10B:
|
|
CMPQ R8, R9
|
|
JAE search_loop_encodeSnappyBetterBlockAsm10B
|
|
MOVQ (BX)(DI*1), R10
|
|
MOVQ (BX)(R8*1), R11
|
|
SHLQ $0x10, R10
|
|
IMULQ SI, R10
|
|
SHRQ $0x34, R10
|
|
SHLQ $0x10, R11
|
|
IMULQ SI, R11
|
|
SHRQ $0x34, R11
|
|
MOVL DI, (AX)(R10*4)
|
|
MOVL R8, (AX)(R11*4)
|
|
ADDQ $0x02, DI
|
|
ADDQ $0x02, R8
|
|
JMP index_loop_encodeSnappyBetterBlockAsm10B
|
|
|
|
emit_remainder_encodeSnappyBetterBlockAsm10B:
|
|
MOVQ src_len+32(FP), AX
|
|
SUBL 12(SP), AX
|
|
LEAQ 3(CX)(AX*1), AX
|
|
CMPQ AX, (SP)
|
|
JB emit_remainder_ok_encodeSnappyBetterBlockAsm10B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeSnappyBetterBlockAsm10B:
|
|
MOVQ src_len+32(FP), AX
|
|
MOVL 12(SP), DX
|
|
CMPL DX, AX
|
|
JEQ emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm10B
|
|
MOVL AX, SI
|
|
MOVL AX, 12(SP)
|
|
LEAQ (BX)(DX*1), AX
|
|
SUBL DX, SI
|
|
LEAL -1(SI), DX
|
|
CMPL DX, $0x3c
|
|
JB one_byte_emit_remainder_encodeSnappyBetterBlockAsm10B
|
|
CMPL DX, $0x00000100
|
|
JB two_bytes_emit_remainder_encodeSnappyBetterBlockAsm10B
|
|
JB three_bytes_emit_remainder_encodeSnappyBetterBlockAsm10B
|
|
|
|
three_bytes_emit_remainder_encodeSnappyBetterBlockAsm10B:
|
|
MOVB $0xf4, (CX)
|
|
MOVW DX, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10B
|
|
|
|
two_bytes_emit_remainder_encodeSnappyBetterBlockAsm10B:
|
|
MOVB $0xf0, (CX)
|
|
MOVB DL, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL DX, $0x40
|
|
JB memmove_emit_remainder_encodeSnappyBetterBlockAsm10B
|
|
JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10B
|
|
|
|
one_byte_emit_remainder_encodeSnappyBetterBlockAsm10B:
|
|
SHLB $0x02, DL
|
|
MOVB DL, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_emit_remainder_encodeSnappyBetterBlockAsm10B:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveShort
|
|
CMPQ BX, $0x03
|
|
JB emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_1or2
|
|
JE emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_3
|
|
CMPQ BX, $0x08
|
|
JB emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_4through7
|
|
CMPQ BX, $0x10
|
|
JBE emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_8through16
|
|
CMPQ BX, $0x20
|
|
JBE emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_1or2:
|
|
MOVB (AX), SI
|
|
MOVB -1(AX)(BX*1), AL
|
|
MOVB SI, (CX)
|
|
MOVB AL, -1(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_3:
|
|
MOVW (AX), SI
|
|
MOVB 2(AX), AL
|
|
MOVW SI, (CX)
|
|
MOVB AL, 2(CX)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_4through7:
|
|
MOVL (AX), SI
|
|
MOVL -4(AX)(BX*1), AX
|
|
MOVL SI, (CX)
|
|
MOVL AX, -4(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_8through16:
|
|
MOVQ (AX), SI
|
|
MOVQ -8(AX)(BX*1), AX
|
|
MOVQ SI, (CX)
|
|
MOVQ AX, -8(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_17through32:
|
|
MOVOU (AX), X0
|
|
MOVOU -16(AX)(BX*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_33through64:
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
|
|
memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm10B:
|
|
MOVQ DX, CX
|
|
JMP emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm10B
|
|
|
|
memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10B:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveLong
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVQ BX, DI
|
|
SHRQ $0x05, DI
|
|
MOVQ CX, SI
|
|
ANDL $0x0000001f, SI
|
|
MOVQ $0x00000040, R8
|
|
SUBQ SI, R8
|
|
DECQ DI
|
|
JA emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10Blarge_forward_sse_loop_32
|
|
LEAQ -32(AX)(R8*1), SI
|
|
LEAQ -32(CX)(R8*1), R9
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10Blarge_big_loop_back:
|
|
MOVOU (SI), X4
|
|
MOVOU 16(SI), X5
|
|
MOVOA X4, (R9)
|
|
MOVOA X5, 16(R9)
|
|
ADDQ $0x20, R9
|
|
ADDQ $0x20, SI
|
|
ADDQ $0x20, R8
|
|
DECQ DI
|
|
JNA emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10Blarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10Blarge_forward_sse_loop_32:
|
|
MOVOU -32(AX)(R8*1), X4
|
|
MOVOU -16(AX)(R8*1), X5
|
|
MOVOA X4, -32(CX)(R8*1)
|
|
MOVOA X5, -16(CX)(R8*1)
|
|
ADDQ $0x20, R8
|
|
CMPQ BX, R8
|
|
JAE emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10Blarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
MOVQ DX, CX
|
|
|
|
emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm10B:
|
|
MOVQ dst_base+0(FP), AX
|
|
SUBQ AX, CX
|
|
MOVQ CX, ret+56(FP)
|
|
RET
|
|
|
|
// func encodeSnappyBetterBlockAsm8B(dst []byte, src []byte, tmp *[5120]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeSnappyBetterBlockAsm8B(SB), $24-64
|
|
MOVQ tmp+48(FP), AX
|
|
MOVQ dst_base+0(FP), CX
|
|
MOVQ $0x00000028, DX
|
|
MOVQ AX, BX
|
|
PXOR X0, X0
|
|
|
|
zero_loop_encodeSnappyBetterBlockAsm8B:
|
|
MOVOU X0, (BX)
|
|
MOVOU X0, 16(BX)
|
|
MOVOU X0, 32(BX)
|
|
MOVOU X0, 48(BX)
|
|
MOVOU X0, 64(BX)
|
|
MOVOU X0, 80(BX)
|
|
MOVOU X0, 96(BX)
|
|
MOVOU X0, 112(BX)
|
|
ADDQ $0x80, BX
|
|
DECQ DX
|
|
JNZ zero_loop_encodeSnappyBetterBlockAsm8B
|
|
MOVL $0x00000000, 12(SP)
|
|
MOVQ src_len+32(FP), DX
|
|
LEAQ -9(DX), BX
|
|
LEAQ -8(DX), SI
|
|
MOVL SI, 8(SP)
|
|
SHRQ $0x05, DX
|
|
SUBL DX, BX
|
|
LEAQ (CX)(BX*1), BX
|
|
MOVQ BX, (SP)
|
|
MOVL $0x00000001, DX
|
|
MOVL $0x00000000, 16(SP)
|
|
MOVQ src_base+24(FP), BX
|
|
|
|
search_loop_encodeSnappyBetterBlockAsm8B:
|
|
MOVL DX, SI
|
|
SUBL 12(SP), SI
|
|
SHRL $0x04, SI
|
|
LEAL 1(DX)(SI*1), SI
|
|
CMPL SI, 8(SP)
|
|
JAE emit_remainder_encodeSnappyBetterBlockAsm8B
|
|
MOVQ (BX)(DX*1), DI
|
|
MOVL SI, 20(SP)
|
|
MOVQ $0x0000cf1bbcdcbf9b, R9
|
|
MOVQ $0x9e3779b1, SI
|
|
MOVQ DI, R10
|
|
MOVQ DI, R11
|
|
SHLQ $0x10, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x36, R10
|
|
SHLQ $0x20, R11
|
|
IMULQ SI, R11
|
|
SHRQ $0x38, R11
|
|
MOVL (AX)(R10*4), SI
|
|
MOVL 4096(AX)(R11*4), R8
|
|
MOVL DX, (AX)(R10*4)
|
|
MOVL DX, 4096(AX)(R11*4)
|
|
MOVQ (BX)(SI*1), R10
|
|
MOVQ (BX)(R8*1), R11
|
|
CMPQ R10, DI
|
|
JEQ candidate_match_encodeSnappyBetterBlockAsm8B
|
|
CMPQ R11, DI
|
|
JNE no_short_found_encodeSnappyBetterBlockAsm8B
|
|
MOVL R8, SI
|
|
JMP candidate_match_encodeSnappyBetterBlockAsm8B
|
|
|
|
no_short_found_encodeSnappyBetterBlockAsm8B:
|
|
CMPL R10, DI
|
|
JEQ candidate_match_encodeSnappyBetterBlockAsm8B
|
|
CMPL R11, DI
|
|
JEQ candidateS_match_encodeSnappyBetterBlockAsm8B
|
|
MOVL 20(SP), DX
|
|
JMP search_loop_encodeSnappyBetterBlockAsm8B
|
|
|
|
candidateS_match_encodeSnappyBetterBlockAsm8B:
|
|
SHRQ $0x08, DI
|
|
MOVQ DI, R10
|
|
SHLQ $0x10, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x36, R10
|
|
MOVL (AX)(R10*4), SI
|
|
INCL DX
|
|
MOVL DX, (AX)(R10*4)
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ candidate_match_encodeSnappyBetterBlockAsm8B
|
|
DECL DX
|
|
MOVL R8, SI
|
|
|
|
candidate_match_encodeSnappyBetterBlockAsm8B:
|
|
MOVL 12(SP), DI
|
|
TESTL SI, SI
|
|
JZ match_extend_back_end_encodeSnappyBetterBlockAsm8B
|
|
|
|
match_extend_back_loop_encodeSnappyBetterBlockAsm8B:
|
|
CMPL DX, DI
|
|
JBE match_extend_back_end_encodeSnappyBetterBlockAsm8B
|
|
MOVB -1(BX)(SI*1), R8
|
|
MOVB -1(BX)(DX*1), R9
|
|
CMPB R8, R9
|
|
JNE match_extend_back_end_encodeSnappyBetterBlockAsm8B
|
|
LEAL -1(DX), DX
|
|
DECL SI
|
|
JZ match_extend_back_end_encodeSnappyBetterBlockAsm8B
|
|
JMP match_extend_back_loop_encodeSnappyBetterBlockAsm8B
|
|
|
|
match_extend_back_end_encodeSnappyBetterBlockAsm8B:
|
|
MOVL DX, DI
|
|
SUBL 12(SP), DI
|
|
LEAQ 3(CX)(DI*1), DI
|
|
CMPQ DI, (SP)
|
|
JB match_dst_size_check_encodeSnappyBetterBlockAsm8B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeSnappyBetterBlockAsm8B:
|
|
MOVL DX, DI
|
|
ADDL $0x04, DX
|
|
ADDL $0x04, SI
|
|
MOVQ src_len+32(FP), R8
|
|
SUBL DX, R8
|
|
LEAQ (BX)(DX*1), R9
|
|
LEAQ (BX)(SI*1), R10
|
|
|
|
// matchLen
|
|
XORL R12, R12
|
|
|
|
matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm8B:
|
|
CMPL R8, $0x10
|
|
JB matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm8B
|
|
MOVQ (R9)(R12*1), R11
|
|
MOVQ 8(R9)(R12*1), R13
|
|
XORQ (R10)(R12*1), R11
|
|
JNZ matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm8B
|
|
XORQ 8(R10)(R12*1), R13
|
|
JNZ matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm8B
|
|
LEAL -16(R8), R8
|
|
LEAL 16(R12), R12
|
|
JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm8B
|
|
|
|
matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm8B:
|
|
TZCNTQ R13, R13
|
|
SARQ $0x03, R13
|
|
LEAL 8(R12)(R13*1), R12
|
|
JMP match_nolit_end_encodeSnappyBetterBlockAsm8B
|
|
|
|
matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm8B:
|
|
CMPL R8, $0x08
|
|
JB matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm8B
|
|
MOVQ (R9)(R12*1), R11
|
|
XORQ (R10)(R12*1), R11
|
|
JNZ matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm8B
|
|
LEAL -8(R8), R8
|
|
LEAL 8(R12), R12
|
|
JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm8B
|
|
|
|
matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm8B:
|
|
TZCNTQ R11, R11
|
|
SARQ $0x03, R11
|
|
LEAL (R12)(R11*1), R12
|
|
JMP match_nolit_end_encodeSnappyBetterBlockAsm8B
|
|
|
|
matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm8B:
|
|
CMPL R8, $0x04
|
|
JB matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm8B
|
|
MOVL (R9)(R12*1), R11
|
|
CMPL (R10)(R12*1), R11
|
|
JNE matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm8B
|
|
LEAL -4(R8), R8
|
|
LEAL 4(R12), R12
|
|
|
|
matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm8B:
|
|
CMPL R8, $0x01
|
|
JE matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm8B
|
|
JB match_nolit_end_encodeSnappyBetterBlockAsm8B
|
|
MOVW (R9)(R12*1), R11
|
|
CMPW (R10)(R12*1), R11
|
|
JNE matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm8B
|
|
LEAL 2(R12), R12
|
|
SUBL $0x02, R8
|
|
JZ match_nolit_end_encodeSnappyBetterBlockAsm8B
|
|
|
|
matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm8B:
|
|
MOVB (R9)(R12*1), R11
|
|
CMPB (R10)(R12*1), R11
|
|
JNE match_nolit_end_encodeSnappyBetterBlockAsm8B
|
|
LEAL 1(R12), R12
|
|
|
|
match_nolit_end_encodeSnappyBetterBlockAsm8B:
|
|
MOVL DX, R8
|
|
SUBL SI, R8
|
|
|
|
// Check if repeat
|
|
MOVL R8, 16(SP)
|
|
MOVL 12(SP), SI
|
|
CMPL SI, DI
|
|
JEQ emit_literal_done_match_emit_encodeSnappyBetterBlockAsm8B
|
|
MOVL DI, R9
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(SI*1), R10
|
|
SUBL SI, R9
|
|
LEAL -1(R9), SI
|
|
CMPL SI, $0x3c
|
|
JB one_byte_match_emit_encodeSnappyBetterBlockAsm8B
|
|
CMPL SI, $0x00000100
|
|
JB two_bytes_match_emit_encodeSnappyBetterBlockAsm8B
|
|
JB three_bytes_match_emit_encodeSnappyBetterBlockAsm8B
|
|
|
|
three_bytes_match_emit_encodeSnappyBetterBlockAsm8B:
|
|
MOVB $0xf4, (CX)
|
|
MOVW SI, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm8B
|
|
|
|
two_bytes_match_emit_encodeSnappyBetterBlockAsm8B:
|
|
MOVB $0xf0, (CX)
|
|
MOVB SI, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL SI, $0x40
|
|
JB memmove_match_emit_encodeSnappyBetterBlockAsm8B
|
|
JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm8B
|
|
|
|
one_byte_match_emit_encodeSnappyBetterBlockAsm8B:
|
|
SHLB $0x02, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_match_emit_encodeSnappyBetterBlockAsm8B:
|
|
LEAQ (CX)(R9*1), SI
|
|
|
|
// genMemMoveShort
|
|
CMPQ R9, $0x08
|
|
JBE emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_8
|
|
CMPQ R9, $0x10
|
|
JBE emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_8through16
|
|
CMPQ R9, $0x20
|
|
JBE emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_8:
|
|
MOVQ (R10), R11
|
|
MOVQ R11, (CX)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_8through16:
|
|
MOVQ (R10), R11
|
|
MOVQ -8(R10)(R9*1), R10
|
|
MOVQ R11, (CX)
|
|
MOVQ R10, -8(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_17through32:
|
|
MOVOU (R10), X0
|
|
MOVOU -16(R10)(R9*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(R9*1)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_33through64:
|
|
MOVOU (R10), X0
|
|
MOVOU 16(R10), X1
|
|
MOVOU -32(R10)(R9*1), X2
|
|
MOVOU -16(R10)(R9*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
|
|
memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm8B:
|
|
MOVQ SI, CX
|
|
JMP emit_literal_done_match_emit_encodeSnappyBetterBlockAsm8B
|
|
|
|
memmove_long_match_emit_encodeSnappyBetterBlockAsm8B:
|
|
LEAQ (CX)(R9*1), SI
|
|
|
|
// genMemMoveLong
|
|
MOVOU (R10), X0
|
|
MOVOU 16(R10), X1
|
|
MOVOU -32(R10)(R9*1), X2
|
|
MOVOU -16(R10)(R9*1), X3
|
|
MOVQ R9, R13
|
|
SHRQ $0x05, R13
|
|
MOVQ CX, R11
|
|
ANDL $0x0000001f, R11
|
|
MOVQ $0x00000040, R14
|
|
SUBQ R11, R14
|
|
DECQ R13
|
|
JA emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm8Blarge_forward_sse_loop_32
|
|
LEAQ -32(R10)(R14*1), R11
|
|
LEAQ -32(CX)(R14*1), R15
|
|
|
|
emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm8Blarge_big_loop_back:
|
|
MOVOU (R11), X4
|
|
MOVOU 16(R11), X5
|
|
MOVOA X4, (R15)
|
|
MOVOA X5, 16(R15)
|
|
ADDQ $0x20, R15
|
|
ADDQ $0x20, R11
|
|
ADDQ $0x20, R14
|
|
DECQ R13
|
|
JNA emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm8Blarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm8Blarge_forward_sse_loop_32:
|
|
MOVOU -32(R10)(R14*1), X4
|
|
MOVOU -16(R10)(R14*1), X5
|
|
MOVOA X4, -32(CX)(R14*1)
|
|
MOVOA X5, -16(CX)(R14*1)
|
|
ADDQ $0x20, R14
|
|
CMPQ R9, R14
|
|
JAE emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm8Blarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(R9*1)
|
|
MOVOU X3, -16(CX)(R9*1)
|
|
MOVQ SI, CX
|
|
|
|
emit_literal_done_match_emit_encodeSnappyBetterBlockAsm8B:
|
|
ADDL R12, DX
|
|
ADDL $0x04, R12
|
|
MOVL DX, 12(SP)
|
|
|
|
// emitCopy
|
|
two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm8B:
|
|
CMPL R12, $0x40
|
|
JBE two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm8B
|
|
MOVB $0xee, (CX)
|
|
MOVW R8, 1(CX)
|
|
LEAL -60(R12), R12
|
|
ADDQ $0x03, CX
|
|
JMP two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm8B
|
|
|
|
two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm8B:
|
|
MOVL R12, SI
|
|
SHLL $0x02, SI
|
|
CMPL R12, $0x0c
|
|
JAE emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm8B
|
|
LEAL -15(SI), SI
|
|
MOVB R8, 1(CX)
|
|
SHRL $0x08, R8
|
|
SHLL $0x05, R8
|
|
ORL R8, SI
|
|
MOVB SI, (CX)
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm8B
|
|
|
|
emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm8B:
|
|
LEAL -2(SI), SI
|
|
MOVB SI, (CX)
|
|
MOVW R8, 1(CX)
|
|
ADDQ $0x03, CX
|
|
|
|
match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm8B:
|
|
CMPL DX, 8(SP)
|
|
JAE emit_remainder_encodeSnappyBetterBlockAsm8B
|
|
CMPQ CX, (SP)
|
|
JB match_nolit_dst_ok_encodeSnappyBetterBlockAsm8B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeSnappyBetterBlockAsm8B:
|
|
MOVQ $0x0000cf1bbcdcbf9b, SI
|
|
MOVQ $0x9e3779b1, R8
|
|
LEAQ 1(DI), DI
|
|
LEAQ -2(DX), R9
|
|
MOVQ (BX)(DI*1), R10
|
|
MOVQ 1(BX)(DI*1), R11
|
|
MOVQ (BX)(R9*1), R12
|
|
MOVQ 1(BX)(R9*1), R13
|
|
SHLQ $0x10, R10
|
|
IMULQ SI, R10
|
|
SHRQ $0x36, R10
|
|
SHLQ $0x20, R11
|
|
IMULQ R8, R11
|
|
SHRQ $0x38, R11
|
|
SHLQ $0x10, R12
|
|
IMULQ SI, R12
|
|
SHRQ $0x36, R12
|
|
SHLQ $0x20, R13
|
|
IMULQ R8, R13
|
|
SHRQ $0x38, R13
|
|
LEAQ 1(DI), R8
|
|
LEAQ 1(R9), R14
|
|
MOVL DI, (AX)(R10*4)
|
|
MOVL R9, (AX)(R12*4)
|
|
MOVL R8, 4096(AX)(R11*4)
|
|
MOVL R14, 4096(AX)(R13*4)
|
|
LEAQ 1(R9)(DI*1), R8
|
|
SHRQ $0x01, R8
|
|
ADDQ $0x01, DI
|
|
SUBQ $0x01, R9
|
|
|
|
index_loop_encodeSnappyBetterBlockAsm8B:
|
|
CMPQ R8, R9
|
|
JAE search_loop_encodeSnappyBetterBlockAsm8B
|
|
MOVQ (BX)(DI*1), R10
|
|
MOVQ (BX)(R8*1), R11
|
|
SHLQ $0x10, R10
|
|
IMULQ SI, R10
|
|
SHRQ $0x36, R10
|
|
SHLQ $0x10, R11
|
|
IMULQ SI, R11
|
|
SHRQ $0x36, R11
|
|
MOVL DI, (AX)(R10*4)
|
|
MOVL R8, (AX)(R11*4)
|
|
ADDQ $0x02, DI
|
|
ADDQ $0x02, R8
|
|
JMP index_loop_encodeSnappyBetterBlockAsm8B
|
|
|
|
emit_remainder_encodeSnappyBetterBlockAsm8B:
|
|
MOVQ src_len+32(FP), AX
|
|
SUBL 12(SP), AX
|
|
LEAQ 3(CX)(AX*1), AX
|
|
CMPQ AX, (SP)
|
|
JB emit_remainder_ok_encodeSnappyBetterBlockAsm8B
|
|
MOVQ $0x00000000, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeSnappyBetterBlockAsm8B:
|
|
MOVQ src_len+32(FP), AX
|
|
MOVL 12(SP), DX
|
|
CMPL DX, AX
|
|
JEQ emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm8B
|
|
MOVL AX, SI
|
|
MOVL AX, 12(SP)
|
|
LEAQ (BX)(DX*1), AX
|
|
SUBL DX, SI
|
|
LEAL -1(SI), DX
|
|
CMPL DX, $0x3c
|
|
JB one_byte_emit_remainder_encodeSnappyBetterBlockAsm8B
|
|
CMPL DX, $0x00000100
|
|
JB two_bytes_emit_remainder_encodeSnappyBetterBlockAsm8B
|
|
JB three_bytes_emit_remainder_encodeSnappyBetterBlockAsm8B
|
|
|
|
three_bytes_emit_remainder_encodeSnappyBetterBlockAsm8B:
|
|
MOVB $0xf4, (CX)
|
|
MOVW DX, 1(CX)
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8B
|
|
|
|
two_bytes_emit_remainder_encodeSnappyBetterBlockAsm8B:
|
|
MOVB $0xf0, (CX)
|
|
MOVB DL, 1(CX)
|
|
ADDQ $0x02, CX
|
|
CMPL DX, $0x40
|
|
JB memmove_emit_remainder_encodeSnappyBetterBlockAsm8B
|
|
JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8B
|
|
|
|
one_byte_emit_remainder_encodeSnappyBetterBlockAsm8B:
|
|
SHLB $0x02, DL
|
|
MOVB DL, (CX)
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_emit_remainder_encodeSnappyBetterBlockAsm8B:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveShort
|
|
CMPQ BX, $0x03
|
|
JB emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_1or2
|
|
JE emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_3
|
|
CMPQ BX, $0x08
|
|
JB emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_4through7
|
|
CMPQ BX, $0x10
|
|
JBE emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_8through16
|
|
CMPQ BX, $0x20
|
|
JBE emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_1or2:
|
|
MOVB (AX), SI
|
|
MOVB -1(AX)(BX*1), AL
|
|
MOVB SI, (CX)
|
|
MOVB AL, -1(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_3:
|
|
MOVW (AX), SI
|
|
MOVB 2(AX), AL
|
|
MOVW SI, (CX)
|
|
MOVB AL, 2(CX)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_4through7:
|
|
MOVL (AX), SI
|
|
MOVL -4(AX)(BX*1), AX
|
|
MOVL SI, (CX)
|
|
MOVL AX, -4(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_8through16:
|
|
MOVQ (AX), SI
|
|
MOVQ -8(AX)(BX*1), AX
|
|
MOVQ SI, (CX)
|
|
MOVQ AX, -8(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_17through32:
|
|
MOVOU (AX), X0
|
|
MOVOU -16(AX)(BX*1), X1
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, -16(CX)(BX*1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_33through64:
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
|
|
memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm8B:
|
|
MOVQ DX, CX
|
|
JMP emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm8B
|
|
|
|
memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8B:
|
|
LEAQ (CX)(SI*1), DX
|
|
MOVL SI, BX
|
|
|
|
// genMemMoveLong
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X1
|
|
MOVOU -32(AX)(BX*1), X2
|
|
MOVOU -16(AX)(BX*1), X3
|
|
MOVQ BX, DI
|
|
SHRQ $0x05, DI
|
|
MOVQ CX, SI
|
|
ANDL $0x0000001f, SI
|
|
MOVQ $0x00000040, R8
|
|
SUBQ SI, R8
|
|
DECQ DI
|
|
JA emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8Blarge_forward_sse_loop_32
|
|
LEAQ -32(AX)(R8*1), SI
|
|
LEAQ -32(CX)(R8*1), R9
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8Blarge_big_loop_back:
|
|
MOVOU (SI), X4
|
|
MOVOU 16(SI), X5
|
|
MOVOA X4, (R9)
|
|
MOVOA X5, 16(R9)
|
|
ADDQ $0x20, R9
|
|
ADDQ $0x20, SI
|
|
ADDQ $0x20, R8
|
|
DECQ DI
|
|
JNA emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8Blarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8Blarge_forward_sse_loop_32:
|
|
MOVOU -32(AX)(R8*1), X4
|
|
MOVOU -16(AX)(R8*1), X5
|
|
MOVOA X4, -32(CX)(R8*1)
|
|
MOVOA X5, -16(CX)(R8*1)
|
|
ADDQ $0x20, R8
|
|
CMPQ BX, R8
|
|
JAE emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8Blarge_forward_sse_loop_32
|
|
MOVOU X0, (CX)
|
|
MOVOU X1, 16(CX)
|
|
MOVOU X2, -32(CX)(BX*1)
|
|
MOVOU X3, -16(CX)(BX*1)
|
|
MOVQ DX, CX
|
|
|
|
emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm8B:
|
|
MOVQ dst_base+0(FP), AX
|
|
SUBQ AX, CX
|
|
MOVQ CX, ret+56(FP)
|
|
RET
|
|
|
|
// func calcBlockSize(src []byte, tmp *[32768]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·calcBlockSize(SB), $24-40
|
|
MOVQ tmp+24(FP), AX
|
|
XORQ CX, CX
|
|
MOVQ $0x00000100, DX
|
|
MOVQ AX, BX
|
|
PXOR X0, X0
|
|
|
|
zero_loop_calcBlockSize:
|
|
MOVOU X0, (BX)
|
|
MOVOU X0, 16(BX)
|
|
MOVOU X0, 32(BX)
|
|
MOVOU X0, 48(BX)
|
|
MOVOU X0, 64(BX)
|
|
MOVOU X0, 80(BX)
|
|
MOVOU X0, 96(BX)
|
|
MOVOU X0, 112(BX)
|
|
ADDQ $0x80, BX
|
|
DECQ DX
|
|
JNZ zero_loop_calcBlockSize
|
|
MOVL $0x00000000, 12(SP)
|
|
MOVQ src_len+8(FP), DX
|
|
LEAQ -9(DX), BX
|
|
LEAQ -8(DX), SI
|
|
MOVL SI, 8(SP)
|
|
SHRQ $0x05, DX
|
|
SUBL DX, BX
|
|
LEAQ (CX)(BX*1), BX
|
|
MOVQ BX, (SP)
|
|
MOVL $0x00000001, DX
|
|
MOVL DX, 16(SP)
|
|
MOVQ src_base+0(FP), BX
|
|
|
|
search_loop_calcBlockSize:
|
|
MOVL DX, SI
|
|
SUBL 12(SP), SI
|
|
SHRL $0x05, SI
|
|
LEAL 4(DX)(SI*1), SI
|
|
CMPL SI, 8(SP)
|
|
JAE emit_remainder_calcBlockSize
|
|
MOVQ (BX)(DX*1), DI
|
|
MOVL SI, 20(SP)
|
|
MOVQ $0x0000cf1bbcdcbf9b, R9
|
|
MOVQ DI, R10
|
|
MOVQ DI, R11
|
|
SHRQ $0x08, R11
|
|
SHLQ $0x10, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x33, R10
|
|
SHLQ $0x10, R11
|
|
IMULQ R9, R11
|
|
SHRQ $0x33, R11
|
|
MOVL (AX)(R10*4), SI
|
|
MOVL (AX)(R11*4), R8
|
|
MOVL DX, (AX)(R10*4)
|
|
LEAL 1(DX), R10
|
|
MOVL R10, (AX)(R11*4)
|
|
MOVQ DI, R10
|
|
SHRQ $0x10, R10
|
|
SHLQ $0x10, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x33, R10
|
|
MOVL DX, R9
|
|
SUBL 16(SP), R9
|
|
MOVL 1(BX)(R9*1), R11
|
|
MOVQ DI, R9
|
|
SHRQ $0x08, R9
|
|
CMPL R9, R11
|
|
JNE no_repeat_found_calcBlockSize
|
|
LEAL 1(DX), DI
|
|
MOVL 12(SP), SI
|
|
MOVL DI, R8
|
|
SUBL 16(SP), R8
|
|
JZ repeat_extend_back_end_calcBlockSize
|
|
|
|
repeat_extend_back_loop_calcBlockSize:
|
|
CMPL DI, SI
|
|
JBE repeat_extend_back_end_calcBlockSize
|
|
MOVB -1(BX)(R8*1), R9
|
|
MOVB -1(BX)(DI*1), R10
|
|
CMPB R9, R10
|
|
JNE repeat_extend_back_end_calcBlockSize
|
|
LEAL -1(DI), DI
|
|
DECL R8
|
|
JNZ repeat_extend_back_loop_calcBlockSize
|
|
|
|
repeat_extend_back_end_calcBlockSize:
|
|
MOVL DI, SI
|
|
SUBL 12(SP), SI
|
|
LEAQ 5(CX)(SI*1), SI
|
|
CMPQ SI, (SP)
|
|
JB repeat_dst_size_check_calcBlockSize
|
|
MOVQ $0x00000000, ret+32(FP)
|
|
RET
|
|
|
|
repeat_dst_size_check_calcBlockSize:
|
|
MOVL 12(SP), SI
|
|
CMPL SI, DI
|
|
JEQ emit_literal_done_repeat_emit_calcBlockSize
|
|
MOVL DI, R8
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(SI*1), R9
|
|
SUBL SI, R8
|
|
LEAL -1(R8), SI
|
|
CMPL SI, $0x3c
|
|
JB one_byte_repeat_emit_calcBlockSize
|
|
CMPL SI, $0x00000100
|
|
JB two_bytes_repeat_emit_calcBlockSize
|
|
CMPL SI, $0x00010000
|
|
JB three_bytes_repeat_emit_calcBlockSize
|
|
CMPL SI, $0x01000000
|
|
JB four_bytes_repeat_emit_calcBlockSize
|
|
ADDQ $0x05, CX
|
|
JMP memmove_long_repeat_emit_calcBlockSize
|
|
|
|
four_bytes_repeat_emit_calcBlockSize:
|
|
ADDQ $0x04, CX
|
|
JMP memmove_long_repeat_emit_calcBlockSize
|
|
|
|
three_bytes_repeat_emit_calcBlockSize:
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_repeat_emit_calcBlockSize
|
|
|
|
two_bytes_repeat_emit_calcBlockSize:
|
|
ADDQ $0x02, CX
|
|
CMPL SI, $0x40
|
|
JB memmove_repeat_emit_calcBlockSize
|
|
JMP memmove_long_repeat_emit_calcBlockSize
|
|
|
|
one_byte_repeat_emit_calcBlockSize:
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_repeat_emit_calcBlockSize:
|
|
LEAQ (CX)(R8*1), CX
|
|
JMP emit_literal_done_repeat_emit_calcBlockSize
|
|
|
|
memmove_long_repeat_emit_calcBlockSize:
|
|
LEAQ (CX)(R8*1), CX
|
|
|
|
emit_literal_done_repeat_emit_calcBlockSize:
|
|
ADDL $0x05, DX
|
|
MOVL DX, SI
|
|
SUBL 16(SP), SI
|
|
MOVQ src_len+8(FP), R8
|
|
SUBL DX, R8
|
|
LEAQ (BX)(DX*1), R9
|
|
LEAQ (BX)(SI*1), SI
|
|
|
|
// matchLen
|
|
XORL R11, R11
|
|
|
|
matchlen_loopback_16_repeat_extend_calcBlockSize:
|
|
CMPL R8, $0x10
|
|
JB matchlen_match8_repeat_extend_calcBlockSize
|
|
MOVQ (R9)(R11*1), R10
|
|
MOVQ 8(R9)(R11*1), R12
|
|
XORQ (SI)(R11*1), R10
|
|
JNZ matchlen_bsf_8_repeat_extend_calcBlockSize
|
|
XORQ 8(SI)(R11*1), R12
|
|
JNZ matchlen_bsf_16repeat_extend_calcBlockSize
|
|
LEAL -16(R8), R8
|
|
LEAL 16(R11), R11
|
|
JMP matchlen_loopback_16_repeat_extend_calcBlockSize
|
|
|
|
matchlen_bsf_16repeat_extend_calcBlockSize:
|
|
TZCNTQ R12, R12
|
|
SARQ $0x03, R12
|
|
LEAL 8(R11)(R12*1), R11
|
|
JMP repeat_extend_forward_end_calcBlockSize
|
|
|
|
matchlen_match8_repeat_extend_calcBlockSize:
|
|
CMPL R8, $0x08
|
|
JB matchlen_match4_repeat_extend_calcBlockSize
|
|
MOVQ (R9)(R11*1), R10
|
|
XORQ (SI)(R11*1), R10
|
|
JNZ matchlen_bsf_8_repeat_extend_calcBlockSize
|
|
LEAL -8(R8), R8
|
|
LEAL 8(R11), R11
|
|
JMP matchlen_match4_repeat_extend_calcBlockSize
|
|
|
|
matchlen_bsf_8_repeat_extend_calcBlockSize:
|
|
TZCNTQ R10, R10
|
|
SARQ $0x03, R10
|
|
LEAL (R11)(R10*1), R11
|
|
JMP repeat_extend_forward_end_calcBlockSize
|
|
|
|
matchlen_match4_repeat_extend_calcBlockSize:
|
|
CMPL R8, $0x04
|
|
JB matchlen_match2_repeat_extend_calcBlockSize
|
|
MOVL (R9)(R11*1), R10
|
|
CMPL (SI)(R11*1), R10
|
|
JNE matchlen_match2_repeat_extend_calcBlockSize
|
|
LEAL -4(R8), R8
|
|
LEAL 4(R11), R11
|
|
|
|
matchlen_match2_repeat_extend_calcBlockSize:
|
|
CMPL R8, $0x01
|
|
JE matchlen_match1_repeat_extend_calcBlockSize
|
|
JB repeat_extend_forward_end_calcBlockSize
|
|
MOVW (R9)(R11*1), R10
|
|
CMPW (SI)(R11*1), R10
|
|
JNE matchlen_match1_repeat_extend_calcBlockSize
|
|
LEAL 2(R11), R11
|
|
SUBL $0x02, R8
|
|
JZ repeat_extend_forward_end_calcBlockSize
|
|
|
|
matchlen_match1_repeat_extend_calcBlockSize:
|
|
MOVB (R9)(R11*1), R10
|
|
CMPB (SI)(R11*1), R10
|
|
JNE repeat_extend_forward_end_calcBlockSize
|
|
LEAL 1(R11), R11
|
|
|
|
repeat_extend_forward_end_calcBlockSize:
|
|
ADDL R11, DX
|
|
MOVL DX, SI
|
|
SUBL DI, SI
|
|
MOVL 16(SP), DI
|
|
|
|
// emitCopy
|
|
CMPL DI, $0x00010000
|
|
JB two_byte_offset_repeat_as_copy_calcBlockSize
|
|
|
|
four_bytes_loop_back_repeat_as_copy_calcBlockSize:
|
|
CMPL SI, $0x40
|
|
JBE four_bytes_remain_repeat_as_copy_calcBlockSize
|
|
LEAL -64(SI), SI
|
|
ADDQ $0x05, CX
|
|
CMPL SI, $0x04
|
|
JB four_bytes_remain_repeat_as_copy_calcBlockSize
|
|
JMP four_bytes_loop_back_repeat_as_copy_calcBlockSize
|
|
|
|
four_bytes_remain_repeat_as_copy_calcBlockSize:
|
|
TESTL SI, SI
|
|
JZ repeat_end_emit_calcBlockSize
|
|
XORL SI, SI
|
|
ADDQ $0x05, CX
|
|
JMP repeat_end_emit_calcBlockSize
|
|
|
|
two_byte_offset_repeat_as_copy_calcBlockSize:
|
|
CMPL SI, $0x40
|
|
JBE two_byte_offset_short_repeat_as_copy_calcBlockSize
|
|
LEAL -60(SI), SI
|
|
ADDQ $0x03, CX
|
|
JMP two_byte_offset_repeat_as_copy_calcBlockSize
|
|
|
|
two_byte_offset_short_repeat_as_copy_calcBlockSize:
|
|
MOVL SI, R8
|
|
SHLL $0x02, R8
|
|
CMPL SI, $0x0c
|
|
JAE emit_copy_three_repeat_as_copy_calcBlockSize
|
|
CMPL DI, $0x00000800
|
|
JAE emit_copy_three_repeat_as_copy_calcBlockSize
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_calcBlockSize
|
|
|
|
emit_copy_three_repeat_as_copy_calcBlockSize:
|
|
ADDQ $0x03, CX
|
|
|
|
repeat_end_emit_calcBlockSize:
|
|
MOVL DX, 12(SP)
|
|
JMP search_loop_calcBlockSize
|
|
|
|
no_repeat_found_calcBlockSize:
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ candidate_match_calcBlockSize
|
|
SHRQ $0x08, DI
|
|
MOVL (AX)(R10*4), SI
|
|
LEAL 2(DX), R9
|
|
CMPL (BX)(R8*1), DI
|
|
JEQ candidate2_match_calcBlockSize
|
|
MOVL R9, (AX)(R10*4)
|
|
SHRQ $0x08, DI
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ candidate3_match_calcBlockSize
|
|
MOVL 20(SP), DX
|
|
JMP search_loop_calcBlockSize
|
|
|
|
candidate3_match_calcBlockSize:
|
|
ADDL $0x02, DX
|
|
JMP candidate_match_calcBlockSize
|
|
|
|
candidate2_match_calcBlockSize:
|
|
MOVL R9, (AX)(R10*4)
|
|
INCL DX
|
|
MOVL R8, SI
|
|
|
|
candidate_match_calcBlockSize:
|
|
MOVL 12(SP), DI
|
|
TESTL SI, SI
|
|
JZ match_extend_back_end_calcBlockSize
|
|
|
|
match_extend_back_loop_calcBlockSize:
|
|
CMPL DX, DI
|
|
JBE match_extend_back_end_calcBlockSize
|
|
MOVB -1(BX)(SI*1), R8
|
|
MOVB -1(BX)(DX*1), R9
|
|
CMPB R8, R9
|
|
JNE match_extend_back_end_calcBlockSize
|
|
LEAL -1(DX), DX
|
|
DECL SI
|
|
JZ match_extend_back_end_calcBlockSize
|
|
JMP match_extend_back_loop_calcBlockSize
|
|
|
|
match_extend_back_end_calcBlockSize:
|
|
MOVL DX, DI
|
|
SUBL 12(SP), DI
|
|
LEAQ 5(CX)(DI*1), DI
|
|
CMPQ DI, (SP)
|
|
JB match_dst_size_check_calcBlockSize
|
|
MOVQ $0x00000000, ret+32(FP)
|
|
RET
|
|
|
|
match_dst_size_check_calcBlockSize:
|
|
MOVL DX, DI
|
|
MOVL 12(SP), R8
|
|
CMPL R8, DI
|
|
JEQ emit_literal_done_match_emit_calcBlockSize
|
|
MOVL DI, R9
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(R8*1), DI
|
|
SUBL R8, R9
|
|
LEAL -1(R9), DI
|
|
CMPL DI, $0x3c
|
|
JB one_byte_match_emit_calcBlockSize
|
|
CMPL DI, $0x00000100
|
|
JB two_bytes_match_emit_calcBlockSize
|
|
CMPL DI, $0x00010000
|
|
JB three_bytes_match_emit_calcBlockSize
|
|
CMPL DI, $0x01000000
|
|
JB four_bytes_match_emit_calcBlockSize
|
|
ADDQ $0x05, CX
|
|
JMP memmove_long_match_emit_calcBlockSize
|
|
|
|
four_bytes_match_emit_calcBlockSize:
|
|
ADDQ $0x04, CX
|
|
JMP memmove_long_match_emit_calcBlockSize
|
|
|
|
three_bytes_match_emit_calcBlockSize:
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_match_emit_calcBlockSize
|
|
|
|
two_bytes_match_emit_calcBlockSize:
|
|
ADDQ $0x02, CX
|
|
CMPL DI, $0x40
|
|
JB memmove_match_emit_calcBlockSize
|
|
JMP memmove_long_match_emit_calcBlockSize
|
|
|
|
one_byte_match_emit_calcBlockSize:
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_match_emit_calcBlockSize:
|
|
LEAQ (CX)(R9*1), CX
|
|
JMP emit_literal_done_match_emit_calcBlockSize
|
|
|
|
memmove_long_match_emit_calcBlockSize:
|
|
LEAQ (CX)(R9*1), CX
|
|
|
|
emit_literal_done_match_emit_calcBlockSize:
|
|
match_nolit_loop_calcBlockSize:
|
|
MOVL DX, DI
|
|
SUBL SI, DI
|
|
MOVL DI, 16(SP)
|
|
ADDL $0x04, DX
|
|
ADDL $0x04, SI
|
|
MOVQ src_len+8(FP), DI
|
|
SUBL DX, DI
|
|
LEAQ (BX)(DX*1), R8
|
|
LEAQ (BX)(SI*1), SI
|
|
|
|
// matchLen
|
|
XORL R10, R10
|
|
|
|
matchlen_loopback_16_match_nolit_calcBlockSize:
|
|
CMPL DI, $0x10
|
|
JB matchlen_match8_match_nolit_calcBlockSize
|
|
MOVQ (R8)(R10*1), R9
|
|
MOVQ 8(R8)(R10*1), R11
|
|
XORQ (SI)(R10*1), R9
|
|
JNZ matchlen_bsf_8_match_nolit_calcBlockSize
|
|
XORQ 8(SI)(R10*1), R11
|
|
JNZ matchlen_bsf_16match_nolit_calcBlockSize
|
|
LEAL -16(DI), DI
|
|
LEAL 16(R10), R10
|
|
JMP matchlen_loopback_16_match_nolit_calcBlockSize
|
|
|
|
matchlen_bsf_16match_nolit_calcBlockSize:
|
|
TZCNTQ R11, R11
|
|
SARQ $0x03, R11
|
|
LEAL 8(R10)(R11*1), R10
|
|
JMP match_nolit_end_calcBlockSize
|
|
|
|
matchlen_match8_match_nolit_calcBlockSize:
|
|
CMPL DI, $0x08
|
|
JB matchlen_match4_match_nolit_calcBlockSize
|
|
MOVQ (R8)(R10*1), R9
|
|
XORQ (SI)(R10*1), R9
|
|
JNZ matchlen_bsf_8_match_nolit_calcBlockSize
|
|
LEAL -8(DI), DI
|
|
LEAL 8(R10), R10
|
|
JMP matchlen_match4_match_nolit_calcBlockSize
|
|
|
|
matchlen_bsf_8_match_nolit_calcBlockSize:
|
|
TZCNTQ R9, R9
|
|
SARQ $0x03, R9
|
|
LEAL (R10)(R9*1), R10
|
|
JMP match_nolit_end_calcBlockSize
|
|
|
|
matchlen_match4_match_nolit_calcBlockSize:
|
|
CMPL DI, $0x04
|
|
JB matchlen_match2_match_nolit_calcBlockSize
|
|
MOVL (R8)(R10*1), R9
|
|
CMPL (SI)(R10*1), R9
|
|
JNE matchlen_match2_match_nolit_calcBlockSize
|
|
LEAL -4(DI), DI
|
|
LEAL 4(R10), R10
|
|
|
|
matchlen_match2_match_nolit_calcBlockSize:
|
|
CMPL DI, $0x01
|
|
JE matchlen_match1_match_nolit_calcBlockSize
|
|
JB match_nolit_end_calcBlockSize
|
|
MOVW (R8)(R10*1), R9
|
|
CMPW (SI)(R10*1), R9
|
|
JNE matchlen_match1_match_nolit_calcBlockSize
|
|
LEAL 2(R10), R10
|
|
SUBL $0x02, DI
|
|
JZ match_nolit_end_calcBlockSize
|
|
|
|
matchlen_match1_match_nolit_calcBlockSize:
|
|
MOVB (R8)(R10*1), R9
|
|
CMPB (SI)(R10*1), R9
|
|
JNE match_nolit_end_calcBlockSize
|
|
LEAL 1(R10), R10
|
|
|
|
match_nolit_end_calcBlockSize:
|
|
ADDL R10, DX
|
|
MOVL 16(SP), SI
|
|
ADDL $0x04, R10
|
|
MOVL DX, 12(SP)
|
|
|
|
// emitCopy
|
|
CMPL SI, $0x00010000
|
|
JB two_byte_offset_match_nolit_calcBlockSize
|
|
|
|
four_bytes_loop_back_match_nolit_calcBlockSize:
|
|
CMPL R10, $0x40
|
|
JBE four_bytes_remain_match_nolit_calcBlockSize
|
|
LEAL -64(R10), R10
|
|
ADDQ $0x05, CX
|
|
CMPL R10, $0x04
|
|
JB four_bytes_remain_match_nolit_calcBlockSize
|
|
JMP four_bytes_loop_back_match_nolit_calcBlockSize
|
|
|
|
four_bytes_remain_match_nolit_calcBlockSize:
|
|
TESTL R10, R10
|
|
JZ match_nolit_emitcopy_end_calcBlockSize
|
|
XORL SI, SI
|
|
ADDQ $0x05, CX
|
|
JMP match_nolit_emitcopy_end_calcBlockSize
|
|
|
|
two_byte_offset_match_nolit_calcBlockSize:
|
|
CMPL R10, $0x40
|
|
JBE two_byte_offset_short_match_nolit_calcBlockSize
|
|
LEAL -60(R10), R10
|
|
ADDQ $0x03, CX
|
|
JMP two_byte_offset_match_nolit_calcBlockSize
|
|
|
|
two_byte_offset_short_match_nolit_calcBlockSize:
|
|
MOVL R10, DI
|
|
SHLL $0x02, DI
|
|
CMPL R10, $0x0c
|
|
JAE emit_copy_three_match_nolit_calcBlockSize
|
|
CMPL SI, $0x00000800
|
|
JAE emit_copy_three_match_nolit_calcBlockSize
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_calcBlockSize
|
|
|
|
emit_copy_three_match_nolit_calcBlockSize:
|
|
ADDQ $0x03, CX
|
|
|
|
match_nolit_emitcopy_end_calcBlockSize:
|
|
CMPL DX, 8(SP)
|
|
JAE emit_remainder_calcBlockSize
|
|
MOVQ -2(BX)(DX*1), DI
|
|
CMPQ CX, (SP)
|
|
JB match_nolit_dst_ok_calcBlockSize
|
|
MOVQ $0x00000000, ret+32(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_calcBlockSize:
|
|
MOVQ $0x0000cf1bbcdcbf9b, R9
|
|
MOVQ DI, R8
|
|
SHRQ $0x10, DI
|
|
MOVQ DI, SI
|
|
SHLQ $0x10, R8
|
|
IMULQ R9, R8
|
|
SHRQ $0x33, R8
|
|
SHLQ $0x10, SI
|
|
IMULQ R9, SI
|
|
SHRQ $0x33, SI
|
|
LEAL -2(DX), R9
|
|
LEAQ (AX)(SI*4), R10
|
|
MOVL (R10), SI
|
|
MOVL R9, (AX)(R8*4)
|
|
MOVL DX, (R10)
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ match_nolit_loop_calcBlockSize
|
|
INCL DX
|
|
JMP search_loop_calcBlockSize
|
|
|
|
emit_remainder_calcBlockSize:
|
|
MOVQ src_len+8(FP), AX
|
|
SUBL 12(SP), AX
|
|
LEAQ 5(CX)(AX*1), AX
|
|
CMPQ AX, (SP)
|
|
JB emit_remainder_ok_calcBlockSize
|
|
MOVQ $0x00000000, ret+32(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_calcBlockSize:
|
|
MOVQ src_len+8(FP), AX
|
|
MOVL 12(SP), DX
|
|
CMPL DX, AX
|
|
JEQ emit_literal_done_emit_remainder_calcBlockSize
|
|
MOVL AX, SI
|
|
MOVL AX, 12(SP)
|
|
LEAQ (BX)(DX*1), AX
|
|
SUBL DX, SI
|
|
LEAL -1(SI), AX
|
|
CMPL AX, $0x3c
|
|
JB one_byte_emit_remainder_calcBlockSize
|
|
CMPL AX, $0x00000100
|
|
JB two_bytes_emit_remainder_calcBlockSize
|
|
CMPL AX, $0x00010000
|
|
JB three_bytes_emit_remainder_calcBlockSize
|
|
CMPL AX, $0x01000000
|
|
JB four_bytes_emit_remainder_calcBlockSize
|
|
ADDQ $0x05, CX
|
|
JMP memmove_long_emit_remainder_calcBlockSize
|
|
|
|
four_bytes_emit_remainder_calcBlockSize:
|
|
ADDQ $0x04, CX
|
|
JMP memmove_long_emit_remainder_calcBlockSize
|
|
|
|
three_bytes_emit_remainder_calcBlockSize:
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_emit_remainder_calcBlockSize
|
|
|
|
two_bytes_emit_remainder_calcBlockSize:
|
|
ADDQ $0x02, CX
|
|
CMPL AX, $0x40
|
|
JB memmove_emit_remainder_calcBlockSize
|
|
JMP memmove_long_emit_remainder_calcBlockSize
|
|
|
|
one_byte_emit_remainder_calcBlockSize:
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_emit_remainder_calcBlockSize:
|
|
LEAQ (CX)(SI*1), AX
|
|
MOVQ AX, CX
|
|
JMP emit_literal_done_emit_remainder_calcBlockSize
|
|
|
|
memmove_long_emit_remainder_calcBlockSize:
|
|
LEAQ (CX)(SI*1), AX
|
|
MOVQ AX, CX
|
|
|
|
emit_literal_done_emit_remainder_calcBlockSize:
|
|
MOVQ CX, ret+32(FP)
|
|
RET
|
|
|
|
// func calcBlockSizeSmall(src []byte, tmp *[2048]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·calcBlockSizeSmall(SB), $24-40
|
|
MOVQ tmp+24(FP), AX
|
|
XORQ CX, CX
|
|
MOVQ $0x00000010, DX
|
|
MOVQ AX, BX
|
|
PXOR X0, X0
|
|
|
|
zero_loop_calcBlockSizeSmall:
|
|
MOVOU X0, (BX)
|
|
MOVOU X0, 16(BX)
|
|
MOVOU X0, 32(BX)
|
|
MOVOU X0, 48(BX)
|
|
MOVOU X0, 64(BX)
|
|
MOVOU X0, 80(BX)
|
|
MOVOU X0, 96(BX)
|
|
MOVOU X0, 112(BX)
|
|
ADDQ $0x80, BX
|
|
DECQ DX
|
|
JNZ zero_loop_calcBlockSizeSmall
|
|
MOVL $0x00000000, 12(SP)
|
|
MOVQ src_len+8(FP), DX
|
|
LEAQ -9(DX), BX
|
|
LEAQ -8(DX), SI
|
|
MOVL SI, 8(SP)
|
|
SHRQ $0x05, DX
|
|
SUBL DX, BX
|
|
LEAQ (CX)(BX*1), BX
|
|
MOVQ BX, (SP)
|
|
MOVL $0x00000001, DX
|
|
MOVL DX, 16(SP)
|
|
MOVQ src_base+0(FP), BX
|
|
|
|
search_loop_calcBlockSizeSmall:
|
|
MOVL DX, SI
|
|
SUBL 12(SP), SI
|
|
SHRL $0x04, SI
|
|
LEAL 4(DX)(SI*1), SI
|
|
CMPL SI, 8(SP)
|
|
JAE emit_remainder_calcBlockSizeSmall
|
|
MOVQ (BX)(DX*1), DI
|
|
MOVL SI, 20(SP)
|
|
MOVQ $0x9e3779b1, R9
|
|
MOVQ DI, R10
|
|
MOVQ DI, R11
|
|
SHRQ $0x08, R11
|
|
SHLQ $0x20, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x37, R10
|
|
SHLQ $0x20, R11
|
|
IMULQ R9, R11
|
|
SHRQ $0x37, R11
|
|
MOVL (AX)(R10*4), SI
|
|
MOVL (AX)(R11*4), R8
|
|
MOVL DX, (AX)(R10*4)
|
|
LEAL 1(DX), R10
|
|
MOVL R10, (AX)(R11*4)
|
|
MOVQ DI, R10
|
|
SHRQ $0x10, R10
|
|
SHLQ $0x20, R10
|
|
IMULQ R9, R10
|
|
SHRQ $0x37, R10
|
|
MOVL DX, R9
|
|
SUBL 16(SP), R9
|
|
MOVL 1(BX)(R9*1), R11
|
|
MOVQ DI, R9
|
|
SHRQ $0x08, R9
|
|
CMPL R9, R11
|
|
JNE no_repeat_found_calcBlockSizeSmall
|
|
LEAL 1(DX), DI
|
|
MOVL 12(SP), SI
|
|
MOVL DI, R8
|
|
SUBL 16(SP), R8
|
|
JZ repeat_extend_back_end_calcBlockSizeSmall
|
|
|
|
repeat_extend_back_loop_calcBlockSizeSmall:
|
|
CMPL DI, SI
|
|
JBE repeat_extend_back_end_calcBlockSizeSmall
|
|
MOVB -1(BX)(R8*1), R9
|
|
MOVB -1(BX)(DI*1), R10
|
|
CMPB R9, R10
|
|
JNE repeat_extend_back_end_calcBlockSizeSmall
|
|
LEAL -1(DI), DI
|
|
DECL R8
|
|
JNZ repeat_extend_back_loop_calcBlockSizeSmall
|
|
|
|
repeat_extend_back_end_calcBlockSizeSmall:
|
|
MOVL DI, SI
|
|
SUBL 12(SP), SI
|
|
LEAQ 3(CX)(SI*1), SI
|
|
CMPQ SI, (SP)
|
|
JB repeat_dst_size_check_calcBlockSizeSmall
|
|
MOVQ $0x00000000, ret+32(FP)
|
|
RET
|
|
|
|
repeat_dst_size_check_calcBlockSizeSmall:
|
|
MOVL 12(SP), SI
|
|
CMPL SI, DI
|
|
JEQ emit_literal_done_repeat_emit_calcBlockSizeSmall
|
|
MOVL DI, R8
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(SI*1), R9
|
|
SUBL SI, R8
|
|
LEAL -1(R8), SI
|
|
CMPL SI, $0x3c
|
|
JB one_byte_repeat_emit_calcBlockSizeSmall
|
|
CMPL SI, $0x00000100
|
|
JB two_bytes_repeat_emit_calcBlockSizeSmall
|
|
JB three_bytes_repeat_emit_calcBlockSizeSmall
|
|
|
|
three_bytes_repeat_emit_calcBlockSizeSmall:
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_repeat_emit_calcBlockSizeSmall
|
|
|
|
two_bytes_repeat_emit_calcBlockSizeSmall:
|
|
ADDQ $0x02, CX
|
|
CMPL SI, $0x40
|
|
JB memmove_repeat_emit_calcBlockSizeSmall
|
|
JMP memmove_long_repeat_emit_calcBlockSizeSmall
|
|
|
|
one_byte_repeat_emit_calcBlockSizeSmall:
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_repeat_emit_calcBlockSizeSmall:
|
|
LEAQ (CX)(R8*1), CX
|
|
JMP emit_literal_done_repeat_emit_calcBlockSizeSmall
|
|
|
|
memmove_long_repeat_emit_calcBlockSizeSmall:
|
|
LEAQ (CX)(R8*1), CX
|
|
|
|
emit_literal_done_repeat_emit_calcBlockSizeSmall:
|
|
ADDL $0x05, DX
|
|
MOVL DX, SI
|
|
SUBL 16(SP), SI
|
|
MOVQ src_len+8(FP), R8
|
|
SUBL DX, R8
|
|
LEAQ (BX)(DX*1), R9
|
|
LEAQ (BX)(SI*1), SI
|
|
|
|
// matchLen
|
|
XORL R11, R11
|
|
|
|
matchlen_loopback_16_repeat_extend_calcBlockSizeSmall:
|
|
CMPL R8, $0x10
|
|
JB matchlen_match8_repeat_extend_calcBlockSizeSmall
|
|
MOVQ (R9)(R11*1), R10
|
|
MOVQ 8(R9)(R11*1), R12
|
|
XORQ (SI)(R11*1), R10
|
|
JNZ matchlen_bsf_8_repeat_extend_calcBlockSizeSmall
|
|
XORQ 8(SI)(R11*1), R12
|
|
JNZ matchlen_bsf_16repeat_extend_calcBlockSizeSmall
|
|
LEAL -16(R8), R8
|
|
LEAL 16(R11), R11
|
|
JMP matchlen_loopback_16_repeat_extend_calcBlockSizeSmall
|
|
|
|
matchlen_bsf_16repeat_extend_calcBlockSizeSmall:
|
|
TZCNTQ R12, R12
|
|
SARQ $0x03, R12
|
|
LEAL 8(R11)(R12*1), R11
|
|
JMP repeat_extend_forward_end_calcBlockSizeSmall
|
|
|
|
matchlen_match8_repeat_extend_calcBlockSizeSmall:
|
|
CMPL R8, $0x08
|
|
JB matchlen_match4_repeat_extend_calcBlockSizeSmall
|
|
MOVQ (R9)(R11*1), R10
|
|
XORQ (SI)(R11*1), R10
|
|
JNZ matchlen_bsf_8_repeat_extend_calcBlockSizeSmall
|
|
LEAL -8(R8), R8
|
|
LEAL 8(R11), R11
|
|
JMP matchlen_match4_repeat_extend_calcBlockSizeSmall
|
|
|
|
matchlen_bsf_8_repeat_extend_calcBlockSizeSmall:
|
|
TZCNTQ R10, R10
|
|
SARQ $0x03, R10
|
|
LEAL (R11)(R10*1), R11
|
|
JMP repeat_extend_forward_end_calcBlockSizeSmall
|
|
|
|
matchlen_match4_repeat_extend_calcBlockSizeSmall:
|
|
CMPL R8, $0x04
|
|
JB matchlen_match2_repeat_extend_calcBlockSizeSmall
|
|
MOVL (R9)(R11*1), R10
|
|
CMPL (SI)(R11*1), R10
|
|
JNE matchlen_match2_repeat_extend_calcBlockSizeSmall
|
|
LEAL -4(R8), R8
|
|
LEAL 4(R11), R11
|
|
|
|
matchlen_match2_repeat_extend_calcBlockSizeSmall:
|
|
CMPL R8, $0x01
|
|
JE matchlen_match1_repeat_extend_calcBlockSizeSmall
|
|
JB repeat_extend_forward_end_calcBlockSizeSmall
|
|
MOVW (R9)(R11*1), R10
|
|
CMPW (SI)(R11*1), R10
|
|
JNE matchlen_match1_repeat_extend_calcBlockSizeSmall
|
|
LEAL 2(R11), R11
|
|
SUBL $0x02, R8
|
|
JZ repeat_extend_forward_end_calcBlockSizeSmall
|
|
|
|
matchlen_match1_repeat_extend_calcBlockSizeSmall:
|
|
MOVB (R9)(R11*1), R10
|
|
CMPB (SI)(R11*1), R10
|
|
JNE repeat_extend_forward_end_calcBlockSizeSmall
|
|
LEAL 1(R11), R11
|
|
|
|
repeat_extend_forward_end_calcBlockSizeSmall:
|
|
ADDL R11, DX
|
|
MOVL DX, SI
|
|
SUBL DI, SI
|
|
MOVL 16(SP), DI
|
|
|
|
// emitCopy
|
|
two_byte_offset_repeat_as_copy_calcBlockSizeSmall:
|
|
CMPL SI, $0x40
|
|
JBE two_byte_offset_short_repeat_as_copy_calcBlockSizeSmall
|
|
LEAL -60(SI), SI
|
|
ADDQ $0x03, CX
|
|
JMP two_byte_offset_repeat_as_copy_calcBlockSizeSmall
|
|
|
|
two_byte_offset_short_repeat_as_copy_calcBlockSizeSmall:
|
|
MOVL SI, DI
|
|
SHLL $0x02, DI
|
|
CMPL SI, $0x0c
|
|
JAE emit_copy_three_repeat_as_copy_calcBlockSizeSmall
|
|
ADDQ $0x02, CX
|
|
JMP repeat_end_emit_calcBlockSizeSmall
|
|
|
|
emit_copy_three_repeat_as_copy_calcBlockSizeSmall:
|
|
ADDQ $0x03, CX
|
|
|
|
repeat_end_emit_calcBlockSizeSmall:
|
|
MOVL DX, 12(SP)
|
|
JMP search_loop_calcBlockSizeSmall
|
|
|
|
no_repeat_found_calcBlockSizeSmall:
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ candidate_match_calcBlockSizeSmall
|
|
SHRQ $0x08, DI
|
|
MOVL (AX)(R10*4), SI
|
|
LEAL 2(DX), R9
|
|
CMPL (BX)(R8*1), DI
|
|
JEQ candidate2_match_calcBlockSizeSmall
|
|
MOVL R9, (AX)(R10*4)
|
|
SHRQ $0x08, DI
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ candidate3_match_calcBlockSizeSmall
|
|
MOVL 20(SP), DX
|
|
JMP search_loop_calcBlockSizeSmall
|
|
|
|
candidate3_match_calcBlockSizeSmall:
|
|
ADDL $0x02, DX
|
|
JMP candidate_match_calcBlockSizeSmall
|
|
|
|
candidate2_match_calcBlockSizeSmall:
|
|
MOVL R9, (AX)(R10*4)
|
|
INCL DX
|
|
MOVL R8, SI
|
|
|
|
candidate_match_calcBlockSizeSmall:
|
|
MOVL 12(SP), DI
|
|
TESTL SI, SI
|
|
JZ match_extend_back_end_calcBlockSizeSmall
|
|
|
|
match_extend_back_loop_calcBlockSizeSmall:
|
|
CMPL DX, DI
|
|
JBE match_extend_back_end_calcBlockSizeSmall
|
|
MOVB -1(BX)(SI*1), R8
|
|
MOVB -1(BX)(DX*1), R9
|
|
CMPB R8, R9
|
|
JNE match_extend_back_end_calcBlockSizeSmall
|
|
LEAL -1(DX), DX
|
|
DECL SI
|
|
JZ match_extend_back_end_calcBlockSizeSmall
|
|
JMP match_extend_back_loop_calcBlockSizeSmall
|
|
|
|
match_extend_back_end_calcBlockSizeSmall:
|
|
MOVL DX, DI
|
|
SUBL 12(SP), DI
|
|
LEAQ 3(CX)(DI*1), DI
|
|
CMPQ DI, (SP)
|
|
JB match_dst_size_check_calcBlockSizeSmall
|
|
MOVQ $0x00000000, ret+32(FP)
|
|
RET
|
|
|
|
match_dst_size_check_calcBlockSizeSmall:
|
|
MOVL DX, DI
|
|
MOVL 12(SP), R8
|
|
CMPL R8, DI
|
|
JEQ emit_literal_done_match_emit_calcBlockSizeSmall
|
|
MOVL DI, R9
|
|
MOVL DI, 12(SP)
|
|
LEAQ (BX)(R8*1), DI
|
|
SUBL R8, R9
|
|
LEAL -1(R9), DI
|
|
CMPL DI, $0x3c
|
|
JB one_byte_match_emit_calcBlockSizeSmall
|
|
CMPL DI, $0x00000100
|
|
JB two_bytes_match_emit_calcBlockSizeSmall
|
|
JB three_bytes_match_emit_calcBlockSizeSmall
|
|
|
|
three_bytes_match_emit_calcBlockSizeSmall:
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_match_emit_calcBlockSizeSmall
|
|
|
|
two_bytes_match_emit_calcBlockSizeSmall:
|
|
ADDQ $0x02, CX
|
|
CMPL DI, $0x40
|
|
JB memmove_match_emit_calcBlockSizeSmall
|
|
JMP memmove_long_match_emit_calcBlockSizeSmall
|
|
|
|
one_byte_match_emit_calcBlockSizeSmall:
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_match_emit_calcBlockSizeSmall:
|
|
LEAQ (CX)(R9*1), CX
|
|
JMP emit_literal_done_match_emit_calcBlockSizeSmall
|
|
|
|
memmove_long_match_emit_calcBlockSizeSmall:
|
|
LEAQ (CX)(R9*1), CX
|
|
|
|
emit_literal_done_match_emit_calcBlockSizeSmall:
|
|
match_nolit_loop_calcBlockSizeSmall:
|
|
MOVL DX, DI
|
|
SUBL SI, DI
|
|
MOVL DI, 16(SP)
|
|
ADDL $0x04, DX
|
|
ADDL $0x04, SI
|
|
MOVQ src_len+8(FP), DI
|
|
SUBL DX, DI
|
|
LEAQ (BX)(DX*1), R8
|
|
LEAQ (BX)(SI*1), SI
|
|
|
|
// matchLen
|
|
XORL R10, R10
|
|
|
|
matchlen_loopback_16_match_nolit_calcBlockSizeSmall:
|
|
CMPL DI, $0x10
|
|
JB matchlen_match8_match_nolit_calcBlockSizeSmall
|
|
MOVQ (R8)(R10*1), R9
|
|
MOVQ 8(R8)(R10*1), R11
|
|
XORQ (SI)(R10*1), R9
|
|
JNZ matchlen_bsf_8_match_nolit_calcBlockSizeSmall
|
|
XORQ 8(SI)(R10*1), R11
|
|
JNZ matchlen_bsf_16match_nolit_calcBlockSizeSmall
|
|
LEAL -16(DI), DI
|
|
LEAL 16(R10), R10
|
|
JMP matchlen_loopback_16_match_nolit_calcBlockSizeSmall
|
|
|
|
matchlen_bsf_16match_nolit_calcBlockSizeSmall:
|
|
TZCNTQ R11, R11
|
|
SARQ $0x03, R11
|
|
LEAL 8(R10)(R11*1), R10
|
|
JMP match_nolit_end_calcBlockSizeSmall
|
|
|
|
matchlen_match8_match_nolit_calcBlockSizeSmall:
|
|
CMPL DI, $0x08
|
|
JB matchlen_match4_match_nolit_calcBlockSizeSmall
|
|
MOVQ (R8)(R10*1), R9
|
|
XORQ (SI)(R10*1), R9
|
|
JNZ matchlen_bsf_8_match_nolit_calcBlockSizeSmall
|
|
LEAL -8(DI), DI
|
|
LEAL 8(R10), R10
|
|
JMP matchlen_match4_match_nolit_calcBlockSizeSmall
|
|
|
|
matchlen_bsf_8_match_nolit_calcBlockSizeSmall:
|
|
TZCNTQ R9, R9
|
|
SARQ $0x03, R9
|
|
LEAL (R10)(R9*1), R10
|
|
JMP match_nolit_end_calcBlockSizeSmall
|
|
|
|
matchlen_match4_match_nolit_calcBlockSizeSmall:
|
|
CMPL DI, $0x04
|
|
JB matchlen_match2_match_nolit_calcBlockSizeSmall
|
|
MOVL (R8)(R10*1), R9
|
|
CMPL (SI)(R10*1), R9
|
|
JNE matchlen_match2_match_nolit_calcBlockSizeSmall
|
|
LEAL -4(DI), DI
|
|
LEAL 4(R10), R10
|
|
|
|
matchlen_match2_match_nolit_calcBlockSizeSmall:
|
|
CMPL DI, $0x01
|
|
JE matchlen_match1_match_nolit_calcBlockSizeSmall
|
|
JB match_nolit_end_calcBlockSizeSmall
|
|
MOVW (R8)(R10*1), R9
|
|
CMPW (SI)(R10*1), R9
|
|
JNE matchlen_match1_match_nolit_calcBlockSizeSmall
|
|
LEAL 2(R10), R10
|
|
SUBL $0x02, DI
|
|
JZ match_nolit_end_calcBlockSizeSmall
|
|
|
|
matchlen_match1_match_nolit_calcBlockSizeSmall:
|
|
MOVB (R8)(R10*1), R9
|
|
CMPB (SI)(R10*1), R9
|
|
JNE match_nolit_end_calcBlockSizeSmall
|
|
LEAL 1(R10), R10
|
|
|
|
match_nolit_end_calcBlockSizeSmall:
|
|
ADDL R10, DX
|
|
MOVL 16(SP), SI
|
|
ADDL $0x04, R10
|
|
MOVL DX, 12(SP)
|
|
|
|
// emitCopy
|
|
two_byte_offset_match_nolit_calcBlockSizeSmall:
|
|
CMPL R10, $0x40
|
|
JBE two_byte_offset_short_match_nolit_calcBlockSizeSmall
|
|
LEAL -60(R10), R10
|
|
ADDQ $0x03, CX
|
|
JMP two_byte_offset_match_nolit_calcBlockSizeSmall
|
|
|
|
two_byte_offset_short_match_nolit_calcBlockSizeSmall:
|
|
MOVL R10, SI
|
|
SHLL $0x02, SI
|
|
CMPL R10, $0x0c
|
|
JAE emit_copy_three_match_nolit_calcBlockSizeSmall
|
|
ADDQ $0x02, CX
|
|
JMP match_nolit_emitcopy_end_calcBlockSizeSmall
|
|
|
|
emit_copy_three_match_nolit_calcBlockSizeSmall:
|
|
ADDQ $0x03, CX
|
|
|
|
match_nolit_emitcopy_end_calcBlockSizeSmall:
|
|
CMPL DX, 8(SP)
|
|
JAE emit_remainder_calcBlockSizeSmall
|
|
MOVQ -2(BX)(DX*1), DI
|
|
CMPQ CX, (SP)
|
|
JB match_nolit_dst_ok_calcBlockSizeSmall
|
|
MOVQ $0x00000000, ret+32(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_calcBlockSizeSmall:
|
|
MOVQ $0x9e3779b1, R9
|
|
MOVQ DI, R8
|
|
SHRQ $0x10, DI
|
|
MOVQ DI, SI
|
|
SHLQ $0x20, R8
|
|
IMULQ R9, R8
|
|
SHRQ $0x37, R8
|
|
SHLQ $0x20, SI
|
|
IMULQ R9, SI
|
|
SHRQ $0x37, SI
|
|
LEAL -2(DX), R9
|
|
LEAQ (AX)(SI*4), R10
|
|
MOVL (R10), SI
|
|
MOVL R9, (AX)(R8*4)
|
|
MOVL DX, (R10)
|
|
CMPL (BX)(SI*1), DI
|
|
JEQ match_nolit_loop_calcBlockSizeSmall
|
|
INCL DX
|
|
JMP search_loop_calcBlockSizeSmall
|
|
|
|
emit_remainder_calcBlockSizeSmall:
|
|
MOVQ src_len+8(FP), AX
|
|
SUBL 12(SP), AX
|
|
LEAQ 3(CX)(AX*1), AX
|
|
CMPQ AX, (SP)
|
|
JB emit_remainder_ok_calcBlockSizeSmall
|
|
MOVQ $0x00000000, ret+32(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_calcBlockSizeSmall:
|
|
MOVQ src_len+8(FP), AX
|
|
MOVL 12(SP), DX
|
|
CMPL DX, AX
|
|
JEQ emit_literal_done_emit_remainder_calcBlockSizeSmall
|
|
MOVL AX, SI
|
|
MOVL AX, 12(SP)
|
|
LEAQ (BX)(DX*1), AX
|
|
SUBL DX, SI
|
|
LEAL -1(SI), AX
|
|
CMPL AX, $0x3c
|
|
JB one_byte_emit_remainder_calcBlockSizeSmall
|
|
CMPL AX, $0x00000100
|
|
JB two_bytes_emit_remainder_calcBlockSizeSmall
|
|
JB three_bytes_emit_remainder_calcBlockSizeSmall
|
|
|
|
three_bytes_emit_remainder_calcBlockSizeSmall:
|
|
ADDQ $0x03, CX
|
|
JMP memmove_long_emit_remainder_calcBlockSizeSmall
|
|
|
|
two_bytes_emit_remainder_calcBlockSizeSmall:
|
|
ADDQ $0x02, CX
|
|
CMPL AX, $0x40
|
|
JB memmove_emit_remainder_calcBlockSizeSmall
|
|
JMP memmove_long_emit_remainder_calcBlockSizeSmall
|
|
|
|
one_byte_emit_remainder_calcBlockSizeSmall:
|
|
ADDQ $0x01, CX
|
|
|
|
memmove_emit_remainder_calcBlockSizeSmall:
|
|
LEAQ (CX)(SI*1), AX
|
|
MOVQ AX, CX
|
|
JMP emit_literal_done_emit_remainder_calcBlockSizeSmall
|
|
|
|
memmove_long_emit_remainder_calcBlockSizeSmall:
|
|
LEAQ (CX)(SI*1), AX
|
|
MOVQ AX, CX
|
|
|
|
emit_literal_done_emit_remainder_calcBlockSizeSmall:
|
|
MOVQ CX, ret+32(FP)
|
|
RET
|
|
|
|
// func emitLiteral(dst []byte, lit []byte) int
|
|
// Requires: SSE2
|
|
TEXT ·emitLiteral(SB), NOSPLIT, $0-56
|
|
MOVQ lit_len+32(FP), DX
|
|
MOVQ dst_base+0(FP), AX
|
|
MOVQ lit_base+24(FP), CX
|
|
TESTQ DX, DX
|
|
JZ emit_literal_end_standalone_skip
|
|
MOVL DX, BX
|
|
LEAL -1(DX), SI
|
|
CMPL SI, $0x3c
|
|
JB one_byte_standalone
|
|
CMPL SI, $0x00000100
|
|
JB two_bytes_standalone
|
|
CMPL SI, $0x00010000
|
|
JB three_bytes_standalone
|
|
CMPL SI, $0x01000000
|
|
JB four_bytes_standalone
|
|
MOVB $0xfc, (AX)
|
|
MOVL SI, 1(AX)
|
|
ADDQ $0x05, BX
|
|
ADDQ $0x05, AX
|
|
JMP memmove_long_standalone
|
|
|
|
four_bytes_standalone:
|
|
MOVL SI, DI
|
|
SHRL $0x10, DI
|
|
MOVB $0xf8, (AX)
|
|
MOVW SI, 1(AX)
|
|
MOVB DI, 3(AX)
|
|
ADDQ $0x04, BX
|
|
ADDQ $0x04, AX
|
|
JMP memmove_long_standalone
|
|
|
|
three_bytes_standalone:
|
|
MOVB $0xf4, (AX)
|
|
MOVW SI, 1(AX)
|
|
ADDQ $0x03, BX
|
|
ADDQ $0x03, AX
|
|
JMP memmove_long_standalone
|
|
|
|
two_bytes_standalone:
|
|
MOVB $0xf0, (AX)
|
|
MOVB SI, 1(AX)
|
|
ADDQ $0x02, BX
|
|
ADDQ $0x02, AX
|
|
CMPL SI, $0x40
|
|
JB memmove_standalone
|
|
JMP memmove_long_standalone
|
|
|
|
one_byte_standalone:
|
|
SHLB $0x02, SI
|
|
MOVB SI, (AX)
|
|
ADDQ $0x01, BX
|
|
ADDQ $0x01, AX
|
|
|
|
memmove_standalone:
|
|
// genMemMoveShort
|
|
CMPQ DX, $0x03
|
|
JB emit_lit_memmove_standalone_memmove_move_1or2
|
|
JE emit_lit_memmove_standalone_memmove_move_3
|
|
CMPQ DX, $0x08
|
|
JB emit_lit_memmove_standalone_memmove_move_4through7
|
|
CMPQ DX, $0x10
|
|
JBE emit_lit_memmove_standalone_memmove_move_8through16
|
|
CMPQ DX, $0x20
|
|
JBE emit_lit_memmove_standalone_memmove_move_17through32
|
|
JMP emit_lit_memmove_standalone_memmove_move_33through64
|
|
|
|
emit_lit_memmove_standalone_memmove_move_1or2:
|
|
MOVB (CX), SI
|
|
MOVB -1(CX)(DX*1), CL
|
|
MOVB SI, (AX)
|
|
MOVB CL, -1(AX)(DX*1)
|
|
JMP emit_literal_end_standalone
|
|
|
|
emit_lit_memmove_standalone_memmove_move_3:
|
|
MOVW (CX), SI
|
|
MOVB 2(CX), CL
|
|
MOVW SI, (AX)
|
|
MOVB CL, 2(AX)
|
|
JMP emit_literal_end_standalone
|
|
|
|
emit_lit_memmove_standalone_memmove_move_4through7:
|
|
MOVL (CX), SI
|
|
MOVL -4(CX)(DX*1), CX
|
|
MOVL SI, (AX)
|
|
MOVL CX, -4(AX)(DX*1)
|
|
JMP emit_literal_end_standalone
|
|
|
|
emit_lit_memmove_standalone_memmove_move_8through16:
|
|
MOVQ (CX), SI
|
|
MOVQ -8(CX)(DX*1), CX
|
|
MOVQ SI, (AX)
|
|
MOVQ CX, -8(AX)(DX*1)
|
|
JMP emit_literal_end_standalone
|
|
|
|
emit_lit_memmove_standalone_memmove_move_17through32:
|
|
MOVOU (CX), X0
|
|
MOVOU -16(CX)(DX*1), X1
|
|
MOVOU X0, (AX)
|
|
MOVOU X1, -16(AX)(DX*1)
|
|
JMP emit_literal_end_standalone
|
|
|
|
emit_lit_memmove_standalone_memmove_move_33through64:
|
|
MOVOU (CX), X0
|
|
MOVOU 16(CX), X1
|
|
MOVOU -32(CX)(DX*1), X2
|
|
MOVOU -16(CX)(DX*1), X3
|
|
MOVOU X0, (AX)
|
|
MOVOU X1, 16(AX)
|
|
MOVOU X2, -32(AX)(DX*1)
|
|
MOVOU X3, -16(AX)(DX*1)
|
|
JMP emit_literal_end_standalone
|
|
JMP emit_literal_end_standalone
|
|
|
|
memmove_long_standalone:
|
|
// genMemMoveLong
|
|
MOVOU (CX), X0
|
|
MOVOU 16(CX), X1
|
|
MOVOU -32(CX)(DX*1), X2
|
|
MOVOU -16(CX)(DX*1), X3
|
|
MOVQ DX, DI
|
|
SHRQ $0x05, DI
|
|
MOVQ AX, SI
|
|
ANDL $0x0000001f, SI
|
|
MOVQ $0x00000040, R8
|
|
SUBQ SI, R8
|
|
DECQ DI
|
|
JA emit_lit_memmove_long_standalonelarge_forward_sse_loop_32
|
|
LEAQ -32(CX)(R8*1), SI
|
|
LEAQ -32(AX)(R8*1), R9
|
|
|
|
emit_lit_memmove_long_standalonelarge_big_loop_back:
|
|
MOVOU (SI), X4
|
|
MOVOU 16(SI), X5
|
|
MOVOA X4, (R9)
|
|
MOVOA X5, 16(R9)
|
|
ADDQ $0x20, R9
|
|
ADDQ $0x20, SI
|
|
ADDQ $0x20, R8
|
|
DECQ DI
|
|
JNA emit_lit_memmove_long_standalonelarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_standalonelarge_forward_sse_loop_32:
|
|
MOVOU -32(CX)(R8*1), X4
|
|
MOVOU -16(CX)(R8*1), X5
|
|
MOVOA X4, -32(AX)(R8*1)
|
|
MOVOA X5, -16(AX)(R8*1)
|
|
ADDQ $0x20, R8
|
|
CMPQ DX, R8
|
|
JAE emit_lit_memmove_long_standalonelarge_forward_sse_loop_32
|
|
MOVOU X0, (AX)
|
|
MOVOU X1, 16(AX)
|
|
MOVOU X2, -32(AX)(DX*1)
|
|
MOVOU X3, -16(AX)(DX*1)
|
|
JMP emit_literal_end_standalone
|
|
JMP emit_literal_end_standalone
|
|
|
|
emit_literal_end_standalone_skip:
|
|
XORQ BX, BX
|
|
|
|
emit_literal_end_standalone:
|
|
MOVQ BX, ret+48(FP)
|
|
RET
|
|
|
|
// func emitRepeat(dst []byte, offset int, length int) int
|
|
TEXT ·emitRepeat(SB), NOSPLIT, $0-48
|
|
XORQ BX, BX
|
|
MOVQ dst_base+0(FP), AX
|
|
MOVQ offset+24(FP), CX
|
|
MOVQ length+32(FP), DX
|
|
|
|
// emitRepeat
|
|
emit_repeat_again_standalone:
|
|
MOVL DX, SI
|
|
LEAL -4(DX), DX
|
|
CMPL SI, $0x08
|
|
JBE repeat_two_standalone
|
|
CMPL SI, $0x0c
|
|
JAE cant_repeat_two_offset_standalone
|
|
CMPL CX, $0x00000800
|
|
JB repeat_two_offset_standalone
|
|
|
|
cant_repeat_two_offset_standalone:
|
|
CMPL DX, $0x00000104
|
|
JB repeat_three_standalone
|
|
CMPL DX, $0x00010100
|
|
JB repeat_four_standalone
|
|
CMPL DX, $0x0100ffff
|
|
JB repeat_five_standalone
|
|
LEAL -16842747(DX), DX
|
|
MOVL $0xfffb001d, (AX)
|
|
MOVB $0xff, 4(AX)
|
|
ADDQ $0x05, AX
|
|
ADDQ $0x05, BX
|
|
JMP emit_repeat_again_standalone
|
|
|
|
repeat_five_standalone:
|
|
LEAL -65536(DX), DX
|
|
MOVL DX, CX
|
|
MOVW $0x001d, (AX)
|
|
MOVW DX, 2(AX)
|
|
SARL $0x10, CX
|
|
MOVB CL, 4(AX)
|
|
ADDQ $0x05, BX
|
|
ADDQ $0x05, AX
|
|
JMP gen_emit_repeat_end
|
|
|
|
repeat_four_standalone:
|
|
LEAL -256(DX), DX
|
|
MOVW $0x0019, (AX)
|
|
MOVW DX, 2(AX)
|
|
ADDQ $0x04, BX
|
|
ADDQ $0x04, AX
|
|
JMP gen_emit_repeat_end
|
|
|
|
repeat_three_standalone:
|
|
LEAL -4(DX), DX
|
|
MOVW $0x0015, (AX)
|
|
MOVB DL, 2(AX)
|
|
ADDQ $0x03, BX
|
|
ADDQ $0x03, AX
|
|
JMP gen_emit_repeat_end
|
|
|
|
repeat_two_standalone:
|
|
SHLL $0x02, DX
|
|
ORL $0x01, DX
|
|
MOVW DX, (AX)
|
|
ADDQ $0x02, BX
|
|
ADDQ $0x02, AX
|
|
JMP gen_emit_repeat_end
|
|
|
|
repeat_two_offset_standalone:
|
|
XORQ SI, SI
|
|
LEAL 1(SI)(DX*4), DX
|
|
MOVB CL, 1(AX)
|
|
SARL $0x08, CX
|
|
SHLL $0x05, CX
|
|
ORL CX, DX
|
|
MOVB DL, (AX)
|
|
ADDQ $0x02, BX
|
|
ADDQ $0x02, AX
|
|
|
|
gen_emit_repeat_end:
|
|
MOVQ BX, ret+40(FP)
|
|
RET
|
|
|
|
// func emitCopy(dst []byte, offset int, length int) int
|
|
TEXT ·emitCopy(SB), NOSPLIT, $0-48
|
|
XORQ BX, BX
|
|
MOVQ dst_base+0(FP), AX
|
|
MOVQ offset+24(FP), CX
|
|
MOVQ length+32(FP), DX
|
|
|
|
// emitCopy
|
|
CMPL CX, $0x00010000
|
|
JB two_byte_offset_standalone
|
|
CMPL DX, $0x40
|
|
JBE four_bytes_remain_standalone
|
|
MOVB $0xff, (AX)
|
|
MOVL CX, 1(AX)
|
|
LEAL -64(DX), DX
|
|
ADDQ $0x05, BX
|
|
ADDQ $0x05, AX
|
|
CMPL DX, $0x04
|
|
JB four_bytes_remain_standalone
|
|
|
|
// emitRepeat
|
|
emit_repeat_again_standalone_emit_copy:
|
|
MOVL DX, SI
|
|
LEAL -4(DX), DX
|
|
CMPL SI, $0x08
|
|
JBE repeat_two_standalone_emit_copy
|
|
CMPL SI, $0x0c
|
|
JAE cant_repeat_two_offset_standalone_emit_copy
|
|
CMPL CX, $0x00000800
|
|
JB repeat_two_offset_standalone_emit_copy
|
|
|
|
cant_repeat_two_offset_standalone_emit_copy:
|
|
CMPL DX, $0x00000104
|
|
JB repeat_three_standalone_emit_copy
|
|
CMPL DX, $0x00010100
|
|
JB repeat_four_standalone_emit_copy
|
|
CMPL DX, $0x0100ffff
|
|
JB repeat_five_standalone_emit_copy
|
|
LEAL -16842747(DX), DX
|
|
MOVL $0xfffb001d, (AX)
|
|
MOVB $0xff, 4(AX)
|
|
ADDQ $0x05, AX
|
|
ADDQ $0x05, BX
|
|
JMP emit_repeat_again_standalone_emit_copy
|
|
|
|
repeat_five_standalone_emit_copy:
|
|
LEAL -65536(DX), DX
|
|
MOVL DX, CX
|
|
MOVW $0x001d, (AX)
|
|
MOVW DX, 2(AX)
|
|
SARL $0x10, CX
|
|
MOVB CL, 4(AX)
|
|
ADDQ $0x05, BX
|
|
ADDQ $0x05, AX
|
|
JMP gen_emit_copy_end
|
|
|
|
repeat_four_standalone_emit_copy:
|
|
LEAL -256(DX), DX
|
|
MOVW $0x0019, (AX)
|
|
MOVW DX, 2(AX)
|
|
ADDQ $0x04, BX
|
|
ADDQ $0x04, AX
|
|
JMP gen_emit_copy_end
|
|
|
|
repeat_three_standalone_emit_copy:
|
|
LEAL -4(DX), DX
|
|
MOVW $0x0015, (AX)
|
|
MOVB DL, 2(AX)
|
|
ADDQ $0x03, BX
|
|
ADDQ $0x03, AX
|
|
JMP gen_emit_copy_end
|
|
|
|
repeat_two_standalone_emit_copy:
|
|
SHLL $0x02, DX
|
|
ORL $0x01, DX
|
|
MOVW DX, (AX)
|
|
ADDQ $0x02, BX
|
|
ADDQ $0x02, AX
|
|
JMP gen_emit_copy_end
|
|
|
|
repeat_two_offset_standalone_emit_copy:
|
|
XORQ SI, SI
|
|
LEAL 1(SI)(DX*4), DX
|
|
MOVB CL, 1(AX)
|
|
SARL $0x08, CX
|
|
SHLL $0x05, CX
|
|
ORL CX, DX
|
|
MOVB DL, (AX)
|
|
ADDQ $0x02, BX
|
|
ADDQ $0x02, AX
|
|
JMP gen_emit_copy_end
|
|
|
|
four_bytes_remain_standalone:
|
|
TESTL DX, DX
|
|
JZ gen_emit_copy_end
|
|
XORL SI, SI
|
|
LEAL -1(SI)(DX*4), DX
|
|
MOVB DL, (AX)
|
|
MOVL CX, 1(AX)
|
|
ADDQ $0x05, BX
|
|
ADDQ $0x05, AX
|
|
JMP gen_emit_copy_end
|
|
|
|
two_byte_offset_standalone:
|
|
CMPL DX, $0x40
|
|
JBE two_byte_offset_short_standalone
|
|
CMPL CX, $0x00000800
|
|
JAE long_offset_short_standalone
|
|
MOVL $0x00000001, SI
|
|
LEAL 16(SI), SI
|
|
MOVB CL, 1(AX)
|
|
MOVL CX, DI
|
|
SHRL $0x08, DI
|
|
SHLL $0x05, DI
|
|
ORL DI, SI
|
|
MOVB SI, (AX)
|
|
ADDQ $0x02, BX
|
|
ADDQ $0x02, AX
|
|
SUBL $0x08, DX
|
|
|
|
// emitRepeat
|
|
LEAL -4(DX), DX
|
|
JMP cant_repeat_two_offset_standalone_emit_copy_short_2b
|
|
|
|
emit_repeat_again_standalone_emit_copy_short_2b:
|
|
MOVL DX, SI
|
|
LEAL -4(DX), DX
|
|
CMPL SI, $0x08
|
|
JBE repeat_two_standalone_emit_copy_short_2b
|
|
CMPL SI, $0x0c
|
|
JAE cant_repeat_two_offset_standalone_emit_copy_short_2b
|
|
CMPL CX, $0x00000800
|
|
JB repeat_two_offset_standalone_emit_copy_short_2b
|
|
|
|
cant_repeat_two_offset_standalone_emit_copy_short_2b:
|
|
CMPL DX, $0x00000104
|
|
JB repeat_three_standalone_emit_copy_short_2b
|
|
CMPL DX, $0x00010100
|
|
JB repeat_four_standalone_emit_copy_short_2b
|
|
CMPL DX, $0x0100ffff
|
|
JB repeat_five_standalone_emit_copy_short_2b
|
|
LEAL -16842747(DX), DX
|
|
MOVL $0xfffb001d, (AX)
|
|
MOVB $0xff, 4(AX)
|
|
ADDQ $0x05, AX
|
|
ADDQ $0x05, BX
|
|
JMP emit_repeat_again_standalone_emit_copy_short_2b
|
|
|
|
repeat_five_standalone_emit_copy_short_2b:
|
|
LEAL -65536(DX), DX
|
|
MOVL DX, CX
|
|
MOVW $0x001d, (AX)
|
|
MOVW DX, 2(AX)
|
|
SARL $0x10, CX
|
|
MOVB CL, 4(AX)
|
|
ADDQ $0x05, BX
|
|
ADDQ $0x05, AX
|
|
JMP gen_emit_copy_end
|
|
|
|
repeat_four_standalone_emit_copy_short_2b:
|
|
LEAL -256(DX), DX
|
|
MOVW $0x0019, (AX)
|
|
MOVW DX, 2(AX)
|
|
ADDQ $0x04, BX
|
|
ADDQ $0x04, AX
|
|
JMP gen_emit_copy_end
|
|
|
|
repeat_three_standalone_emit_copy_short_2b:
|
|
LEAL -4(DX), DX
|
|
MOVW $0x0015, (AX)
|
|
MOVB DL, 2(AX)
|
|
ADDQ $0x03, BX
|
|
ADDQ $0x03, AX
|
|
JMP gen_emit_copy_end
|
|
|
|
repeat_two_standalone_emit_copy_short_2b:
|
|
SHLL $0x02, DX
|
|
ORL $0x01, DX
|
|
MOVW DX, (AX)
|
|
ADDQ $0x02, BX
|
|
ADDQ $0x02, AX
|
|
JMP gen_emit_copy_end
|
|
|
|
repeat_two_offset_standalone_emit_copy_short_2b:
|
|
XORQ SI, SI
|
|
LEAL 1(SI)(DX*4), DX
|
|
MOVB CL, 1(AX)
|
|
SARL $0x08, CX
|
|
SHLL $0x05, CX
|
|
ORL CX, DX
|
|
MOVB DL, (AX)
|
|
ADDQ $0x02, BX
|
|
ADDQ $0x02, AX
|
|
JMP gen_emit_copy_end
|
|
|
|
long_offset_short_standalone:
|
|
MOVB $0xee, (AX)
|
|
MOVW CX, 1(AX)
|
|
LEAL -60(DX), DX
|
|
ADDQ $0x03, AX
|
|
ADDQ $0x03, BX
|
|
|
|
// emitRepeat
|
|
emit_repeat_again_standalone_emit_copy_short:
|
|
MOVL DX, SI
|
|
LEAL -4(DX), DX
|
|
CMPL SI, $0x08
|
|
JBE repeat_two_standalone_emit_copy_short
|
|
CMPL SI, $0x0c
|
|
JAE cant_repeat_two_offset_standalone_emit_copy_short
|
|
CMPL CX, $0x00000800
|
|
JB repeat_two_offset_standalone_emit_copy_short
|
|
|
|
cant_repeat_two_offset_standalone_emit_copy_short:
|
|
CMPL DX, $0x00000104
|
|
JB repeat_three_standalone_emit_copy_short
|
|
CMPL DX, $0x00010100
|
|
JB repeat_four_standalone_emit_copy_short
|
|
CMPL DX, $0x0100ffff
|
|
JB repeat_five_standalone_emit_copy_short
|
|
LEAL -16842747(DX), DX
|
|
MOVL $0xfffb001d, (AX)
|
|
MOVB $0xff, 4(AX)
|
|
ADDQ $0x05, AX
|
|
ADDQ $0x05, BX
|
|
JMP emit_repeat_again_standalone_emit_copy_short
|
|
|
|
repeat_five_standalone_emit_copy_short:
|
|
LEAL -65536(DX), DX
|
|
MOVL DX, CX
|
|
MOVW $0x001d, (AX)
|
|
MOVW DX, 2(AX)
|
|
SARL $0x10, CX
|
|
MOVB CL, 4(AX)
|
|
ADDQ $0x05, BX
|
|
ADDQ $0x05, AX
|
|
JMP gen_emit_copy_end
|
|
|
|
repeat_four_standalone_emit_copy_short:
|
|
LEAL -256(DX), DX
|
|
MOVW $0x0019, (AX)
|
|
MOVW DX, 2(AX)
|
|
ADDQ $0x04, BX
|
|
ADDQ $0x04, AX
|
|
JMP gen_emit_copy_end
|
|
|
|
repeat_three_standalone_emit_copy_short:
|
|
LEAL -4(DX), DX
|
|
MOVW $0x0015, (AX)
|
|
MOVB DL, 2(AX)
|
|
ADDQ $0x03, BX
|
|
ADDQ $0x03, AX
|
|
JMP gen_emit_copy_end
|
|
|
|
repeat_two_standalone_emit_copy_short:
|
|
SHLL $0x02, DX
|
|
ORL $0x01, DX
|
|
MOVW DX, (AX)
|
|
ADDQ $0x02, BX
|
|
ADDQ $0x02, AX
|
|
JMP gen_emit_copy_end
|
|
|
|
repeat_two_offset_standalone_emit_copy_short:
|
|
XORQ SI, SI
|
|
LEAL 1(SI)(DX*4), DX
|
|
MOVB CL, 1(AX)
|
|
SARL $0x08, CX
|
|
SHLL $0x05, CX
|
|
ORL CX, DX
|
|
MOVB DL, (AX)
|
|
ADDQ $0x02, BX
|
|
ADDQ $0x02, AX
|
|
JMP gen_emit_copy_end
|
|
|
|
two_byte_offset_short_standalone:
|
|
MOVL DX, SI
|
|
SHLL $0x02, SI
|
|
CMPL DX, $0x0c
|
|
JAE emit_copy_three_standalone
|
|
CMPL CX, $0x00000800
|
|
JAE emit_copy_three_standalone
|
|
LEAL -15(SI), SI
|
|
MOVB CL, 1(AX)
|
|
SHRL $0x08, CX
|
|
SHLL $0x05, CX
|
|
ORL CX, SI
|
|
MOVB SI, (AX)
|
|
ADDQ $0x02, BX
|
|
ADDQ $0x02, AX
|
|
JMP gen_emit_copy_end
|
|
|
|
emit_copy_three_standalone:
|
|
LEAL -2(SI), SI
|
|
MOVB SI, (AX)
|
|
MOVW CX, 1(AX)
|
|
ADDQ $0x03, BX
|
|
ADDQ $0x03, AX
|
|
|
|
gen_emit_copy_end:
|
|
MOVQ BX, ret+40(FP)
|
|
RET
|
|
|
|
// func emitCopyNoRepeat(dst []byte, offset int, length int) int
|
|
TEXT ·emitCopyNoRepeat(SB), NOSPLIT, $0-48
|
|
XORQ BX, BX
|
|
MOVQ dst_base+0(FP), AX
|
|
MOVQ offset+24(FP), CX
|
|
MOVQ length+32(FP), DX
|
|
|
|
// emitCopy
|
|
CMPL CX, $0x00010000
|
|
JB two_byte_offset_standalone_snappy
|
|
|
|
four_bytes_loop_back_standalone_snappy:
|
|
CMPL DX, $0x40
|
|
JBE four_bytes_remain_standalone_snappy
|
|
MOVB $0xff, (AX)
|
|
MOVL CX, 1(AX)
|
|
LEAL -64(DX), DX
|
|
ADDQ $0x05, BX
|
|
ADDQ $0x05, AX
|
|
CMPL DX, $0x04
|
|
JB four_bytes_remain_standalone_snappy
|
|
JMP four_bytes_loop_back_standalone_snappy
|
|
|
|
four_bytes_remain_standalone_snappy:
|
|
TESTL DX, DX
|
|
JZ gen_emit_copy_end_snappy
|
|
XORL SI, SI
|
|
LEAL -1(SI)(DX*4), DX
|
|
MOVB DL, (AX)
|
|
MOVL CX, 1(AX)
|
|
ADDQ $0x05, BX
|
|
ADDQ $0x05, AX
|
|
JMP gen_emit_copy_end_snappy
|
|
|
|
two_byte_offset_standalone_snappy:
|
|
CMPL DX, $0x40
|
|
JBE two_byte_offset_short_standalone_snappy
|
|
MOVB $0xee, (AX)
|
|
MOVW CX, 1(AX)
|
|
LEAL -60(DX), DX
|
|
ADDQ $0x03, AX
|
|
ADDQ $0x03, BX
|
|
JMP two_byte_offset_standalone_snappy
|
|
|
|
two_byte_offset_short_standalone_snappy:
|
|
MOVL DX, SI
|
|
SHLL $0x02, SI
|
|
CMPL DX, $0x0c
|
|
JAE emit_copy_three_standalone_snappy
|
|
CMPL CX, $0x00000800
|
|
JAE emit_copy_three_standalone_snappy
|
|
LEAL -15(SI), SI
|
|
MOVB CL, 1(AX)
|
|
SHRL $0x08, CX
|
|
SHLL $0x05, CX
|
|
ORL CX, SI
|
|
MOVB SI, (AX)
|
|
ADDQ $0x02, BX
|
|
ADDQ $0x02, AX
|
|
JMP gen_emit_copy_end_snappy
|
|
|
|
emit_copy_three_standalone_snappy:
|
|
LEAL -2(SI), SI
|
|
MOVB SI, (AX)
|
|
MOVW CX, 1(AX)
|
|
ADDQ $0x03, BX
|
|
ADDQ $0x03, AX
|
|
|
|
gen_emit_copy_end_snappy:
|
|
MOVQ BX, ret+40(FP)
|
|
RET
|
|
|
|
// func matchLen(a []byte, b []byte) int
|
|
// Requires: BMI
|
|
TEXT ·matchLen(SB), NOSPLIT, $0-56
|
|
MOVQ a_base+0(FP), AX
|
|
MOVQ b_base+24(FP), CX
|
|
MOVQ a_len+8(FP), DX
|
|
|
|
// matchLen
|
|
XORL SI, SI
|
|
|
|
matchlen_loopback_16_standalone:
|
|
CMPL DX, $0x10
|
|
JB matchlen_match8_standalone
|
|
MOVQ (AX)(SI*1), BX
|
|
MOVQ 8(AX)(SI*1), DI
|
|
XORQ (CX)(SI*1), BX
|
|
JNZ matchlen_bsf_8_standalone
|
|
XORQ 8(CX)(SI*1), DI
|
|
JNZ matchlen_bsf_16standalone
|
|
LEAL -16(DX), DX
|
|
LEAL 16(SI), SI
|
|
JMP matchlen_loopback_16_standalone
|
|
|
|
matchlen_bsf_16standalone:
|
|
TZCNTQ DI, DI
|
|
SARQ $0x03, DI
|
|
LEAL 8(SI)(DI*1), SI
|
|
JMP gen_match_len_end
|
|
|
|
matchlen_match8_standalone:
|
|
CMPL DX, $0x08
|
|
JB matchlen_match4_standalone
|
|
MOVQ (AX)(SI*1), BX
|
|
XORQ (CX)(SI*1), BX
|
|
JNZ matchlen_bsf_8_standalone
|
|
LEAL -8(DX), DX
|
|
LEAL 8(SI), SI
|
|
JMP matchlen_match4_standalone
|
|
|
|
matchlen_bsf_8_standalone:
|
|
TZCNTQ BX, BX
|
|
SARQ $0x03, BX
|
|
LEAL (SI)(BX*1), SI
|
|
JMP gen_match_len_end
|
|
|
|
matchlen_match4_standalone:
|
|
CMPL DX, $0x04
|
|
JB matchlen_match2_standalone
|
|
MOVL (AX)(SI*1), BX
|
|
CMPL (CX)(SI*1), BX
|
|
JNE matchlen_match2_standalone
|
|
LEAL -4(DX), DX
|
|
LEAL 4(SI), SI
|
|
|
|
matchlen_match2_standalone:
|
|
CMPL DX, $0x01
|
|
JE matchlen_match1_standalone
|
|
JB gen_match_len_end
|
|
MOVW (AX)(SI*1), BX
|
|
CMPW (CX)(SI*1), BX
|
|
JNE matchlen_match1_standalone
|
|
LEAL 2(SI), SI
|
|
SUBL $0x02, DX
|
|
JZ gen_match_len_end
|
|
|
|
matchlen_match1_standalone:
|
|
MOVB (AX)(SI*1), BL
|
|
CMPB (CX)(SI*1), BL
|
|
JNE gen_match_len_end
|
|
LEAL 1(SI), SI
|
|
|
|
gen_match_len_end:
|
|
MOVQ SI, ret+48(FP)
|
|
RET
|
|
|
|
// func cvtLZ4BlockAsm(dst []byte, src []byte) (uncompressed int, dstUsed int)
|
|
// Requires: SSE2
|
|
TEXT ·cvtLZ4BlockAsm(SB), NOSPLIT, $0-64
|
|
XORQ SI, SI
|
|
MOVQ dst_base+0(FP), AX
|
|
MOVQ dst_len+8(FP), CX
|
|
MOVQ src_base+24(FP), DX
|
|
MOVQ src_len+32(FP), BX
|
|
LEAQ (DX)(BX*1), BX
|
|
LEAQ -8(AX)(CX*1), CX
|
|
XORQ DI, DI
|
|
|
|
lz4_s2_loop:
|
|
CMPQ DX, BX
|
|
JAE lz4_s2_corrupt
|
|
CMPQ AX, CX
|
|
JAE lz4_s2_dstfull
|
|
MOVBQZX (DX), R8
|
|
MOVQ R8, R9
|
|
MOVQ R8, R10
|
|
SHRQ $0x04, R9
|
|
ANDQ $0x0f, R10
|
|
CMPQ R8, $0xf0
|
|
JB lz4_s2_ll_end
|
|
|
|
lz4_s2_ll_loop:
|
|
INCQ DX
|
|
CMPQ DX, BX
|
|
JAE lz4_s2_corrupt
|
|
MOVBQZX (DX), R8
|
|
ADDQ R8, R9
|
|
CMPQ R8, $0xff
|
|
JEQ lz4_s2_ll_loop
|
|
|
|
lz4_s2_ll_end:
|
|
LEAQ (DX)(R9*1), R8
|
|
ADDQ $0x04, R10
|
|
CMPQ R8, BX
|
|
JAE lz4_s2_corrupt
|
|
INCQ DX
|
|
INCQ R8
|
|
TESTQ R9, R9
|
|
JZ lz4_s2_lits_done
|
|
LEAQ (AX)(R9*1), R11
|
|
CMPQ R11, CX
|
|
JAE lz4_s2_dstfull
|
|
ADDQ R9, SI
|
|
LEAL -1(R9), R11
|
|
CMPL R11, $0x3c
|
|
JB one_byte_lz4_s2
|
|
CMPL R11, $0x00000100
|
|
JB two_bytes_lz4_s2
|
|
CMPL R11, $0x00010000
|
|
JB three_bytes_lz4_s2
|
|
CMPL R11, $0x01000000
|
|
JB four_bytes_lz4_s2
|
|
MOVB $0xfc, (AX)
|
|
MOVL R11, 1(AX)
|
|
ADDQ $0x05, AX
|
|
JMP memmove_long_lz4_s2
|
|
|
|
four_bytes_lz4_s2:
|
|
MOVL R11, R12
|
|
SHRL $0x10, R12
|
|
MOVB $0xf8, (AX)
|
|
MOVW R11, 1(AX)
|
|
MOVB R12, 3(AX)
|
|
ADDQ $0x04, AX
|
|
JMP memmove_long_lz4_s2
|
|
|
|
three_bytes_lz4_s2:
|
|
MOVB $0xf4, (AX)
|
|
MOVW R11, 1(AX)
|
|
ADDQ $0x03, AX
|
|
JMP memmove_long_lz4_s2
|
|
|
|
two_bytes_lz4_s2:
|
|
MOVB $0xf0, (AX)
|
|
MOVB R11, 1(AX)
|
|
ADDQ $0x02, AX
|
|
CMPL R11, $0x40
|
|
JB memmove_lz4_s2
|
|
JMP memmove_long_lz4_s2
|
|
|
|
one_byte_lz4_s2:
|
|
SHLB $0x02, R11
|
|
MOVB R11, (AX)
|
|
ADDQ $0x01, AX
|
|
|
|
memmove_lz4_s2:
|
|
LEAQ (AX)(R9*1), R11
|
|
|
|
// genMemMoveShort
|
|
CMPQ R9, $0x08
|
|
JBE emit_lit_memmove_lz4_s2_memmove_move_8
|
|
CMPQ R9, $0x10
|
|
JBE emit_lit_memmove_lz4_s2_memmove_move_8through16
|
|
CMPQ R9, $0x20
|
|
JBE emit_lit_memmove_lz4_s2_memmove_move_17through32
|
|
JMP emit_lit_memmove_lz4_s2_memmove_move_33through64
|
|
|
|
emit_lit_memmove_lz4_s2_memmove_move_8:
|
|
MOVQ (DX), R12
|
|
MOVQ R12, (AX)
|
|
JMP memmove_end_copy_lz4_s2
|
|
|
|
emit_lit_memmove_lz4_s2_memmove_move_8through16:
|
|
MOVQ (DX), R12
|
|
MOVQ -8(DX)(R9*1), DX
|
|
MOVQ R12, (AX)
|
|
MOVQ DX, -8(AX)(R9*1)
|
|
JMP memmove_end_copy_lz4_s2
|
|
|
|
emit_lit_memmove_lz4_s2_memmove_move_17through32:
|
|
MOVOU (DX), X0
|
|
MOVOU -16(DX)(R9*1), X1
|
|
MOVOU X0, (AX)
|
|
MOVOU X1, -16(AX)(R9*1)
|
|
JMP memmove_end_copy_lz4_s2
|
|
|
|
emit_lit_memmove_lz4_s2_memmove_move_33through64:
|
|
MOVOU (DX), X0
|
|
MOVOU 16(DX), X1
|
|
MOVOU -32(DX)(R9*1), X2
|
|
MOVOU -16(DX)(R9*1), X3
|
|
MOVOU X0, (AX)
|
|
MOVOU X1, 16(AX)
|
|
MOVOU X2, -32(AX)(R9*1)
|
|
MOVOU X3, -16(AX)(R9*1)
|
|
|
|
memmove_end_copy_lz4_s2:
|
|
MOVQ R11, AX
|
|
JMP lz4_s2_lits_emit_done
|
|
|
|
memmove_long_lz4_s2:
|
|
LEAQ (AX)(R9*1), R11
|
|
|
|
// genMemMoveLong
|
|
MOVOU (DX), X0
|
|
MOVOU 16(DX), X1
|
|
MOVOU -32(DX)(R9*1), X2
|
|
MOVOU -16(DX)(R9*1), X3
|
|
MOVQ R9, R13
|
|
SHRQ $0x05, R13
|
|
MOVQ AX, R12
|
|
ANDL $0x0000001f, R12
|
|
MOVQ $0x00000040, R14
|
|
SUBQ R12, R14
|
|
DECQ R13
|
|
JA emit_lit_memmove_long_lz4_s2large_forward_sse_loop_32
|
|
LEAQ -32(DX)(R14*1), R12
|
|
LEAQ -32(AX)(R14*1), R15
|
|
|
|
emit_lit_memmove_long_lz4_s2large_big_loop_back:
|
|
MOVOU (R12), X4
|
|
MOVOU 16(R12), X5
|
|
MOVOA X4, (R15)
|
|
MOVOA X5, 16(R15)
|
|
ADDQ $0x20, R15
|
|
ADDQ $0x20, R12
|
|
ADDQ $0x20, R14
|
|
DECQ R13
|
|
JNA emit_lit_memmove_long_lz4_s2large_big_loop_back
|
|
|
|
emit_lit_memmove_long_lz4_s2large_forward_sse_loop_32:
|
|
MOVOU -32(DX)(R14*1), X4
|
|
MOVOU -16(DX)(R14*1), X5
|
|
MOVOA X4, -32(AX)(R14*1)
|
|
MOVOA X5, -16(AX)(R14*1)
|
|
ADDQ $0x20, R14
|
|
CMPQ R9, R14
|
|
JAE emit_lit_memmove_long_lz4_s2large_forward_sse_loop_32
|
|
MOVOU X0, (AX)
|
|
MOVOU X1, 16(AX)
|
|
MOVOU X2, -32(AX)(R9*1)
|
|
MOVOU X3, -16(AX)(R9*1)
|
|
MOVQ R11, AX
|
|
|
|
lz4_s2_lits_emit_done:
|
|
MOVQ R8, DX
|
|
|
|
lz4_s2_lits_done:
|
|
CMPQ DX, BX
|
|
JNE lz4_s2_match
|
|
CMPQ R10, $0x04
|
|
JEQ lz4_s2_done
|
|
JMP lz4_s2_corrupt
|
|
|
|
lz4_s2_match:
|
|
LEAQ 2(DX), R8
|
|
CMPQ R8, BX
|
|
JAE lz4_s2_corrupt
|
|
MOVWQZX (DX), R9
|
|
MOVQ R8, DX
|
|
TESTQ R9, R9
|
|
JZ lz4_s2_corrupt
|
|
CMPQ R9, SI
|
|
JA lz4_s2_corrupt
|
|
CMPQ R10, $0x13
|
|
JNE lz4_s2_ml_done
|
|
|
|
lz4_s2_ml_loop:
|
|
MOVBQZX (DX), R8
|
|
INCQ DX
|
|
ADDQ R8, R10
|
|
CMPQ DX, BX
|
|
JAE lz4_s2_corrupt
|
|
CMPQ R8, $0xff
|
|
JEQ lz4_s2_ml_loop
|
|
|
|
lz4_s2_ml_done:
|
|
ADDQ R10, SI
|
|
CMPQ R9, DI
|
|
JNE lz4_s2_docopy
|
|
|
|
// emitRepeat
|
|
emit_repeat_again_lz4_s2:
|
|
MOVL R10, R8
|
|
LEAL -4(R10), R10
|
|
CMPL R8, $0x08
|
|
JBE repeat_two_lz4_s2
|
|
CMPL R8, $0x0c
|
|
JAE cant_repeat_two_offset_lz4_s2
|
|
CMPL R9, $0x00000800
|
|
JB repeat_two_offset_lz4_s2
|
|
|
|
cant_repeat_two_offset_lz4_s2:
|
|
CMPL R10, $0x00000104
|
|
JB repeat_three_lz4_s2
|
|
CMPL R10, $0x00010100
|
|
JB repeat_four_lz4_s2
|
|
CMPL R10, $0x0100ffff
|
|
JB repeat_five_lz4_s2
|
|
LEAL -16842747(R10), R10
|
|
MOVL $0xfffb001d, (AX)
|
|
MOVB $0xff, 4(AX)
|
|
ADDQ $0x05, AX
|
|
JMP emit_repeat_again_lz4_s2
|
|
|
|
repeat_five_lz4_s2:
|
|
LEAL -65536(R10), R10
|
|
MOVL R10, R9
|
|
MOVW $0x001d, (AX)
|
|
MOVW R10, 2(AX)
|
|
SARL $0x10, R9
|
|
MOVB R9, 4(AX)
|
|
ADDQ $0x05, AX
|
|
JMP lz4_s2_loop
|
|
|
|
repeat_four_lz4_s2:
|
|
LEAL -256(R10), R10
|
|
MOVW $0x0019, (AX)
|
|
MOVW R10, 2(AX)
|
|
ADDQ $0x04, AX
|
|
JMP lz4_s2_loop
|
|
|
|
repeat_three_lz4_s2:
|
|
LEAL -4(R10), R10
|
|
MOVW $0x0015, (AX)
|
|
MOVB R10, 2(AX)
|
|
ADDQ $0x03, AX
|
|
JMP lz4_s2_loop
|
|
|
|
repeat_two_lz4_s2:
|
|
SHLL $0x02, R10
|
|
ORL $0x01, R10
|
|
MOVW R10, (AX)
|
|
ADDQ $0x02, AX
|
|
JMP lz4_s2_loop
|
|
|
|
repeat_two_offset_lz4_s2:
|
|
XORQ R8, R8
|
|
LEAL 1(R8)(R10*4), R10
|
|
MOVB R9, 1(AX)
|
|
SARL $0x08, R9
|
|
SHLL $0x05, R9
|
|
ORL R9, R10
|
|
MOVB R10, (AX)
|
|
ADDQ $0x02, AX
|
|
JMP lz4_s2_loop
|
|
|
|
lz4_s2_docopy:
|
|
MOVQ R9, DI
|
|
|
|
// emitCopy
|
|
CMPL R10, $0x40
|
|
JBE two_byte_offset_short_lz4_s2
|
|
CMPL R9, $0x00000800
|
|
JAE long_offset_short_lz4_s2
|
|
MOVL $0x00000001, R8
|
|
LEAL 16(R8), R8
|
|
MOVB R9, 1(AX)
|
|
MOVL R9, R11
|
|
SHRL $0x08, R11
|
|
SHLL $0x05, R11
|
|
ORL R11, R8
|
|
MOVB R8, (AX)
|
|
ADDQ $0x02, AX
|
|
SUBL $0x08, R10
|
|
|
|
// emitRepeat
|
|
LEAL -4(R10), R10
|
|
JMP cant_repeat_two_offset_lz4_s2_emit_copy_short_2b
|
|
|
|
emit_repeat_again_lz4_s2_emit_copy_short_2b:
|
|
MOVL R10, R8
|
|
LEAL -4(R10), R10
|
|
CMPL R8, $0x08
|
|
JBE repeat_two_lz4_s2_emit_copy_short_2b
|
|
CMPL R8, $0x0c
|
|
JAE cant_repeat_two_offset_lz4_s2_emit_copy_short_2b
|
|
CMPL R9, $0x00000800
|
|
JB repeat_two_offset_lz4_s2_emit_copy_short_2b
|
|
|
|
cant_repeat_two_offset_lz4_s2_emit_copy_short_2b:
|
|
CMPL R10, $0x00000104
|
|
JB repeat_three_lz4_s2_emit_copy_short_2b
|
|
CMPL R10, $0x00010100
|
|
JB repeat_four_lz4_s2_emit_copy_short_2b
|
|
CMPL R10, $0x0100ffff
|
|
JB repeat_five_lz4_s2_emit_copy_short_2b
|
|
LEAL -16842747(R10), R10
|
|
MOVL $0xfffb001d, (AX)
|
|
MOVB $0xff, 4(AX)
|
|
ADDQ $0x05, AX
|
|
JMP emit_repeat_again_lz4_s2_emit_copy_short_2b
|
|
|
|
repeat_five_lz4_s2_emit_copy_short_2b:
|
|
LEAL -65536(R10), R10
|
|
MOVL R10, R9
|
|
MOVW $0x001d, (AX)
|
|
MOVW R10, 2(AX)
|
|
SARL $0x10, R9
|
|
MOVB R9, 4(AX)
|
|
ADDQ $0x05, AX
|
|
JMP lz4_s2_loop
|
|
|
|
repeat_four_lz4_s2_emit_copy_short_2b:
|
|
LEAL -256(R10), R10
|
|
MOVW $0x0019, (AX)
|
|
MOVW R10, 2(AX)
|
|
ADDQ $0x04, AX
|
|
JMP lz4_s2_loop
|
|
|
|
repeat_three_lz4_s2_emit_copy_short_2b:
|
|
LEAL -4(R10), R10
|
|
MOVW $0x0015, (AX)
|
|
MOVB R10, 2(AX)
|
|
ADDQ $0x03, AX
|
|
JMP lz4_s2_loop
|
|
|
|
repeat_two_lz4_s2_emit_copy_short_2b:
|
|
SHLL $0x02, R10
|
|
ORL $0x01, R10
|
|
MOVW R10, (AX)
|
|
ADDQ $0x02, AX
|
|
JMP lz4_s2_loop
|
|
|
|
repeat_two_offset_lz4_s2_emit_copy_short_2b:
|
|
XORQ R8, R8
|
|
LEAL 1(R8)(R10*4), R10
|
|
MOVB R9, 1(AX)
|
|
SARL $0x08, R9
|
|
SHLL $0x05, R9
|
|
ORL R9, R10
|
|
MOVB R10, (AX)
|
|
ADDQ $0x02, AX
|
|
JMP lz4_s2_loop
|
|
|
|
long_offset_short_lz4_s2:
|
|
MOVB $0xee, (AX)
|
|
MOVW R9, 1(AX)
|
|
LEAL -60(R10), R10
|
|
ADDQ $0x03, AX
|
|
|
|
// emitRepeat
|
|
emit_repeat_again_lz4_s2_emit_copy_short:
|
|
MOVL R10, R8
|
|
LEAL -4(R10), R10
|
|
CMPL R8, $0x08
|
|
JBE repeat_two_lz4_s2_emit_copy_short
|
|
CMPL R8, $0x0c
|
|
JAE cant_repeat_two_offset_lz4_s2_emit_copy_short
|
|
CMPL R9, $0x00000800
|
|
JB repeat_two_offset_lz4_s2_emit_copy_short
|
|
|
|
cant_repeat_two_offset_lz4_s2_emit_copy_short:
|
|
CMPL R10, $0x00000104
|
|
JB repeat_three_lz4_s2_emit_copy_short
|
|
CMPL R10, $0x00010100
|
|
JB repeat_four_lz4_s2_emit_copy_short
|
|
CMPL R10, $0x0100ffff
|
|
JB repeat_five_lz4_s2_emit_copy_short
|
|
LEAL -16842747(R10), R10
|
|
MOVL $0xfffb001d, (AX)
|
|
MOVB $0xff, 4(AX)
|
|
ADDQ $0x05, AX
|
|
JMP emit_repeat_again_lz4_s2_emit_copy_short
|
|
|
|
repeat_five_lz4_s2_emit_copy_short:
|
|
LEAL -65536(R10), R10
|
|
MOVL R10, R9
|
|
MOVW $0x001d, (AX)
|
|
MOVW R10, 2(AX)
|
|
SARL $0x10, R9
|
|
MOVB R9, 4(AX)
|
|
ADDQ $0x05, AX
|
|
JMP lz4_s2_loop
|
|
|
|
repeat_four_lz4_s2_emit_copy_short:
|
|
LEAL -256(R10), R10
|
|
MOVW $0x0019, (AX)
|
|
MOVW R10, 2(AX)
|
|
ADDQ $0x04, AX
|
|
JMP lz4_s2_loop
|
|
|
|
repeat_three_lz4_s2_emit_copy_short:
|
|
LEAL -4(R10), R10
|
|
MOVW $0x0015, (AX)
|
|
MOVB R10, 2(AX)
|
|
ADDQ $0x03, AX
|
|
JMP lz4_s2_loop
|
|
|
|
repeat_two_lz4_s2_emit_copy_short:
|
|
SHLL $0x02, R10
|
|
ORL $0x01, R10
|
|
MOVW R10, (AX)
|
|
ADDQ $0x02, AX
|
|
JMP lz4_s2_loop
|
|
|
|
repeat_two_offset_lz4_s2_emit_copy_short:
|
|
XORQ R8, R8
|
|
LEAL 1(R8)(R10*4), R10
|
|
MOVB R9, 1(AX)
|
|
SARL $0x08, R9
|
|
SHLL $0x05, R9
|
|
ORL R9, R10
|
|
MOVB R10, (AX)
|
|
ADDQ $0x02, AX
|
|
JMP lz4_s2_loop
|
|
|
|
two_byte_offset_short_lz4_s2:
|
|
MOVL R10, R8
|
|
SHLL $0x02, R8
|
|
CMPL R10, $0x0c
|
|
JAE emit_copy_three_lz4_s2
|
|
CMPL R9, $0x00000800
|
|
JAE emit_copy_three_lz4_s2
|
|
LEAL -15(R8), R8
|
|
MOVB R9, 1(AX)
|
|
SHRL $0x08, R9
|
|
SHLL $0x05, R9
|
|
ORL R9, R8
|
|
MOVB R8, (AX)
|
|
ADDQ $0x02, AX
|
|
JMP lz4_s2_loop
|
|
|
|
emit_copy_three_lz4_s2:
|
|
LEAL -2(R8), R8
|
|
MOVB R8, (AX)
|
|
MOVW R9, 1(AX)
|
|
ADDQ $0x03, AX
|
|
JMP lz4_s2_loop
|
|
|
|
lz4_s2_done:
|
|
MOVQ dst_base+0(FP), CX
|
|
SUBQ CX, AX
|
|
MOVQ SI, uncompressed+48(FP)
|
|
MOVQ AX, dstUsed+56(FP)
|
|
RET
|
|
|
|
lz4_s2_corrupt:
|
|
XORQ AX, AX
|
|
LEAQ -1(AX), SI
|
|
MOVQ SI, uncompressed+48(FP)
|
|
RET
|
|
|
|
lz4_s2_dstfull:
|
|
XORQ AX, AX
|
|
LEAQ -2(AX), SI
|
|
MOVQ SI, uncompressed+48(FP)
|
|
RET
|
|
|
|
// func cvtLZ4sBlockAsm(dst []byte, src []byte) (uncompressed int, dstUsed int)
|
|
// Requires: SSE2
|
|
TEXT ·cvtLZ4sBlockAsm(SB), NOSPLIT, $0-64
|
|
XORQ SI, SI
|
|
MOVQ dst_base+0(FP), AX
|
|
MOVQ dst_len+8(FP), CX
|
|
MOVQ src_base+24(FP), DX
|
|
MOVQ src_len+32(FP), BX
|
|
LEAQ (DX)(BX*1), BX
|
|
LEAQ -8(AX)(CX*1), CX
|
|
XORQ DI, DI
|
|
|
|
lz4s_s2_loop:
|
|
CMPQ DX, BX
|
|
JAE lz4s_s2_corrupt
|
|
CMPQ AX, CX
|
|
JAE lz4s_s2_dstfull
|
|
MOVBQZX (DX), R8
|
|
MOVQ R8, R9
|
|
MOVQ R8, R10
|
|
SHRQ $0x04, R9
|
|
ANDQ $0x0f, R10
|
|
CMPQ R8, $0xf0
|
|
JB lz4s_s2_ll_end
|
|
|
|
lz4s_s2_ll_loop:
|
|
INCQ DX
|
|
CMPQ DX, BX
|
|
JAE lz4s_s2_corrupt
|
|
MOVBQZX (DX), R8
|
|
ADDQ R8, R9
|
|
CMPQ R8, $0xff
|
|
JEQ lz4s_s2_ll_loop
|
|
|
|
lz4s_s2_ll_end:
|
|
LEAQ (DX)(R9*1), R8
|
|
ADDQ $0x03, R10
|
|
CMPQ R8, BX
|
|
JAE lz4s_s2_corrupt
|
|
INCQ DX
|
|
INCQ R8
|
|
TESTQ R9, R9
|
|
JZ lz4s_s2_lits_done
|
|
LEAQ (AX)(R9*1), R11
|
|
CMPQ R11, CX
|
|
JAE lz4s_s2_dstfull
|
|
ADDQ R9, SI
|
|
LEAL -1(R9), R11
|
|
CMPL R11, $0x3c
|
|
JB one_byte_lz4s_s2
|
|
CMPL R11, $0x00000100
|
|
JB two_bytes_lz4s_s2
|
|
CMPL R11, $0x00010000
|
|
JB three_bytes_lz4s_s2
|
|
CMPL R11, $0x01000000
|
|
JB four_bytes_lz4s_s2
|
|
MOVB $0xfc, (AX)
|
|
MOVL R11, 1(AX)
|
|
ADDQ $0x05, AX
|
|
JMP memmove_long_lz4s_s2
|
|
|
|
four_bytes_lz4s_s2:
|
|
MOVL R11, R12
|
|
SHRL $0x10, R12
|
|
MOVB $0xf8, (AX)
|
|
MOVW R11, 1(AX)
|
|
MOVB R12, 3(AX)
|
|
ADDQ $0x04, AX
|
|
JMP memmove_long_lz4s_s2
|
|
|
|
three_bytes_lz4s_s2:
|
|
MOVB $0xf4, (AX)
|
|
MOVW R11, 1(AX)
|
|
ADDQ $0x03, AX
|
|
JMP memmove_long_lz4s_s2
|
|
|
|
two_bytes_lz4s_s2:
|
|
MOVB $0xf0, (AX)
|
|
MOVB R11, 1(AX)
|
|
ADDQ $0x02, AX
|
|
CMPL R11, $0x40
|
|
JB memmove_lz4s_s2
|
|
JMP memmove_long_lz4s_s2
|
|
|
|
one_byte_lz4s_s2:
|
|
SHLB $0x02, R11
|
|
MOVB R11, (AX)
|
|
ADDQ $0x01, AX
|
|
|
|
memmove_lz4s_s2:
|
|
LEAQ (AX)(R9*1), R11
|
|
|
|
// genMemMoveShort
|
|
CMPQ R9, $0x08
|
|
JBE emit_lit_memmove_lz4s_s2_memmove_move_8
|
|
CMPQ R9, $0x10
|
|
JBE emit_lit_memmove_lz4s_s2_memmove_move_8through16
|
|
CMPQ R9, $0x20
|
|
JBE emit_lit_memmove_lz4s_s2_memmove_move_17through32
|
|
JMP emit_lit_memmove_lz4s_s2_memmove_move_33through64
|
|
|
|
emit_lit_memmove_lz4s_s2_memmove_move_8:
|
|
MOVQ (DX), R12
|
|
MOVQ R12, (AX)
|
|
JMP memmove_end_copy_lz4s_s2
|
|
|
|
emit_lit_memmove_lz4s_s2_memmove_move_8through16:
|
|
MOVQ (DX), R12
|
|
MOVQ -8(DX)(R9*1), DX
|
|
MOVQ R12, (AX)
|
|
MOVQ DX, -8(AX)(R9*1)
|
|
JMP memmove_end_copy_lz4s_s2
|
|
|
|
emit_lit_memmove_lz4s_s2_memmove_move_17through32:
|
|
MOVOU (DX), X0
|
|
MOVOU -16(DX)(R9*1), X1
|
|
MOVOU X0, (AX)
|
|
MOVOU X1, -16(AX)(R9*1)
|
|
JMP memmove_end_copy_lz4s_s2
|
|
|
|
emit_lit_memmove_lz4s_s2_memmove_move_33through64:
|
|
MOVOU (DX), X0
|
|
MOVOU 16(DX), X1
|
|
MOVOU -32(DX)(R9*1), X2
|
|
MOVOU -16(DX)(R9*1), X3
|
|
MOVOU X0, (AX)
|
|
MOVOU X1, 16(AX)
|
|
MOVOU X2, -32(AX)(R9*1)
|
|
MOVOU X3, -16(AX)(R9*1)
|
|
|
|
memmove_end_copy_lz4s_s2:
|
|
MOVQ R11, AX
|
|
JMP lz4s_s2_lits_emit_done
|
|
|
|
memmove_long_lz4s_s2:
|
|
LEAQ (AX)(R9*1), R11
|
|
|
|
// genMemMoveLong
|
|
MOVOU (DX), X0
|
|
MOVOU 16(DX), X1
|
|
MOVOU -32(DX)(R9*1), X2
|
|
MOVOU -16(DX)(R9*1), X3
|
|
MOVQ R9, R13
|
|
SHRQ $0x05, R13
|
|
MOVQ AX, R12
|
|
ANDL $0x0000001f, R12
|
|
MOVQ $0x00000040, R14
|
|
SUBQ R12, R14
|
|
DECQ R13
|
|
JA emit_lit_memmove_long_lz4s_s2large_forward_sse_loop_32
|
|
LEAQ -32(DX)(R14*1), R12
|
|
LEAQ -32(AX)(R14*1), R15
|
|
|
|
emit_lit_memmove_long_lz4s_s2large_big_loop_back:
|
|
MOVOU (R12), X4
|
|
MOVOU 16(R12), X5
|
|
MOVOA X4, (R15)
|
|
MOVOA X5, 16(R15)
|
|
ADDQ $0x20, R15
|
|
ADDQ $0x20, R12
|
|
ADDQ $0x20, R14
|
|
DECQ R13
|
|
JNA emit_lit_memmove_long_lz4s_s2large_big_loop_back
|
|
|
|
emit_lit_memmove_long_lz4s_s2large_forward_sse_loop_32:
|
|
MOVOU -32(DX)(R14*1), X4
|
|
MOVOU -16(DX)(R14*1), X5
|
|
MOVOA X4, -32(AX)(R14*1)
|
|
MOVOA X5, -16(AX)(R14*1)
|
|
ADDQ $0x20, R14
|
|
CMPQ R9, R14
|
|
JAE emit_lit_memmove_long_lz4s_s2large_forward_sse_loop_32
|
|
MOVOU X0, (AX)
|
|
MOVOU X1, 16(AX)
|
|
MOVOU X2, -32(AX)(R9*1)
|
|
MOVOU X3, -16(AX)(R9*1)
|
|
MOVQ R11, AX
|
|
|
|
lz4s_s2_lits_emit_done:
|
|
MOVQ R8, DX
|
|
|
|
lz4s_s2_lits_done:
|
|
CMPQ DX, BX
|
|
JNE lz4s_s2_match
|
|
CMPQ R10, $0x03
|
|
JEQ lz4s_s2_done
|
|
JMP lz4s_s2_corrupt
|
|
|
|
lz4s_s2_match:
|
|
CMPQ R10, $0x03
|
|
JEQ lz4s_s2_loop
|
|
LEAQ 2(DX), R8
|
|
CMPQ R8, BX
|
|
JAE lz4s_s2_corrupt
|
|
MOVWQZX (DX), R9
|
|
MOVQ R8, DX
|
|
TESTQ R9, R9
|
|
JZ lz4s_s2_corrupt
|
|
CMPQ R9, SI
|
|
JA lz4s_s2_corrupt
|
|
CMPQ R10, $0x12
|
|
JNE lz4s_s2_ml_done
|
|
|
|
lz4s_s2_ml_loop:
|
|
MOVBQZX (DX), R8
|
|
INCQ DX
|
|
ADDQ R8, R10
|
|
CMPQ DX, BX
|
|
JAE lz4s_s2_corrupt
|
|
CMPQ R8, $0xff
|
|
JEQ lz4s_s2_ml_loop
|
|
|
|
lz4s_s2_ml_done:
|
|
ADDQ R10, SI
|
|
CMPQ R9, DI
|
|
JNE lz4s_s2_docopy
|
|
|
|
// emitRepeat
|
|
emit_repeat_again_lz4_s2:
|
|
MOVL R10, R8
|
|
LEAL -4(R10), R10
|
|
CMPL R8, $0x08
|
|
JBE repeat_two_lz4_s2
|
|
CMPL R8, $0x0c
|
|
JAE cant_repeat_two_offset_lz4_s2
|
|
CMPL R9, $0x00000800
|
|
JB repeat_two_offset_lz4_s2
|
|
|
|
cant_repeat_two_offset_lz4_s2:
|
|
CMPL R10, $0x00000104
|
|
JB repeat_three_lz4_s2
|
|
CMPL R10, $0x00010100
|
|
JB repeat_four_lz4_s2
|
|
CMPL R10, $0x0100ffff
|
|
JB repeat_five_lz4_s2
|
|
LEAL -16842747(R10), R10
|
|
MOVL $0xfffb001d, (AX)
|
|
MOVB $0xff, 4(AX)
|
|
ADDQ $0x05, AX
|
|
JMP emit_repeat_again_lz4_s2
|
|
|
|
repeat_five_lz4_s2:
|
|
LEAL -65536(R10), R10
|
|
MOVL R10, R9
|
|
MOVW $0x001d, (AX)
|
|
MOVW R10, 2(AX)
|
|
SARL $0x10, R9
|
|
MOVB R9, 4(AX)
|
|
ADDQ $0x05, AX
|
|
JMP lz4s_s2_loop
|
|
|
|
repeat_four_lz4_s2:
|
|
LEAL -256(R10), R10
|
|
MOVW $0x0019, (AX)
|
|
MOVW R10, 2(AX)
|
|
ADDQ $0x04, AX
|
|
JMP lz4s_s2_loop
|
|
|
|
repeat_three_lz4_s2:
|
|
LEAL -4(R10), R10
|
|
MOVW $0x0015, (AX)
|
|
MOVB R10, 2(AX)
|
|
ADDQ $0x03, AX
|
|
JMP lz4s_s2_loop
|
|
|
|
repeat_two_lz4_s2:
|
|
SHLL $0x02, R10
|
|
ORL $0x01, R10
|
|
MOVW R10, (AX)
|
|
ADDQ $0x02, AX
|
|
JMP lz4s_s2_loop
|
|
|
|
repeat_two_offset_lz4_s2:
|
|
XORQ R8, R8
|
|
LEAL 1(R8)(R10*4), R10
|
|
MOVB R9, 1(AX)
|
|
SARL $0x08, R9
|
|
SHLL $0x05, R9
|
|
ORL R9, R10
|
|
MOVB R10, (AX)
|
|
ADDQ $0x02, AX
|
|
JMP lz4s_s2_loop
|
|
|
|
lz4s_s2_docopy:
|
|
MOVQ R9, DI
|
|
|
|
// emitCopy
|
|
CMPL R10, $0x40
|
|
JBE two_byte_offset_short_lz4_s2
|
|
CMPL R9, $0x00000800
|
|
JAE long_offset_short_lz4_s2
|
|
MOVL $0x00000001, R8
|
|
LEAL 16(R8), R8
|
|
MOVB R9, 1(AX)
|
|
MOVL R9, R11
|
|
SHRL $0x08, R11
|
|
SHLL $0x05, R11
|
|
ORL R11, R8
|
|
MOVB R8, (AX)
|
|
ADDQ $0x02, AX
|
|
SUBL $0x08, R10
|
|
|
|
// emitRepeat
|
|
LEAL -4(R10), R10
|
|
JMP cant_repeat_two_offset_lz4_s2_emit_copy_short_2b
|
|
|
|
emit_repeat_again_lz4_s2_emit_copy_short_2b:
|
|
MOVL R10, R8
|
|
LEAL -4(R10), R10
|
|
CMPL R8, $0x08
|
|
JBE repeat_two_lz4_s2_emit_copy_short_2b
|
|
CMPL R8, $0x0c
|
|
JAE cant_repeat_two_offset_lz4_s2_emit_copy_short_2b
|
|
CMPL R9, $0x00000800
|
|
JB repeat_two_offset_lz4_s2_emit_copy_short_2b
|
|
|
|
cant_repeat_two_offset_lz4_s2_emit_copy_short_2b:
|
|
CMPL R10, $0x00000104
|
|
JB repeat_three_lz4_s2_emit_copy_short_2b
|
|
CMPL R10, $0x00010100
|
|
JB repeat_four_lz4_s2_emit_copy_short_2b
|
|
CMPL R10, $0x0100ffff
|
|
JB repeat_five_lz4_s2_emit_copy_short_2b
|
|
LEAL -16842747(R10), R10
|
|
MOVL $0xfffb001d, (AX)
|
|
MOVB $0xff, 4(AX)
|
|
ADDQ $0x05, AX
|
|
JMP emit_repeat_again_lz4_s2_emit_copy_short_2b
|
|
|
|
repeat_five_lz4_s2_emit_copy_short_2b:
|
|
LEAL -65536(R10), R10
|
|
MOVL R10, R9
|
|
MOVW $0x001d, (AX)
|
|
MOVW R10, 2(AX)
|
|
SARL $0x10, R9
|
|
MOVB R9, 4(AX)
|
|
ADDQ $0x05, AX
|
|
JMP lz4s_s2_loop
|
|
|
|
repeat_four_lz4_s2_emit_copy_short_2b:
|
|
LEAL -256(R10), R10
|
|
MOVW $0x0019, (AX)
|
|
MOVW R10, 2(AX)
|
|
ADDQ $0x04, AX
|
|
JMP lz4s_s2_loop
|
|
|
|
repeat_three_lz4_s2_emit_copy_short_2b:
|
|
LEAL -4(R10), R10
|
|
MOVW $0x0015, (AX)
|
|
MOVB R10, 2(AX)
|
|
ADDQ $0x03, AX
|
|
JMP lz4s_s2_loop
|
|
|
|
repeat_two_lz4_s2_emit_copy_short_2b:
|
|
SHLL $0x02, R10
|
|
ORL $0x01, R10
|
|
MOVW R10, (AX)
|
|
ADDQ $0x02, AX
|
|
JMP lz4s_s2_loop
|
|
|
|
repeat_two_offset_lz4_s2_emit_copy_short_2b:
|
|
XORQ R8, R8
|
|
LEAL 1(R8)(R10*4), R10
|
|
MOVB R9, 1(AX)
|
|
SARL $0x08, R9
|
|
SHLL $0x05, R9
|
|
ORL R9, R10
|
|
MOVB R10, (AX)
|
|
ADDQ $0x02, AX
|
|
JMP lz4s_s2_loop
|
|
|
|
long_offset_short_lz4_s2:
|
|
MOVB $0xee, (AX)
|
|
MOVW R9, 1(AX)
|
|
LEAL -60(R10), R10
|
|
ADDQ $0x03, AX
|
|
|
|
// emitRepeat
|
|
emit_repeat_again_lz4_s2_emit_copy_short:
|
|
MOVL R10, R8
|
|
LEAL -4(R10), R10
|
|
CMPL R8, $0x08
|
|
JBE repeat_two_lz4_s2_emit_copy_short
|
|
CMPL R8, $0x0c
|
|
JAE cant_repeat_two_offset_lz4_s2_emit_copy_short
|
|
CMPL R9, $0x00000800
|
|
JB repeat_two_offset_lz4_s2_emit_copy_short
|
|
|
|
cant_repeat_two_offset_lz4_s2_emit_copy_short:
|
|
CMPL R10, $0x00000104
|
|
JB repeat_three_lz4_s2_emit_copy_short
|
|
CMPL R10, $0x00010100
|
|
JB repeat_four_lz4_s2_emit_copy_short
|
|
CMPL R10, $0x0100ffff
|
|
JB repeat_five_lz4_s2_emit_copy_short
|
|
LEAL -16842747(R10), R10
|
|
MOVL $0xfffb001d, (AX)
|
|
MOVB $0xff, 4(AX)
|
|
ADDQ $0x05, AX
|
|
JMP emit_repeat_again_lz4_s2_emit_copy_short
|
|
|
|
repeat_five_lz4_s2_emit_copy_short:
|
|
LEAL -65536(R10), R10
|
|
MOVL R10, R9
|
|
MOVW $0x001d, (AX)
|
|
MOVW R10, 2(AX)
|
|
SARL $0x10, R9
|
|
MOVB R9, 4(AX)
|
|
ADDQ $0x05, AX
|
|
JMP lz4s_s2_loop
|
|
|
|
repeat_four_lz4_s2_emit_copy_short:
|
|
LEAL -256(R10), R10
|
|
MOVW $0x0019, (AX)
|
|
MOVW R10, 2(AX)
|
|
ADDQ $0x04, AX
|
|
JMP lz4s_s2_loop
|
|
|
|
repeat_three_lz4_s2_emit_copy_short:
|
|
LEAL -4(R10), R10
|
|
MOVW $0x0015, (AX)
|
|
MOVB R10, 2(AX)
|
|
ADDQ $0x03, AX
|
|
JMP lz4s_s2_loop
|
|
|
|
repeat_two_lz4_s2_emit_copy_short:
|
|
SHLL $0x02, R10
|
|
ORL $0x01, R10
|
|
MOVW R10, (AX)
|
|
ADDQ $0x02, AX
|
|
JMP lz4s_s2_loop
|
|
|
|
repeat_two_offset_lz4_s2_emit_copy_short:
|
|
XORQ R8, R8
|
|
LEAL 1(R8)(R10*4), R10
|
|
MOVB R9, 1(AX)
|
|
SARL $0x08, R9
|
|
SHLL $0x05, R9
|
|
ORL R9, R10
|
|
MOVB R10, (AX)
|
|
ADDQ $0x02, AX
|
|
JMP lz4s_s2_loop
|
|
|
|
two_byte_offset_short_lz4_s2:
|
|
MOVL R10, R8
|
|
SHLL $0x02, R8
|
|
CMPL R10, $0x0c
|
|
JAE emit_copy_three_lz4_s2
|
|
CMPL R9, $0x00000800
|
|
JAE emit_copy_three_lz4_s2
|
|
LEAL -15(R8), R8
|
|
MOVB R9, 1(AX)
|
|
SHRL $0x08, R9
|
|
SHLL $0x05, R9
|
|
ORL R9, R8
|
|
MOVB R8, (AX)
|
|
ADDQ $0x02, AX
|
|
JMP lz4s_s2_loop
|
|
|
|
emit_copy_three_lz4_s2:
|
|
LEAL -2(R8), R8
|
|
MOVB R8, (AX)
|
|
MOVW R9, 1(AX)
|
|
ADDQ $0x03, AX
|
|
JMP lz4s_s2_loop
|
|
|
|
lz4s_s2_done:
|
|
MOVQ dst_base+0(FP), CX
|
|
SUBQ CX, AX
|
|
MOVQ SI, uncompressed+48(FP)
|
|
MOVQ AX, dstUsed+56(FP)
|
|
RET
|
|
|
|
lz4s_s2_corrupt:
|
|
XORQ AX, AX
|
|
LEAQ -1(AX), SI
|
|
MOVQ SI, uncompressed+48(FP)
|
|
RET
|
|
|
|
lz4s_s2_dstfull:
|
|
XORQ AX, AX
|
|
LEAQ -2(AX), SI
|
|
MOVQ SI, uncompressed+48(FP)
|
|
RET
|
|
|
|
// func cvtLZ4BlockSnappyAsm(dst []byte, src []byte) (uncompressed int, dstUsed int)
|
|
// Requires: SSE2
|
|
TEXT ·cvtLZ4BlockSnappyAsm(SB), NOSPLIT, $0-64
|
|
XORQ SI, SI
|
|
MOVQ dst_base+0(FP), AX
|
|
MOVQ dst_len+8(FP), CX
|
|
MOVQ src_base+24(FP), DX
|
|
MOVQ src_len+32(FP), BX
|
|
LEAQ (DX)(BX*1), BX
|
|
LEAQ -8(AX)(CX*1), CX
|
|
|
|
lz4_snappy_loop:
|
|
CMPQ DX, BX
|
|
JAE lz4_snappy_corrupt
|
|
CMPQ AX, CX
|
|
JAE lz4_snappy_dstfull
|
|
MOVBQZX (DX), DI
|
|
MOVQ DI, R8
|
|
MOVQ DI, R9
|
|
SHRQ $0x04, R8
|
|
ANDQ $0x0f, R9
|
|
CMPQ DI, $0xf0
|
|
JB lz4_snappy_ll_end
|
|
|
|
lz4_snappy_ll_loop:
|
|
INCQ DX
|
|
CMPQ DX, BX
|
|
JAE lz4_snappy_corrupt
|
|
MOVBQZX (DX), DI
|
|
ADDQ DI, R8
|
|
CMPQ DI, $0xff
|
|
JEQ lz4_snappy_ll_loop
|
|
|
|
lz4_snappy_ll_end:
|
|
LEAQ (DX)(R8*1), DI
|
|
ADDQ $0x04, R9
|
|
CMPQ DI, BX
|
|
JAE lz4_snappy_corrupt
|
|
INCQ DX
|
|
INCQ DI
|
|
TESTQ R8, R8
|
|
JZ lz4_snappy_lits_done
|
|
LEAQ (AX)(R8*1), R10
|
|
CMPQ R10, CX
|
|
JAE lz4_snappy_dstfull
|
|
ADDQ R8, SI
|
|
LEAL -1(R8), R10
|
|
CMPL R10, $0x3c
|
|
JB one_byte_lz4_snappy
|
|
CMPL R10, $0x00000100
|
|
JB two_bytes_lz4_snappy
|
|
CMPL R10, $0x00010000
|
|
JB three_bytes_lz4_snappy
|
|
CMPL R10, $0x01000000
|
|
JB four_bytes_lz4_snappy
|
|
MOVB $0xfc, (AX)
|
|
MOVL R10, 1(AX)
|
|
ADDQ $0x05, AX
|
|
JMP memmove_long_lz4_snappy
|
|
|
|
four_bytes_lz4_snappy:
|
|
MOVL R10, R11
|
|
SHRL $0x10, R11
|
|
MOVB $0xf8, (AX)
|
|
MOVW R10, 1(AX)
|
|
MOVB R11, 3(AX)
|
|
ADDQ $0x04, AX
|
|
JMP memmove_long_lz4_snappy
|
|
|
|
three_bytes_lz4_snappy:
|
|
MOVB $0xf4, (AX)
|
|
MOVW R10, 1(AX)
|
|
ADDQ $0x03, AX
|
|
JMP memmove_long_lz4_snappy
|
|
|
|
two_bytes_lz4_snappy:
|
|
MOVB $0xf0, (AX)
|
|
MOVB R10, 1(AX)
|
|
ADDQ $0x02, AX
|
|
CMPL R10, $0x40
|
|
JB memmove_lz4_snappy
|
|
JMP memmove_long_lz4_snappy
|
|
|
|
one_byte_lz4_snappy:
|
|
SHLB $0x02, R10
|
|
MOVB R10, (AX)
|
|
ADDQ $0x01, AX
|
|
|
|
memmove_lz4_snappy:
|
|
LEAQ (AX)(R8*1), R10
|
|
|
|
// genMemMoveShort
|
|
CMPQ R8, $0x08
|
|
JBE emit_lit_memmove_lz4_snappy_memmove_move_8
|
|
CMPQ R8, $0x10
|
|
JBE emit_lit_memmove_lz4_snappy_memmove_move_8through16
|
|
CMPQ R8, $0x20
|
|
JBE emit_lit_memmove_lz4_snappy_memmove_move_17through32
|
|
JMP emit_lit_memmove_lz4_snappy_memmove_move_33through64
|
|
|
|
emit_lit_memmove_lz4_snappy_memmove_move_8:
|
|
MOVQ (DX), R11
|
|
MOVQ R11, (AX)
|
|
JMP memmove_end_copy_lz4_snappy
|
|
|
|
emit_lit_memmove_lz4_snappy_memmove_move_8through16:
|
|
MOVQ (DX), R11
|
|
MOVQ -8(DX)(R8*1), DX
|
|
MOVQ R11, (AX)
|
|
MOVQ DX, -8(AX)(R8*1)
|
|
JMP memmove_end_copy_lz4_snappy
|
|
|
|
emit_lit_memmove_lz4_snappy_memmove_move_17through32:
|
|
MOVOU (DX), X0
|
|
MOVOU -16(DX)(R8*1), X1
|
|
MOVOU X0, (AX)
|
|
MOVOU X1, -16(AX)(R8*1)
|
|
JMP memmove_end_copy_lz4_snappy
|
|
|
|
emit_lit_memmove_lz4_snappy_memmove_move_33through64:
|
|
MOVOU (DX), X0
|
|
MOVOU 16(DX), X1
|
|
MOVOU -32(DX)(R8*1), X2
|
|
MOVOU -16(DX)(R8*1), X3
|
|
MOVOU X0, (AX)
|
|
MOVOU X1, 16(AX)
|
|
MOVOU X2, -32(AX)(R8*1)
|
|
MOVOU X3, -16(AX)(R8*1)
|
|
|
|
memmove_end_copy_lz4_snappy:
|
|
MOVQ R10, AX
|
|
JMP lz4_snappy_lits_emit_done
|
|
|
|
memmove_long_lz4_snappy:
|
|
LEAQ (AX)(R8*1), R10
|
|
|
|
// genMemMoveLong
|
|
MOVOU (DX), X0
|
|
MOVOU 16(DX), X1
|
|
MOVOU -32(DX)(R8*1), X2
|
|
MOVOU -16(DX)(R8*1), X3
|
|
MOVQ R8, R12
|
|
SHRQ $0x05, R12
|
|
MOVQ AX, R11
|
|
ANDL $0x0000001f, R11
|
|
MOVQ $0x00000040, R13
|
|
SUBQ R11, R13
|
|
DECQ R12
|
|
JA emit_lit_memmove_long_lz4_snappylarge_forward_sse_loop_32
|
|
LEAQ -32(DX)(R13*1), R11
|
|
LEAQ -32(AX)(R13*1), R14
|
|
|
|
emit_lit_memmove_long_lz4_snappylarge_big_loop_back:
|
|
MOVOU (R11), X4
|
|
MOVOU 16(R11), X5
|
|
MOVOA X4, (R14)
|
|
MOVOA X5, 16(R14)
|
|
ADDQ $0x20, R14
|
|
ADDQ $0x20, R11
|
|
ADDQ $0x20, R13
|
|
DECQ R12
|
|
JNA emit_lit_memmove_long_lz4_snappylarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_lz4_snappylarge_forward_sse_loop_32:
|
|
MOVOU -32(DX)(R13*1), X4
|
|
MOVOU -16(DX)(R13*1), X5
|
|
MOVOA X4, -32(AX)(R13*1)
|
|
MOVOA X5, -16(AX)(R13*1)
|
|
ADDQ $0x20, R13
|
|
CMPQ R8, R13
|
|
JAE emit_lit_memmove_long_lz4_snappylarge_forward_sse_loop_32
|
|
MOVOU X0, (AX)
|
|
MOVOU X1, 16(AX)
|
|
MOVOU X2, -32(AX)(R8*1)
|
|
MOVOU X3, -16(AX)(R8*1)
|
|
MOVQ R10, AX
|
|
|
|
lz4_snappy_lits_emit_done:
|
|
MOVQ DI, DX
|
|
|
|
lz4_snappy_lits_done:
|
|
CMPQ DX, BX
|
|
JNE lz4_snappy_match
|
|
CMPQ R9, $0x04
|
|
JEQ lz4_snappy_done
|
|
JMP lz4_snappy_corrupt
|
|
|
|
lz4_snappy_match:
|
|
LEAQ 2(DX), DI
|
|
CMPQ DI, BX
|
|
JAE lz4_snappy_corrupt
|
|
MOVWQZX (DX), R8
|
|
MOVQ DI, DX
|
|
TESTQ R8, R8
|
|
JZ lz4_snappy_corrupt
|
|
CMPQ R8, SI
|
|
JA lz4_snappy_corrupt
|
|
CMPQ R9, $0x13
|
|
JNE lz4_snappy_ml_done
|
|
|
|
lz4_snappy_ml_loop:
|
|
MOVBQZX (DX), DI
|
|
INCQ DX
|
|
ADDQ DI, R9
|
|
CMPQ DX, BX
|
|
JAE lz4_snappy_corrupt
|
|
CMPQ DI, $0xff
|
|
JEQ lz4_snappy_ml_loop
|
|
|
|
lz4_snappy_ml_done:
|
|
ADDQ R9, SI
|
|
|
|
// emitCopy
|
|
two_byte_offset_lz4_s2:
|
|
CMPL R9, $0x40
|
|
JBE two_byte_offset_short_lz4_s2
|
|
MOVB $0xee, (AX)
|
|
MOVW R8, 1(AX)
|
|
LEAL -60(R9), R9
|
|
ADDQ $0x03, AX
|
|
CMPQ AX, CX
|
|
JAE lz4_snappy_loop
|
|
JMP two_byte_offset_lz4_s2
|
|
|
|
two_byte_offset_short_lz4_s2:
|
|
MOVL R9, DI
|
|
SHLL $0x02, DI
|
|
CMPL R9, $0x0c
|
|
JAE emit_copy_three_lz4_s2
|
|
CMPL R8, $0x00000800
|
|
JAE emit_copy_three_lz4_s2
|
|
LEAL -15(DI), DI
|
|
MOVB R8, 1(AX)
|
|
SHRL $0x08, R8
|
|
SHLL $0x05, R8
|
|
ORL R8, DI
|
|
MOVB DI, (AX)
|
|
ADDQ $0x02, AX
|
|
JMP lz4_snappy_loop
|
|
|
|
emit_copy_three_lz4_s2:
|
|
LEAL -2(DI), DI
|
|
MOVB DI, (AX)
|
|
MOVW R8, 1(AX)
|
|
ADDQ $0x03, AX
|
|
JMP lz4_snappy_loop
|
|
|
|
lz4_snappy_done:
|
|
MOVQ dst_base+0(FP), CX
|
|
SUBQ CX, AX
|
|
MOVQ SI, uncompressed+48(FP)
|
|
MOVQ AX, dstUsed+56(FP)
|
|
RET
|
|
|
|
lz4_snappy_corrupt:
|
|
XORQ AX, AX
|
|
LEAQ -1(AX), SI
|
|
MOVQ SI, uncompressed+48(FP)
|
|
RET
|
|
|
|
lz4_snappy_dstfull:
|
|
XORQ AX, AX
|
|
LEAQ -2(AX), SI
|
|
MOVQ SI, uncompressed+48(FP)
|
|
RET
|
|
|
|
// func cvtLZ4sBlockSnappyAsm(dst []byte, src []byte) (uncompressed int, dstUsed int)
|
|
// Requires: SSE2
|
|
TEXT ·cvtLZ4sBlockSnappyAsm(SB), NOSPLIT, $0-64
|
|
XORQ SI, SI
|
|
MOVQ dst_base+0(FP), AX
|
|
MOVQ dst_len+8(FP), CX
|
|
MOVQ src_base+24(FP), DX
|
|
MOVQ src_len+32(FP), BX
|
|
LEAQ (DX)(BX*1), BX
|
|
LEAQ -8(AX)(CX*1), CX
|
|
|
|
lz4s_snappy_loop:
|
|
CMPQ DX, BX
|
|
JAE lz4s_snappy_corrupt
|
|
CMPQ AX, CX
|
|
JAE lz4s_snappy_dstfull
|
|
MOVBQZX (DX), DI
|
|
MOVQ DI, R8
|
|
MOVQ DI, R9
|
|
SHRQ $0x04, R8
|
|
ANDQ $0x0f, R9
|
|
CMPQ DI, $0xf0
|
|
JB lz4s_snappy_ll_end
|
|
|
|
lz4s_snappy_ll_loop:
|
|
INCQ DX
|
|
CMPQ DX, BX
|
|
JAE lz4s_snappy_corrupt
|
|
MOVBQZX (DX), DI
|
|
ADDQ DI, R8
|
|
CMPQ DI, $0xff
|
|
JEQ lz4s_snappy_ll_loop
|
|
|
|
lz4s_snappy_ll_end:
|
|
LEAQ (DX)(R8*1), DI
|
|
ADDQ $0x03, R9
|
|
CMPQ DI, BX
|
|
JAE lz4s_snappy_corrupt
|
|
INCQ DX
|
|
INCQ DI
|
|
TESTQ R8, R8
|
|
JZ lz4s_snappy_lits_done
|
|
LEAQ (AX)(R8*1), R10
|
|
CMPQ R10, CX
|
|
JAE lz4s_snappy_dstfull
|
|
ADDQ R8, SI
|
|
LEAL -1(R8), R10
|
|
CMPL R10, $0x3c
|
|
JB one_byte_lz4s_snappy
|
|
CMPL R10, $0x00000100
|
|
JB two_bytes_lz4s_snappy
|
|
CMPL R10, $0x00010000
|
|
JB three_bytes_lz4s_snappy
|
|
CMPL R10, $0x01000000
|
|
JB four_bytes_lz4s_snappy
|
|
MOVB $0xfc, (AX)
|
|
MOVL R10, 1(AX)
|
|
ADDQ $0x05, AX
|
|
JMP memmove_long_lz4s_snappy
|
|
|
|
four_bytes_lz4s_snappy:
|
|
MOVL R10, R11
|
|
SHRL $0x10, R11
|
|
MOVB $0xf8, (AX)
|
|
MOVW R10, 1(AX)
|
|
MOVB R11, 3(AX)
|
|
ADDQ $0x04, AX
|
|
JMP memmove_long_lz4s_snappy
|
|
|
|
three_bytes_lz4s_snappy:
|
|
MOVB $0xf4, (AX)
|
|
MOVW R10, 1(AX)
|
|
ADDQ $0x03, AX
|
|
JMP memmove_long_lz4s_snappy
|
|
|
|
two_bytes_lz4s_snappy:
|
|
MOVB $0xf0, (AX)
|
|
MOVB R10, 1(AX)
|
|
ADDQ $0x02, AX
|
|
CMPL R10, $0x40
|
|
JB memmove_lz4s_snappy
|
|
JMP memmove_long_lz4s_snappy
|
|
|
|
one_byte_lz4s_snappy:
|
|
SHLB $0x02, R10
|
|
MOVB R10, (AX)
|
|
ADDQ $0x01, AX
|
|
|
|
memmove_lz4s_snappy:
|
|
LEAQ (AX)(R8*1), R10
|
|
|
|
// genMemMoveShort
|
|
CMPQ R8, $0x08
|
|
JBE emit_lit_memmove_lz4s_snappy_memmove_move_8
|
|
CMPQ R8, $0x10
|
|
JBE emit_lit_memmove_lz4s_snappy_memmove_move_8through16
|
|
CMPQ R8, $0x20
|
|
JBE emit_lit_memmove_lz4s_snappy_memmove_move_17through32
|
|
JMP emit_lit_memmove_lz4s_snappy_memmove_move_33through64
|
|
|
|
emit_lit_memmove_lz4s_snappy_memmove_move_8:
|
|
MOVQ (DX), R11
|
|
MOVQ R11, (AX)
|
|
JMP memmove_end_copy_lz4s_snappy
|
|
|
|
emit_lit_memmove_lz4s_snappy_memmove_move_8through16:
|
|
MOVQ (DX), R11
|
|
MOVQ -8(DX)(R8*1), DX
|
|
MOVQ R11, (AX)
|
|
MOVQ DX, -8(AX)(R8*1)
|
|
JMP memmove_end_copy_lz4s_snappy
|
|
|
|
emit_lit_memmove_lz4s_snappy_memmove_move_17through32:
|
|
MOVOU (DX), X0
|
|
MOVOU -16(DX)(R8*1), X1
|
|
MOVOU X0, (AX)
|
|
MOVOU X1, -16(AX)(R8*1)
|
|
JMP memmove_end_copy_lz4s_snappy
|
|
|
|
emit_lit_memmove_lz4s_snappy_memmove_move_33through64:
|
|
MOVOU (DX), X0
|
|
MOVOU 16(DX), X1
|
|
MOVOU -32(DX)(R8*1), X2
|
|
MOVOU -16(DX)(R8*1), X3
|
|
MOVOU X0, (AX)
|
|
MOVOU X1, 16(AX)
|
|
MOVOU X2, -32(AX)(R8*1)
|
|
MOVOU X3, -16(AX)(R8*1)
|
|
|
|
memmove_end_copy_lz4s_snappy:
|
|
MOVQ R10, AX
|
|
JMP lz4s_snappy_lits_emit_done
|
|
|
|
memmove_long_lz4s_snappy:
|
|
LEAQ (AX)(R8*1), R10
|
|
|
|
// genMemMoveLong
|
|
MOVOU (DX), X0
|
|
MOVOU 16(DX), X1
|
|
MOVOU -32(DX)(R8*1), X2
|
|
MOVOU -16(DX)(R8*1), X3
|
|
MOVQ R8, R12
|
|
SHRQ $0x05, R12
|
|
MOVQ AX, R11
|
|
ANDL $0x0000001f, R11
|
|
MOVQ $0x00000040, R13
|
|
SUBQ R11, R13
|
|
DECQ R12
|
|
JA emit_lit_memmove_long_lz4s_snappylarge_forward_sse_loop_32
|
|
LEAQ -32(DX)(R13*1), R11
|
|
LEAQ -32(AX)(R13*1), R14
|
|
|
|
emit_lit_memmove_long_lz4s_snappylarge_big_loop_back:
|
|
MOVOU (R11), X4
|
|
MOVOU 16(R11), X5
|
|
MOVOA X4, (R14)
|
|
MOVOA X5, 16(R14)
|
|
ADDQ $0x20, R14
|
|
ADDQ $0x20, R11
|
|
ADDQ $0x20, R13
|
|
DECQ R12
|
|
JNA emit_lit_memmove_long_lz4s_snappylarge_big_loop_back
|
|
|
|
emit_lit_memmove_long_lz4s_snappylarge_forward_sse_loop_32:
|
|
MOVOU -32(DX)(R13*1), X4
|
|
MOVOU -16(DX)(R13*1), X5
|
|
MOVOA X4, -32(AX)(R13*1)
|
|
MOVOA X5, -16(AX)(R13*1)
|
|
ADDQ $0x20, R13
|
|
CMPQ R8, R13
|
|
JAE emit_lit_memmove_long_lz4s_snappylarge_forward_sse_loop_32
|
|
MOVOU X0, (AX)
|
|
MOVOU X1, 16(AX)
|
|
MOVOU X2, -32(AX)(R8*1)
|
|
MOVOU X3, -16(AX)(R8*1)
|
|
MOVQ R10, AX
|
|
|
|
lz4s_snappy_lits_emit_done:
|
|
MOVQ DI, DX
|
|
|
|
lz4s_snappy_lits_done:
|
|
CMPQ DX, BX
|
|
JNE lz4s_snappy_match
|
|
CMPQ R9, $0x03
|
|
JEQ lz4s_snappy_done
|
|
JMP lz4s_snappy_corrupt
|
|
|
|
lz4s_snappy_match:
|
|
CMPQ R9, $0x03
|
|
JEQ lz4s_snappy_loop
|
|
LEAQ 2(DX), DI
|
|
CMPQ DI, BX
|
|
JAE lz4s_snappy_corrupt
|
|
MOVWQZX (DX), R8
|
|
MOVQ DI, DX
|
|
TESTQ R8, R8
|
|
JZ lz4s_snappy_corrupt
|
|
CMPQ R8, SI
|
|
JA lz4s_snappy_corrupt
|
|
CMPQ R9, $0x12
|
|
JNE lz4s_snappy_ml_done
|
|
|
|
lz4s_snappy_ml_loop:
|
|
MOVBQZX (DX), DI
|
|
INCQ DX
|
|
ADDQ DI, R9
|
|
CMPQ DX, BX
|
|
JAE lz4s_snappy_corrupt
|
|
CMPQ DI, $0xff
|
|
JEQ lz4s_snappy_ml_loop
|
|
|
|
lz4s_snappy_ml_done:
|
|
ADDQ R9, SI
|
|
|
|
// emitCopy
|
|
two_byte_offset_lz4_s2:
|
|
CMPL R9, $0x40
|
|
JBE two_byte_offset_short_lz4_s2
|
|
MOVB $0xee, (AX)
|
|
MOVW R8, 1(AX)
|
|
LEAL -60(R9), R9
|
|
ADDQ $0x03, AX
|
|
CMPQ AX, CX
|
|
JAE lz4s_snappy_loop
|
|
JMP two_byte_offset_lz4_s2
|
|
|
|
two_byte_offset_short_lz4_s2:
|
|
MOVL R9, DI
|
|
SHLL $0x02, DI
|
|
CMPL R9, $0x0c
|
|
JAE emit_copy_three_lz4_s2
|
|
CMPL R8, $0x00000800
|
|
JAE emit_copy_three_lz4_s2
|
|
LEAL -15(DI), DI
|
|
MOVB R8, 1(AX)
|
|
SHRL $0x08, R8
|
|
SHLL $0x05, R8
|
|
ORL R8, DI
|
|
MOVB DI, (AX)
|
|
ADDQ $0x02, AX
|
|
JMP lz4s_snappy_loop
|
|
|
|
emit_copy_three_lz4_s2:
|
|
LEAL -2(DI), DI
|
|
MOVB DI, (AX)
|
|
MOVW R8, 1(AX)
|
|
ADDQ $0x03, AX
|
|
JMP lz4s_snappy_loop
|
|
|
|
lz4s_snappy_done:
|
|
MOVQ dst_base+0(FP), CX
|
|
SUBQ CX, AX
|
|
MOVQ SI, uncompressed+48(FP)
|
|
MOVQ AX, dstUsed+56(FP)
|
|
RET
|
|
|
|
lz4s_snappy_corrupt:
|
|
XORQ AX, AX
|
|
LEAQ -1(AX), SI
|
|
MOVQ SI, uncompressed+48(FP)
|
|
RET
|
|
|
|
lz4s_snappy_dstfull:
|
|
XORQ AX, AX
|
|
LEAQ -2(AX), SI
|
|
MOVQ SI, uncompressed+48(FP)
|
|
RET
|