jochen
be92762969
The live tests reached one shared bus and assert, read and remove the mesh's own objects by their fixed names, so packages run in parallel deleted what each other read and the suite passed only one package at a time; a red suite read as noise. internal/testbus starts a server per test, linked in at the nats-server release go.mod pins, and a test holds that pin to the catalogue's bus image and to the facts snapshot's bus when there is one, so the tests never run a bus the mesh does not. The waiter test read a timing (the most connections held at one look) and now reads the state it means (the fewest held across the wait). make check runs the packages in parallel under the race detector, with a timeout.
23563 lines
564 KiB
ArmAsm
23563 lines
564 KiB
ArmAsm
// Code generated by command: go run gen.go -out ../encodeblock.s -arch arm64 -arm64gen -pkg=s2. DO NOT EDIT.
|
|
// EXPERIMENTAL arm64 output lowered from an amd64 avo program.
|
|
|
|
//go:build arm64 && ((amd64 || arm64) && !appengine && !noasm && gc && !noasm)
|
|
|
|
#include "textflag.h"
|
|
|
|
// func encodeBlockAsm(dst []byte, src []byte, tmp *[65536]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeBlockAsm(SB), $24-64
|
|
MOVD tmp+48(FP), R0
|
|
MOVD dst_base+0(FP), R1
|
|
MOVD $0x00000200, R2
|
|
MOVD R0, R3
|
|
VEOR V0.B16, V0.B16, V0.B16
|
|
|
|
zero_loop_encodeBlockAsm:
|
|
FMOVQ F0, (R3)
|
|
FMOVQ F0, 16(R3)
|
|
FMOVQ F0, 32(R3)
|
|
FMOVQ F0, 48(R3)
|
|
FMOVQ F0, 64(R3)
|
|
FMOVQ F0, 80(R3)
|
|
FMOVQ F0, 96(R3)
|
|
FMOVQ F0, 112(R3)
|
|
ADD $0x80, R3, R3
|
|
SUBS $1, R2, R2
|
|
BNE zero_loop_encodeBlockAsm
|
|
MOVD $0x00000000, R16
|
|
MOVW R16, 20(RSP)
|
|
MOVD src_len+32(FP), R2
|
|
SUB $9, R2, R3
|
|
SUB $8, R2, R5
|
|
MOVW R5, 16(RSP)
|
|
LSR $0x05, R2, R2
|
|
SUBW R2, R3, R3
|
|
ADD R3, R1, R3
|
|
MOVD R3, 8(RSP)
|
|
MOVD $0x00000001, R2
|
|
MOVW R2, 24(RSP)
|
|
MOVD src_base+24(FP), R3
|
|
|
|
search_loop_encodeBlockAsm:
|
|
MOVWU R2, R5
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R5, R5
|
|
LSRW $0x06, R5, R5
|
|
ADD R5, R2, R5
|
|
ADD $4, R5, R5
|
|
MOVWU R5, R5
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R5
|
|
BHS emit_remainder_encodeBlockAsm
|
|
MOVD (R3)(R2), R6
|
|
MOVW R5, 28(RSP)
|
|
MOVD $0x0000cf1bbcdcbf9b, R8
|
|
MOVD R6, R9
|
|
MOVD R6, R10
|
|
LSR $0x08, R10, R10
|
|
LSL $0x10, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x32, R9, R9
|
|
LSL $0x10, R10, R10
|
|
MUL R8, R10, R10
|
|
LSR $0x32, R10, R10
|
|
MOVWU (R0)(R9<<2), R5
|
|
MOVWU (R0)(R10<<2), R7
|
|
MOVW R2, (R0)(R9<<2)
|
|
ADD $1, R2, R9
|
|
MOVWU R9, R9
|
|
MOVW R9, (R0)(R10<<2)
|
|
MOVD R6, R9
|
|
LSR $0x10, R9, R9
|
|
LSL $0x10, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x32, R9, R9
|
|
MOVWU R2, R8
|
|
MOVWU 24(RSP), R16
|
|
SUBW R16, R8, R8
|
|
ADD R8, R3, R15
|
|
MOVWU 1(R15), R10
|
|
MOVD R6, R8
|
|
LSR $0x08, R8, R8
|
|
CMPW R10, R8
|
|
BNE no_repeat_found_encodeBlockAsm
|
|
ADD $1, R2, R6
|
|
MOVWU R6, R6
|
|
MOVWU 20(RSP), R7
|
|
MOVWU R6, R5
|
|
MOVWU 24(RSP), R16
|
|
SUBSW R16, R5, R5
|
|
BEQ repeat_extend_back_end_encodeBlockAsm
|
|
|
|
repeat_extend_back_loop_encodeBlockAsm:
|
|
CMPW R7, R6
|
|
BLS repeat_extend_back_end_encodeBlockAsm
|
|
ADD R5, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R8
|
|
ADD R6, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R9
|
|
AND $0xff, R9, R16
|
|
AND $0xff, R8, R15
|
|
CMP R16, R15
|
|
BNE repeat_extend_back_end_encodeBlockAsm
|
|
SUB $1, R6, R6
|
|
MOVWU R6, R6
|
|
SUBSW $1, R5, R5
|
|
BNE repeat_extend_back_loop_encodeBlockAsm
|
|
|
|
repeat_extend_back_end_encodeBlockAsm:
|
|
MOVWU R6, R5
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R5, R5
|
|
ADD R5, R1, R5
|
|
ADD $5, R5, R5
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R5
|
|
BLO repeat_dst_size_check_encodeBlockAsm
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
repeat_dst_size_check_encodeBlockAsm:
|
|
MOVWU 20(RSP), R5
|
|
CMPW R6, R5
|
|
BEQ emit_literal_done_repeat_emit_encodeBlockAsm
|
|
MOVWU R6, R8
|
|
MOVW R6, 20(RSP)
|
|
ADD R5, R3, R9
|
|
SUBW R5, R8, R8
|
|
SUB $1, R8, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x3c, R5
|
|
BLO one_byte_repeat_emit_encodeBlockAsm
|
|
CMPW $0x00000100, R5
|
|
BLO two_bytes_repeat_emit_encodeBlockAsm
|
|
CMPW $0x00010000, R5
|
|
BLO three_bytes_repeat_emit_encodeBlockAsm
|
|
CMPW $0x01000000, R5
|
|
BLO four_bytes_repeat_emit_encodeBlockAsm
|
|
MOVD $0xfc, R16
|
|
MOVB R16, (R1)
|
|
MOVW R5, 1(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP memmove_long_repeat_emit_encodeBlockAsm
|
|
|
|
four_bytes_repeat_emit_encodeBlockAsm:
|
|
MOVWU R5, R10
|
|
LSRW $0x10, R10, R10
|
|
MOVD $0xf8, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
MOVB R10, 3(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP memmove_long_repeat_emit_encodeBlockAsm
|
|
|
|
three_bytes_repeat_emit_encodeBlockAsm:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_repeat_emit_encodeBlockAsm
|
|
|
|
two_bytes_repeat_emit_encodeBlockAsm:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R5, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R5
|
|
BLO memmove_repeat_emit_encodeBlockAsm
|
|
JMP memmove_long_repeat_emit_encodeBlockAsm
|
|
|
|
one_byte_repeat_emit_encodeBlockAsm:
|
|
LSLW $0x02, R5, R16
|
|
BFI $0, R16, $8, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_repeat_emit_encodeBlockAsm:
|
|
ADD R8, R1, R5
|
|
|
|
// genMemMoveShort
|
|
CMP $0x08, R8
|
|
BLS emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_8
|
|
CMP $0x10, R8
|
|
BLS emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_8through16
|
|
CMP $0x20, R8
|
|
BLS emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_17through32
|
|
JMP emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_33through64
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_8:
|
|
MOVD (R9), R10
|
|
MOVD R10, (R1)
|
|
JMP memmove_end_copy_repeat_emit_encodeBlockAsm
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_8through16:
|
|
MOVD (R9), R10
|
|
ADD R8, R9, R15
|
|
MOVD -8(R15), R9
|
|
MOVD R10, (R1)
|
|
ADD R8, R1, R15
|
|
MOVD R9, -8(R15)
|
|
JMP memmove_end_copy_repeat_emit_encodeBlockAsm
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_17through32:
|
|
FMOVQ (R9), F0
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_repeat_emit_encodeBlockAsm
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_33through64:
|
|
FMOVQ (R9), F0
|
|
FMOVQ 16(R9), F1
|
|
ADD R8, R9, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_repeat_emit_encodeBlockAsm:
|
|
MOVD R5, R1
|
|
JMP emit_literal_done_repeat_emit_encodeBlockAsm
|
|
|
|
memmove_long_repeat_emit_encodeBlockAsm:
|
|
ADD R8, R1, R5
|
|
|
|
// genMemMoveLong
|
|
MOVD R9, R10
|
|
MOVD R1, R11
|
|
MOVD R8, R12
|
|
|
|
emit_lit_memmove_long_repeat_emit_encodeBlockAsmlarge_big_loop_back:
|
|
FMOVQ (R10), F0
|
|
FMOVQ 16(R10), F1
|
|
FMOVQ F0, (R11)
|
|
FMOVQ F1, 16(R11)
|
|
ADD $0x20, R10, R10
|
|
ADD $0x20, R11, R11
|
|
SUB $0x20, R12, R12
|
|
CMP $0x20, R12
|
|
BHS emit_lit_memmove_long_repeat_emit_encodeBlockAsmlarge_big_loop_back
|
|
ADD R8, R9, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R8, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R5, R1
|
|
|
|
emit_literal_done_repeat_emit_encodeBlockAsm:
|
|
ADDW $0x05, R2, R2
|
|
MOVWU R2, R5
|
|
MOVWU 24(RSP), R16
|
|
SUBW R16, R5, R5
|
|
MOVD src_len+32(FP), R8
|
|
SUBW R2, R8, R8
|
|
ADD R2, R3, R9
|
|
ADD R5, R3, R5
|
|
|
|
// matchLen
|
|
MOVD $0, R11
|
|
|
|
matchlen_loopback_16_repeat_extend_encodeBlockAsm:
|
|
CMPW $0x10, R8
|
|
BLO matchlen_match8_repeat_extend_encodeBlockAsm
|
|
MOVD (R9)(R11), R10
|
|
ADD R11, R9, R15
|
|
MOVD 8(R15), R12
|
|
MOVD (R5)(R11), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_8_repeat_extend_encodeBlockAsm
|
|
ADD R11, R5, R15
|
|
MOVD 8(R15), R16
|
|
EOR R16, R12, R12
|
|
TST R12, R12
|
|
BNE matchlen_bsf_16repeat_extend_encodeBlockAsm
|
|
SUB $16, R8, R8
|
|
MOVWU R8, R8
|
|
ADD $16, R11, R11
|
|
MOVWU R11, R11
|
|
JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm
|
|
|
|
matchlen_bsf_16repeat_extend_encodeBlockAsm:
|
|
RBIT R12, R12
|
|
CLZ R12, R12
|
|
ASR $0x03, R12, R12
|
|
ADD R12, R11, R11
|
|
ADD $8, R11, R11
|
|
MOVWU R11, R11
|
|
JMP repeat_extend_forward_end_encodeBlockAsm
|
|
|
|
matchlen_match8_repeat_extend_encodeBlockAsm:
|
|
CMPW $0x08, R8
|
|
BLO matchlen_match4_repeat_extend_encodeBlockAsm
|
|
MOVD (R9)(R11), R10
|
|
MOVD (R5)(R11), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_8_repeat_extend_encodeBlockAsm
|
|
SUB $8, R8, R8
|
|
MOVWU R8, R8
|
|
ADD $8, R11, R11
|
|
MOVWU R11, R11
|
|
JMP matchlen_match4_repeat_extend_encodeBlockAsm
|
|
|
|
matchlen_bsf_8_repeat_extend_encodeBlockAsm:
|
|
RBIT R10, R10
|
|
CLZ R10, R10
|
|
ASR $0x03, R10, R10
|
|
ADD R10, R11, R11
|
|
MOVWU R11, R11
|
|
JMP repeat_extend_forward_end_encodeBlockAsm
|
|
|
|
matchlen_match4_repeat_extend_encodeBlockAsm:
|
|
CMPW $0x04, R8
|
|
BLO matchlen_match2_repeat_extend_encodeBlockAsm
|
|
MOVWU (R9)(R11), R10
|
|
MOVWU (R5)(R11), R16
|
|
CMPW R10, R16
|
|
BNE matchlen_match2_repeat_extend_encodeBlockAsm
|
|
SUB $4, R8, R8
|
|
MOVWU R8, R8
|
|
ADD $4, R11, R11
|
|
MOVWU R11, R11
|
|
|
|
matchlen_match2_repeat_extend_encodeBlockAsm:
|
|
CMPW $0x01, R8
|
|
BEQ matchlen_match1_repeat_extend_encodeBlockAsm
|
|
BLO repeat_extend_forward_end_encodeBlockAsm
|
|
MOVHU (R9)(R11), R16
|
|
BFI $0, R16, $16, R10
|
|
ADD R11, R5, R15
|
|
MOVHU (R15), R15
|
|
AND $0xffff, R10, R16
|
|
CMP R16, R15
|
|
BNE matchlen_match1_repeat_extend_encodeBlockAsm
|
|
ADD $2, R11, R11
|
|
MOVWU R11, R11
|
|
SUBSW $0x02, R8, R8
|
|
BEQ repeat_extend_forward_end_encodeBlockAsm
|
|
|
|
matchlen_match1_repeat_extend_encodeBlockAsm:
|
|
MOVBU (R9)(R11), R16
|
|
BFI $0, R16, $8, R10
|
|
ADD R11, R5, R15
|
|
MOVBU (R15), R15
|
|
AND $0xff, R10, R16
|
|
CMP R16, R15
|
|
BNE repeat_extend_forward_end_encodeBlockAsm
|
|
ADD $1, R11, R11
|
|
MOVWU R11, R11
|
|
|
|
repeat_extend_forward_end_encodeBlockAsm:
|
|
ADDW R11, R2, R2
|
|
MOVWU R2, R5
|
|
SUBW R6, R5, R5
|
|
MOVWU 24(RSP), R6
|
|
TSTW R7, R7
|
|
BEQ repeat_as_copy_encodeBlockAsm
|
|
|
|
// emitRepeat
|
|
emit_repeat_again_match_repeat_encodeBlockAsm:
|
|
MOVWU R5, R7
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x08, R7
|
|
BLS repeat_two_match_repeat_encodeBlockAsm
|
|
CMPW $0x0c, R7
|
|
BHS cant_repeat_two_offset_match_repeat_encodeBlockAsm
|
|
CMPW $0x00000800, R6
|
|
BLO repeat_two_offset_match_repeat_encodeBlockAsm
|
|
|
|
cant_repeat_two_offset_match_repeat_encodeBlockAsm:
|
|
CMPW $0x00000104, R5
|
|
BLO repeat_three_match_repeat_encodeBlockAsm
|
|
CMPW $0x00010100, R5
|
|
BLO repeat_four_match_repeat_encodeBlockAsm
|
|
CMPW $0x0100ffff, R5
|
|
BLO repeat_five_match_repeat_encodeBlockAsm
|
|
SUB $16842747, R5, R5
|
|
MOVWU R5, R5
|
|
MOVD $0xfffb001d, R16
|
|
MOVW R16, (R1)
|
|
MOVD $0xff, R16
|
|
MOVB R16, 4(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP emit_repeat_again_match_repeat_encodeBlockAsm
|
|
|
|
repeat_five_match_repeat_encodeBlockAsm:
|
|
SUB $65536, R5, R5
|
|
MOVWU R5, R5
|
|
MOVWU R5, R6
|
|
MOVD $0x001d, R16
|
|
MOVH R16, (R1)
|
|
MOVH R5, 2(R1)
|
|
ASRW $0x10, R6, R6
|
|
MOVB R6, 4(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
repeat_four_match_repeat_encodeBlockAsm:
|
|
SUB $256, R5, R5
|
|
MOVWU R5, R5
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R5, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
repeat_three_match_repeat_encodeBlockAsm:
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R5, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
repeat_two_match_repeat_encodeBlockAsm:
|
|
LSLW $0x02, R5, R5
|
|
ORRW $0x01, R5, R5
|
|
MOVH R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
repeat_two_offset_match_repeat_encodeBlockAsm:
|
|
MOVD $0, R7
|
|
ADD R5<<2, R7, R5
|
|
ADD $1, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R6, 1(R1)
|
|
ASRW $0x08, R6, R6
|
|
LSLW $0x05, R6, R6
|
|
ORRW R6, R5, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
repeat_as_copy_encodeBlockAsm:
|
|
// emitCopy
|
|
CMPW $0x00010000, R6
|
|
BLO two_byte_offset_repeat_as_copy_encodeBlockAsm
|
|
CMPW $0x40, R5
|
|
BLS four_bytes_remain_repeat_as_copy_encodeBlockAsm
|
|
MOVD $0xff, R16
|
|
MOVB R16, (R1)
|
|
MOVW R6, 1(R1)
|
|
SUB $64, R5, R5
|
|
MOVWU R5, R5
|
|
ADD $0x05, R1, R1
|
|
CMPW $0x04, R5
|
|
BLO four_bytes_remain_repeat_as_copy_encodeBlockAsm
|
|
|
|
// emitRepeat
|
|
emit_repeat_again_repeat_as_copy_encodeBlockAsm_emit_copy:
|
|
MOVWU R5, R7
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x08, R7
|
|
BLS repeat_two_repeat_as_copy_encodeBlockAsm_emit_copy
|
|
CMPW $0x0c, R7
|
|
BHS cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy
|
|
CMPW $0x00000800, R6
|
|
BLO repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy
|
|
|
|
cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy:
|
|
CMPW $0x00000104, R5
|
|
BLO repeat_three_repeat_as_copy_encodeBlockAsm_emit_copy
|
|
CMPW $0x00010100, R5
|
|
BLO repeat_four_repeat_as_copy_encodeBlockAsm_emit_copy
|
|
CMPW $0x0100ffff, R5
|
|
BLO repeat_five_repeat_as_copy_encodeBlockAsm_emit_copy
|
|
SUB $16842747, R5, R5
|
|
MOVWU R5, R5
|
|
MOVD $0xfffb001d, R16
|
|
MOVW R16, (R1)
|
|
MOVD $0xff, R16
|
|
MOVB R16, 4(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP emit_repeat_again_repeat_as_copy_encodeBlockAsm_emit_copy
|
|
|
|
repeat_five_repeat_as_copy_encodeBlockAsm_emit_copy:
|
|
SUB $65536, R5, R5
|
|
MOVWU R5, R5
|
|
MOVWU R5, R6
|
|
MOVD $0x001d, R16
|
|
MOVH R16, (R1)
|
|
MOVH R5, 2(R1)
|
|
ASRW $0x10, R6, R6
|
|
MOVB R6, 4(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
repeat_four_repeat_as_copy_encodeBlockAsm_emit_copy:
|
|
SUB $256, R5, R5
|
|
MOVWU R5, R5
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R5, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
repeat_three_repeat_as_copy_encodeBlockAsm_emit_copy:
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R5, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
repeat_two_repeat_as_copy_encodeBlockAsm_emit_copy:
|
|
LSLW $0x02, R5, R5
|
|
ORRW $0x01, R5, R5
|
|
MOVH R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy:
|
|
MOVD $0, R7
|
|
ADD R5<<2, R7, R5
|
|
ADD $1, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R6, 1(R1)
|
|
ASRW $0x08, R6, R6
|
|
LSLW $0x05, R6, R6
|
|
ORRW R6, R5, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
four_bytes_remain_repeat_as_copy_encodeBlockAsm:
|
|
TSTW R5, R5
|
|
BEQ repeat_end_emit_encodeBlockAsm
|
|
MOVD $0, R7
|
|
ADD R5<<2, R7, R5
|
|
SUB $1, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R5, (R1)
|
|
MOVW R6, 1(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
two_byte_offset_repeat_as_copy_encodeBlockAsm:
|
|
CMPW $0x40, R5
|
|
BLS two_byte_offset_short_repeat_as_copy_encodeBlockAsm
|
|
CMPW $0x00000800, R6
|
|
BHS long_offset_short_repeat_as_copy_encodeBlockAsm
|
|
MOVD $0x00000001, R7
|
|
ADD $16, R7, R7
|
|
MOVWU R7, R7
|
|
MOVB R6, 1(R1)
|
|
MOVWU R6, R8
|
|
LSRW $0x08, R8, R8
|
|
LSLW $0x05, R8, R8
|
|
ORRW R8, R7, R7
|
|
MOVB R7, (R1)
|
|
ADD $0x02, R1, R1
|
|
SUBW $0x08, R5, R5
|
|
|
|
// emitRepeat
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
JMP cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b
|
|
|
|
emit_repeat_again_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b:
|
|
MOVWU R5, R7
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x08, R7
|
|
BLS repeat_two_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b
|
|
CMPW $0x0c, R7
|
|
BHS cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b
|
|
CMPW $0x00000800, R6
|
|
BLO repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b
|
|
|
|
cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b:
|
|
CMPW $0x00000104, R5
|
|
BLO repeat_three_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b
|
|
CMPW $0x00010100, R5
|
|
BLO repeat_four_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b
|
|
CMPW $0x0100ffff, R5
|
|
BLO repeat_five_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b
|
|
SUB $16842747, R5, R5
|
|
MOVWU R5, R5
|
|
MOVD $0xfffb001d, R16
|
|
MOVW R16, (R1)
|
|
MOVD $0xff, R16
|
|
MOVB R16, 4(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP emit_repeat_again_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b
|
|
|
|
repeat_five_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b:
|
|
SUB $65536, R5, R5
|
|
MOVWU R5, R5
|
|
MOVWU R5, R6
|
|
MOVD $0x001d, R16
|
|
MOVH R16, (R1)
|
|
MOVH R5, 2(R1)
|
|
ASRW $0x10, R6, R6
|
|
MOVB R6, 4(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
repeat_four_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b:
|
|
SUB $256, R5, R5
|
|
MOVWU R5, R5
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R5, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
repeat_three_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b:
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R5, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
repeat_two_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b:
|
|
LSLW $0x02, R5, R5
|
|
ORRW $0x01, R5, R5
|
|
MOVH R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b:
|
|
MOVD $0, R7
|
|
ADD R5<<2, R7, R5
|
|
ADD $1, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R6, 1(R1)
|
|
ASRW $0x08, R6, R6
|
|
LSLW $0x05, R6, R6
|
|
ORRW R6, R5, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
long_offset_short_repeat_as_copy_encodeBlockAsm:
|
|
MOVD $0xee, R16
|
|
MOVB R16, (R1)
|
|
MOVH R6, 1(R1)
|
|
SUB $60, R5, R5
|
|
MOVWU R5, R5
|
|
ADD $0x03, R1, R1
|
|
|
|
// emitRepeat
|
|
emit_repeat_again_repeat_as_copy_encodeBlockAsm_emit_copy_short:
|
|
MOVWU R5, R7
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x08, R7
|
|
BLS repeat_two_repeat_as_copy_encodeBlockAsm_emit_copy_short
|
|
CMPW $0x0c, R7
|
|
BHS cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short
|
|
CMPW $0x00000800, R6
|
|
BLO repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short
|
|
|
|
cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short:
|
|
CMPW $0x00000104, R5
|
|
BLO repeat_three_repeat_as_copy_encodeBlockAsm_emit_copy_short
|
|
CMPW $0x00010100, R5
|
|
BLO repeat_four_repeat_as_copy_encodeBlockAsm_emit_copy_short
|
|
CMPW $0x0100ffff, R5
|
|
BLO repeat_five_repeat_as_copy_encodeBlockAsm_emit_copy_short
|
|
SUB $16842747, R5, R5
|
|
MOVWU R5, R5
|
|
MOVD $0xfffb001d, R16
|
|
MOVW R16, (R1)
|
|
MOVD $0xff, R16
|
|
MOVB R16, 4(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP emit_repeat_again_repeat_as_copy_encodeBlockAsm_emit_copy_short
|
|
|
|
repeat_five_repeat_as_copy_encodeBlockAsm_emit_copy_short:
|
|
SUB $65536, R5, R5
|
|
MOVWU R5, R5
|
|
MOVWU R5, R6
|
|
MOVD $0x001d, R16
|
|
MOVH R16, (R1)
|
|
MOVH R5, 2(R1)
|
|
ASRW $0x10, R6, R6
|
|
MOVB R6, 4(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
repeat_four_repeat_as_copy_encodeBlockAsm_emit_copy_short:
|
|
SUB $256, R5, R5
|
|
MOVWU R5, R5
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R5, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
repeat_three_repeat_as_copy_encodeBlockAsm_emit_copy_short:
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R5, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
repeat_two_repeat_as_copy_encodeBlockAsm_emit_copy_short:
|
|
LSLW $0x02, R5, R5
|
|
ORRW $0x01, R5, R5
|
|
MOVH R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short:
|
|
MOVD $0, R7
|
|
ADD R5<<2, R7, R5
|
|
ADD $1, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R6, 1(R1)
|
|
ASRW $0x08, R6, R6
|
|
LSLW $0x05, R6, R6
|
|
ORRW R6, R5, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
two_byte_offset_short_repeat_as_copy_encodeBlockAsm:
|
|
MOVWU R5, R7
|
|
LSLW $0x02, R7, R7
|
|
CMPW $0x0c, R5
|
|
BHS emit_copy_three_repeat_as_copy_encodeBlockAsm
|
|
CMPW $0x00000800, R6
|
|
BHS emit_copy_three_repeat_as_copy_encodeBlockAsm
|
|
SUB $15, R7, R7
|
|
MOVWU R7, R7
|
|
MOVB R6, 1(R1)
|
|
LSRW $0x08, R6, R6
|
|
LSLW $0x05, R6, R6
|
|
ORRW R6, R7, R7
|
|
MOVB R7, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm
|
|
|
|
emit_copy_three_repeat_as_copy_encodeBlockAsm:
|
|
SUB $2, R7, R7
|
|
MOVWU R7, R7
|
|
MOVB R7, (R1)
|
|
MOVH R6, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
|
|
repeat_end_emit_encodeBlockAsm:
|
|
MOVW R2, 20(RSP)
|
|
JMP search_loop_encodeBlockAsm
|
|
|
|
no_repeat_found_encodeBlockAsm:
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ candidate_match_encodeBlockAsm
|
|
LSR $0x08, R6, R6
|
|
MOVWU (R0)(R9<<2), R5
|
|
ADD $2, R2, R8
|
|
MOVWU R8, R8
|
|
MOVWU (R3)(R7), R16
|
|
CMPW R6, R16
|
|
BEQ candidate2_match_encodeBlockAsm
|
|
MOVW R8, (R0)(R9<<2)
|
|
LSR $0x08, R6, R6
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ candidate3_match_encodeBlockAsm
|
|
MOVWU 28(RSP), R2
|
|
JMP search_loop_encodeBlockAsm
|
|
|
|
candidate3_match_encodeBlockAsm:
|
|
ADDW $0x02, R2, R2
|
|
JMP candidate_match_encodeBlockAsm
|
|
|
|
candidate2_match_encodeBlockAsm:
|
|
MOVW R8, (R0)(R9<<2)
|
|
ADDW $1, R2, R2
|
|
MOVWU R7, R5
|
|
|
|
candidate_match_encodeBlockAsm:
|
|
MOVWU 20(RSP), R6
|
|
TSTW R5, R5
|
|
BEQ match_extend_back_end_encodeBlockAsm
|
|
|
|
match_extend_back_loop_encodeBlockAsm:
|
|
CMPW R6, R2
|
|
BLS match_extend_back_end_encodeBlockAsm
|
|
ADD R5, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R7
|
|
ADD R2, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R8
|
|
AND $0xff, R8, R16
|
|
AND $0xff, R7, R15
|
|
CMP R16, R15
|
|
BNE match_extend_back_end_encodeBlockAsm
|
|
SUB $1, R2, R2
|
|
MOVWU R2, R2
|
|
SUBSW $1, R5, R5
|
|
BEQ match_extend_back_end_encodeBlockAsm
|
|
JMP match_extend_back_loop_encodeBlockAsm
|
|
|
|
match_extend_back_end_encodeBlockAsm:
|
|
MOVWU R2, R6
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R6, R6
|
|
ADD R6, R1, R6
|
|
ADD $5, R6, R6
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R6
|
|
BLO match_dst_size_check_encodeBlockAsm
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeBlockAsm:
|
|
MOVWU R2, R6
|
|
MOVWU 20(RSP), R7
|
|
CMPW R6, R7
|
|
BEQ emit_literal_done_match_emit_encodeBlockAsm
|
|
MOVWU R6, R8
|
|
MOVW R6, 20(RSP)
|
|
ADD R7, R3, R6
|
|
SUBW R7, R8, R8
|
|
SUB $1, R8, R7
|
|
MOVWU R7, R7
|
|
CMPW $0x3c, R7
|
|
BLO one_byte_match_emit_encodeBlockAsm
|
|
CMPW $0x00000100, R7
|
|
BLO two_bytes_match_emit_encodeBlockAsm
|
|
CMPW $0x00010000, R7
|
|
BLO three_bytes_match_emit_encodeBlockAsm
|
|
CMPW $0x01000000, R7
|
|
BLO four_bytes_match_emit_encodeBlockAsm
|
|
MOVD $0xfc, R16
|
|
MOVB R16, (R1)
|
|
MOVW R7, 1(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP memmove_long_match_emit_encodeBlockAsm
|
|
|
|
four_bytes_match_emit_encodeBlockAsm:
|
|
MOVWU R7, R9
|
|
LSRW $0x10, R9, R9
|
|
MOVD $0xf8, R16
|
|
MOVB R16, (R1)
|
|
MOVH R7, 1(R1)
|
|
MOVB R9, 3(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP memmove_long_match_emit_encodeBlockAsm
|
|
|
|
three_bytes_match_emit_encodeBlockAsm:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R7, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_match_emit_encodeBlockAsm
|
|
|
|
two_bytes_match_emit_encodeBlockAsm:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R7, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R7
|
|
BLO memmove_match_emit_encodeBlockAsm
|
|
JMP memmove_long_match_emit_encodeBlockAsm
|
|
|
|
one_byte_match_emit_encodeBlockAsm:
|
|
LSLW $0x02, R7, R16
|
|
BFI $0, R16, $8, R7
|
|
MOVB R7, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_match_emit_encodeBlockAsm:
|
|
ADD R8, R1, R7
|
|
|
|
// genMemMoveShort
|
|
CMP $0x08, R8
|
|
BLS emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_8
|
|
CMP $0x10, R8
|
|
BLS emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_8through16
|
|
CMP $0x20, R8
|
|
BLS emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_8:
|
|
MOVD (R6), R9
|
|
MOVD R9, (R1)
|
|
JMP memmove_end_copy_match_emit_encodeBlockAsm
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_8through16:
|
|
MOVD (R6), R9
|
|
ADD R8, R6, R15
|
|
MOVD -8(R15), R6
|
|
MOVD R9, (R1)
|
|
ADD R8, R1, R15
|
|
MOVD R6, -8(R15)
|
|
JMP memmove_end_copy_match_emit_encodeBlockAsm
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_17through32:
|
|
FMOVQ (R6), F0
|
|
ADD R8, R6, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_match_emit_encodeBlockAsm
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_33through64:
|
|
FMOVQ (R6), F0
|
|
FMOVQ 16(R6), F1
|
|
ADD R8, R6, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R8, R6, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_match_emit_encodeBlockAsm:
|
|
MOVD R7, R1
|
|
JMP emit_literal_done_match_emit_encodeBlockAsm
|
|
|
|
memmove_long_match_emit_encodeBlockAsm:
|
|
ADD R8, R1, R7
|
|
|
|
// genMemMoveLong
|
|
MOVD R6, R9
|
|
MOVD R1, R10
|
|
MOVD R8, R11
|
|
|
|
emit_lit_memmove_long_match_emit_encodeBlockAsmlarge_big_loop_back:
|
|
FMOVQ (R9), F0
|
|
FMOVQ 16(R9), F1
|
|
FMOVQ F0, (R10)
|
|
FMOVQ F1, 16(R10)
|
|
ADD $0x20, R9, R9
|
|
ADD $0x20, R10, R10
|
|
SUB $0x20, R11, R11
|
|
CMP $0x20, R11
|
|
BHS emit_lit_memmove_long_match_emit_encodeBlockAsmlarge_big_loop_back
|
|
ADD R8, R6, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R8, R6, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R8, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R7, R1
|
|
|
|
emit_literal_done_match_emit_encodeBlockAsm:
|
|
match_nolit_loop_encodeBlockAsm:
|
|
MOVWU R2, R6
|
|
SUBW R5, R6, R6
|
|
MOVW R6, 24(RSP)
|
|
ADDW $0x04, R2, R2
|
|
ADDW $0x04, R5, R5
|
|
MOVD src_len+32(FP), R6
|
|
SUBW R2, R6, R6
|
|
ADD R2, R3, R7
|
|
ADD R5, R3, R5
|
|
|
|
// matchLen
|
|
MOVD $0, R9
|
|
|
|
matchlen_loopback_16_match_nolit_encodeBlockAsm:
|
|
CMPW $0x10, R6
|
|
BLO matchlen_match8_match_nolit_encodeBlockAsm
|
|
MOVD (R7)(R9), R8
|
|
ADD R9, R7, R15
|
|
MOVD 8(R15), R10
|
|
MOVD (R5)(R9), R16
|
|
EOR R16, R8, R8
|
|
TST R8, R8
|
|
BNE matchlen_bsf_8_match_nolit_encodeBlockAsm
|
|
ADD R9, R5, R15
|
|
MOVD 8(R15), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_16match_nolit_encodeBlockAsm
|
|
SUB $16, R6, R6
|
|
MOVWU R6, R6
|
|
ADD $16, R9, R9
|
|
MOVWU R9, R9
|
|
JMP matchlen_loopback_16_match_nolit_encodeBlockAsm
|
|
|
|
matchlen_bsf_16match_nolit_encodeBlockAsm:
|
|
RBIT R10, R10
|
|
CLZ R10, R10
|
|
ASR $0x03, R10, R10
|
|
ADD R10, R9, R9
|
|
ADD $8, R9, R9
|
|
MOVWU R9, R9
|
|
JMP match_nolit_end_encodeBlockAsm
|
|
|
|
matchlen_match8_match_nolit_encodeBlockAsm:
|
|
CMPW $0x08, R6
|
|
BLO matchlen_match4_match_nolit_encodeBlockAsm
|
|
MOVD (R7)(R9), R8
|
|
MOVD (R5)(R9), R16
|
|
EOR R16, R8, R8
|
|
TST R8, R8
|
|
BNE matchlen_bsf_8_match_nolit_encodeBlockAsm
|
|
SUB $8, R6, R6
|
|
MOVWU R6, R6
|
|
ADD $8, R9, R9
|
|
MOVWU R9, R9
|
|
JMP matchlen_match4_match_nolit_encodeBlockAsm
|
|
|
|
matchlen_bsf_8_match_nolit_encodeBlockAsm:
|
|
RBIT R8, R8
|
|
CLZ R8, R8
|
|
ASR $0x03, R8, R8
|
|
ADD R8, R9, R9
|
|
MOVWU R9, R9
|
|
JMP match_nolit_end_encodeBlockAsm
|
|
|
|
matchlen_match4_match_nolit_encodeBlockAsm:
|
|
CMPW $0x04, R6
|
|
BLO matchlen_match2_match_nolit_encodeBlockAsm
|
|
MOVWU (R7)(R9), R8
|
|
MOVWU (R5)(R9), R16
|
|
CMPW R8, R16
|
|
BNE matchlen_match2_match_nolit_encodeBlockAsm
|
|
SUB $4, R6, R6
|
|
MOVWU R6, R6
|
|
ADD $4, R9, R9
|
|
MOVWU R9, R9
|
|
|
|
matchlen_match2_match_nolit_encodeBlockAsm:
|
|
CMPW $0x01, R6
|
|
BEQ matchlen_match1_match_nolit_encodeBlockAsm
|
|
BLO match_nolit_end_encodeBlockAsm
|
|
MOVHU (R7)(R9), R16
|
|
BFI $0, R16, $16, R8
|
|
ADD R9, R5, R15
|
|
MOVHU (R15), R15
|
|
AND $0xffff, R8, R16
|
|
CMP R16, R15
|
|
BNE matchlen_match1_match_nolit_encodeBlockAsm
|
|
ADD $2, R9, R9
|
|
MOVWU R9, R9
|
|
SUBSW $0x02, R6, R6
|
|
BEQ match_nolit_end_encodeBlockAsm
|
|
|
|
matchlen_match1_match_nolit_encodeBlockAsm:
|
|
MOVBU (R7)(R9), R16
|
|
BFI $0, R16, $8, R8
|
|
ADD R9, R5, R15
|
|
MOVBU (R15), R15
|
|
AND $0xff, R8, R16
|
|
CMP R16, R15
|
|
BNE match_nolit_end_encodeBlockAsm
|
|
ADD $1, R9, R9
|
|
MOVWU R9, R9
|
|
|
|
match_nolit_end_encodeBlockAsm:
|
|
ADDW R9, R2, R2
|
|
MOVWU 24(RSP), R5
|
|
ADDW $0x04, R9, R9
|
|
MOVW R2, 20(RSP)
|
|
|
|
// emitCopy
|
|
CMPW $0x00010000, R5
|
|
BLO two_byte_offset_match_nolit_encodeBlockAsm
|
|
CMPW $0x40, R9
|
|
BLS four_bytes_remain_match_nolit_encodeBlockAsm
|
|
MOVD $0xff, R16
|
|
MOVB R16, (R1)
|
|
MOVW R5, 1(R1)
|
|
SUB $64, R9, R9
|
|
MOVWU R9, R9
|
|
ADD $0x05, R1, R1
|
|
CMPW $0x04, R9
|
|
BLO four_bytes_remain_match_nolit_encodeBlockAsm
|
|
|
|
// emitRepeat
|
|
emit_repeat_again_match_nolit_encodeBlockAsm_emit_copy:
|
|
MOVWU R9, R6
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
CMPW $0x08, R6
|
|
BLS repeat_two_match_nolit_encodeBlockAsm_emit_copy
|
|
CMPW $0x0c, R6
|
|
BHS cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy
|
|
CMPW $0x00000800, R5
|
|
BLO repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy:
|
|
CMPW $0x00000104, R9
|
|
BLO repeat_three_match_nolit_encodeBlockAsm_emit_copy
|
|
CMPW $0x00010100, R9
|
|
BLO repeat_four_match_nolit_encodeBlockAsm_emit_copy
|
|
CMPW $0x0100ffff, R9
|
|
BLO repeat_five_match_nolit_encodeBlockAsm_emit_copy
|
|
SUB $16842747, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0xfffb001d, R16
|
|
MOVW R16, (R1)
|
|
MOVD $0xff, R16
|
|
MOVB R16, 4(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP emit_repeat_again_match_nolit_encodeBlockAsm_emit_copy
|
|
|
|
repeat_five_match_nolit_encodeBlockAsm_emit_copy:
|
|
SUB $65536, R9, R9
|
|
MOVWU R9, R9
|
|
MOVWU R9, R5
|
|
MOVD $0x001d, R16
|
|
MOVH R16, (R1)
|
|
MOVH R9, 2(R1)
|
|
ASRW $0x10, R5, R5
|
|
MOVB R5, 4(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm
|
|
|
|
repeat_four_match_nolit_encodeBlockAsm_emit_copy:
|
|
SUB $256, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R9, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm
|
|
|
|
repeat_three_match_nolit_encodeBlockAsm_emit_copy:
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R9, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm
|
|
|
|
repeat_two_match_nolit_encodeBlockAsm_emit_copy:
|
|
LSLW $0x02, R9, R9
|
|
ORRW $0x01, R9, R9
|
|
MOVH R9, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm
|
|
|
|
repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy:
|
|
MOVD $0, R6
|
|
ADD R9<<2, R6, R9
|
|
ADD $1, R9, R9
|
|
MOVWU R9, R9
|
|
MOVB R5, 1(R1)
|
|
ASRW $0x08, R5, R5
|
|
LSLW $0x05, R5, R5
|
|
ORRW R5, R9, R9
|
|
MOVB R9, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm
|
|
|
|
four_bytes_remain_match_nolit_encodeBlockAsm:
|
|
TSTW R9, R9
|
|
BEQ match_nolit_emitcopy_end_encodeBlockAsm
|
|
MOVD $0, R6
|
|
ADD R9<<2, R6, R9
|
|
SUB $1, R9, R9
|
|
MOVWU R9, R9
|
|
MOVB R9, (R1)
|
|
MOVW R5, 1(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm
|
|
|
|
two_byte_offset_match_nolit_encodeBlockAsm:
|
|
CMPW $0x40, R9
|
|
BLS two_byte_offset_short_match_nolit_encodeBlockAsm
|
|
CMPW $0x00000800, R5
|
|
BHS long_offset_short_match_nolit_encodeBlockAsm
|
|
MOVD $0x00000001, R6
|
|
ADD $16, R6, R6
|
|
MOVWU R6, R6
|
|
MOVB R5, 1(R1)
|
|
MOVWU R5, R7
|
|
LSRW $0x08, R7, R7
|
|
LSLW $0x05, R7, R7
|
|
ORRW R7, R6, R6
|
|
MOVB R6, (R1)
|
|
ADD $0x02, R1, R1
|
|
SUBW $0x08, R9, R9
|
|
|
|
// emitRepeat
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
JMP cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short_2b
|
|
|
|
emit_repeat_again_match_nolit_encodeBlockAsm_emit_copy_short_2b:
|
|
MOVWU R9, R6
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
CMPW $0x08, R6
|
|
BLS repeat_two_match_nolit_encodeBlockAsm_emit_copy_short_2b
|
|
CMPW $0x0c, R6
|
|
BHS cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short_2b
|
|
CMPW $0x00000800, R5
|
|
BLO repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short_2b
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short_2b:
|
|
CMPW $0x00000104, R9
|
|
BLO repeat_three_match_nolit_encodeBlockAsm_emit_copy_short_2b
|
|
CMPW $0x00010100, R9
|
|
BLO repeat_four_match_nolit_encodeBlockAsm_emit_copy_short_2b
|
|
CMPW $0x0100ffff, R9
|
|
BLO repeat_five_match_nolit_encodeBlockAsm_emit_copy_short_2b
|
|
SUB $16842747, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0xfffb001d, R16
|
|
MOVW R16, (R1)
|
|
MOVD $0xff, R16
|
|
MOVB R16, 4(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP emit_repeat_again_match_nolit_encodeBlockAsm_emit_copy_short_2b
|
|
|
|
repeat_five_match_nolit_encodeBlockAsm_emit_copy_short_2b:
|
|
SUB $65536, R9, R9
|
|
MOVWU R9, R9
|
|
MOVWU R9, R5
|
|
MOVD $0x001d, R16
|
|
MOVH R16, (R1)
|
|
MOVH R9, 2(R1)
|
|
ASRW $0x10, R5, R5
|
|
MOVB R5, 4(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm
|
|
|
|
repeat_four_match_nolit_encodeBlockAsm_emit_copy_short_2b:
|
|
SUB $256, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R9, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm
|
|
|
|
repeat_three_match_nolit_encodeBlockAsm_emit_copy_short_2b:
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R9, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm
|
|
|
|
repeat_two_match_nolit_encodeBlockAsm_emit_copy_short_2b:
|
|
LSLW $0x02, R9, R9
|
|
ORRW $0x01, R9, R9
|
|
MOVH R9, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm
|
|
|
|
repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short_2b:
|
|
MOVD $0, R6
|
|
ADD R9<<2, R6, R9
|
|
ADD $1, R9, R9
|
|
MOVWU R9, R9
|
|
MOVB R5, 1(R1)
|
|
ASRW $0x08, R5, R5
|
|
LSLW $0x05, R5, R5
|
|
ORRW R5, R9, R9
|
|
MOVB R9, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm
|
|
|
|
long_offset_short_match_nolit_encodeBlockAsm:
|
|
MOVD $0xee, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
SUB $60, R9, R9
|
|
MOVWU R9, R9
|
|
ADD $0x03, R1, R1
|
|
|
|
// emitRepeat
|
|
emit_repeat_again_match_nolit_encodeBlockAsm_emit_copy_short:
|
|
MOVWU R9, R6
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
CMPW $0x08, R6
|
|
BLS repeat_two_match_nolit_encodeBlockAsm_emit_copy_short
|
|
CMPW $0x0c, R6
|
|
BHS cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short
|
|
CMPW $0x00000800, R5
|
|
BLO repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short:
|
|
CMPW $0x00000104, R9
|
|
BLO repeat_three_match_nolit_encodeBlockAsm_emit_copy_short
|
|
CMPW $0x00010100, R9
|
|
BLO repeat_four_match_nolit_encodeBlockAsm_emit_copy_short
|
|
CMPW $0x0100ffff, R9
|
|
BLO repeat_five_match_nolit_encodeBlockAsm_emit_copy_short
|
|
SUB $16842747, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0xfffb001d, R16
|
|
MOVW R16, (R1)
|
|
MOVD $0xff, R16
|
|
MOVB R16, 4(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP emit_repeat_again_match_nolit_encodeBlockAsm_emit_copy_short
|
|
|
|
repeat_five_match_nolit_encodeBlockAsm_emit_copy_short:
|
|
SUB $65536, R9, R9
|
|
MOVWU R9, R9
|
|
MOVWU R9, R5
|
|
MOVD $0x001d, R16
|
|
MOVH R16, (R1)
|
|
MOVH R9, 2(R1)
|
|
ASRW $0x10, R5, R5
|
|
MOVB R5, 4(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm
|
|
|
|
repeat_four_match_nolit_encodeBlockAsm_emit_copy_short:
|
|
SUB $256, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R9, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm
|
|
|
|
repeat_three_match_nolit_encodeBlockAsm_emit_copy_short:
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R9, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm
|
|
|
|
repeat_two_match_nolit_encodeBlockAsm_emit_copy_short:
|
|
LSLW $0x02, R9, R9
|
|
ORRW $0x01, R9, R9
|
|
MOVH R9, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm
|
|
|
|
repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short:
|
|
MOVD $0, R6
|
|
ADD R9<<2, R6, R9
|
|
ADD $1, R9, R9
|
|
MOVWU R9, R9
|
|
MOVB R5, 1(R1)
|
|
ASRW $0x08, R5, R5
|
|
LSLW $0x05, R5, R5
|
|
ORRW R5, R9, R9
|
|
MOVB R9, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm
|
|
|
|
two_byte_offset_short_match_nolit_encodeBlockAsm:
|
|
MOVWU R9, R6
|
|
LSLW $0x02, R6, R6
|
|
CMPW $0x0c, R9
|
|
BHS emit_copy_three_match_nolit_encodeBlockAsm
|
|
CMPW $0x00000800, R5
|
|
BHS emit_copy_three_match_nolit_encodeBlockAsm
|
|
SUB $15, R6, R6
|
|
MOVWU R6, R6
|
|
MOVB R5, 1(R1)
|
|
LSRW $0x08, R5, R5
|
|
LSLW $0x05, R5, R5
|
|
ORRW R5, R6, R6
|
|
MOVB R6, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm
|
|
|
|
emit_copy_three_match_nolit_encodeBlockAsm:
|
|
SUB $2, R6, R6
|
|
MOVWU R6, R6
|
|
MOVB R6, (R1)
|
|
MOVH R5, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
|
|
match_nolit_emitcopy_end_encodeBlockAsm:
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R2
|
|
BHS emit_remainder_encodeBlockAsm
|
|
ADD R2, R3, R15
|
|
MOVD -2(R15), R6
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R1
|
|
BLO match_nolit_dst_ok_encodeBlockAsm
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeBlockAsm:
|
|
MOVD $0x0000cf1bbcdcbf9b, R8
|
|
MOVD R6, R7
|
|
LSR $0x10, R6, R6
|
|
MOVD R6, R5
|
|
LSL $0x10, R7, R7
|
|
MUL R8, R7, R7
|
|
LSR $0x32, R7, R7
|
|
LSL $0x10, R5, R5
|
|
MUL R8, R5, R5
|
|
LSR $0x32, R5, R5
|
|
SUB $2, R2, R8
|
|
MOVWU R8, R8
|
|
ADD R5<<2, R0, R9
|
|
MOVWU (R9), R5
|
|
MOVW R8, (R0)(R7<<2)
|
|
MOVW R2, (R9)
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ match_nolit_loop_encodeBlockAsm
|
|
ADDW $1, R2, R2
|
|
JMP search_loop_encodeBlockAsm
|
|
|
|
emit_remainder_encodeBlockAsm:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R0, R0
|
|
ADD R0, R1, R0
|
|
ADD $5, R0, R0
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R0
|
|
BLO emit_remainder_ok_encodeBlockAsm
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeBlockAsm:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R2
|
|
CMPW R0, R2
|
|
BEQ emit_literal_done_emit_remainder_encodeBlockAsm
|
|
MOVWU R0, R5
|
|
MOVW R0, 20(RSP)
|
|
ADD R2, R3, R0
|
|
SUBW R2, R5, R5
|
|
SUB $1, R5, R2
|
|
MOVWU R2, R2
|
|
CMPW $0x3c, R2
|
|
BLO one_byte_emit_remainder_encodeBlockAsm
|
|
CMPW $0x00000100, R2
|
|
BLO two_bytes_emit_remainder_encodeBlockAsm
|
|
CMPW $0x00010000, R2
|
|
BLO three_bytes_emit_remainder_encodeBlockAsm
|
|
CMPW $0x01000000, R2
|
|
BLO four_bytes_emit_remainder_encodeBlockAsm
|
|
MOVD $0xfc, R16
|
|
MOVB R16, (R1)
|
|
MOVW R2, 1(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP memmove_long_emit_remainder_encodeBlockAsm
|
|
|
|
four_bytes_emit_remainder_encodeBlockAsm:
|
|
MOVWU R2, R3
|
|
LSRW $0x10, R3, R3
|
|
MOVD $0xf8, R16
|
|
MOVB R16, (R1)
|
|
MOVH R2, 1(R1)
|
|
MOVB R3, 3(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP memmove_long_emit_remainder_encodeBlockAsm
|
|
|
|
three_bytes_emit_remainder_encodeBlockAsm:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R2, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_emit_remainder_encodeBlockAsm
|
|
|
|
two_bytes_emit_remainder_encodeBlockAsm:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R2, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R2
|
|
BLO memmove_emit_remainder_encodeBlockAsm
|
|
JMP memmove_long_emit_remainder_encodeBlockAsm
|
|
|
|
one_byte_emit_remainder_encodeBlockAsm:
|
|
LSLW $0x02, R2, R16
|
|
BFI $0, R16, $8, R2
|
|
MOVB R2, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_emit_remainder_encodeBlockAsm:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveShort
|
|
CMP $0x03, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_1or2
|
|
BEQ emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_3
|
|
CMP $0x08, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_4through7
|
|
CMP $0x10, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_8through16
|
|
CMP $0x20, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_1or2:
|
|
MOVBU (R0), R16
|
|
BFI $0, R16, $8, R5
|
|
ADD R3, R0, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVB R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVB R0, -1(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_3:
|
|
MOVHU (R0), R16
|
|
BFI $0, R16, $16, R5
|
|
MOVBU 2(R0), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVH R5, (R1)
|
|
MOVB R0, 2(R1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_4through7:
|
|
MOVWU (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVWU -4(R15), R0
|
|
MOVW R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVW R0, -4(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_8through16:
|
|
MOVD (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVD -8(R15), R0
|
|
MOVD R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVD R0, -8(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_17through32:
|
|
FMOVQ (R0), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_33through64:
|
|
FMOVQ (R0), F0
|
|
FMOVQ 16(R0), F1
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_emit_remainder_encodeBlockAsm:
|
|
MOVD R2, R1
|
|
JMP emit_literal_done_emit_remainder_encodeBlockAsm
|
|
|
|
memmove_long_emit_remainder_encodeBlockAsm:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveLong
|
|
MOVD R0, R5
|
|
MOVD R1, R6
|
|
MOVD R3, R7
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeBlockAsmlarge_big_loop_back:
|
|
FMOVQ (R5), F0
|
|
FMOVQ 16(R5), F1
|
|
FMOVQ F0, (R6)
|
|
FMOVQ F1, 16(R6)
|
|
ADD $0x20, R5, R5
|
|
ADD $0x20, R6, R6
|
|
SUB $0x20, R7, R7
|
|
CMP $0x20, R7
|
|
BHS emit_lit_memmove_long_emit_remainder_encodeBlockAsmlarge_big_loop_back
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R3, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R2, R1
|
|
|
|
emit_literal_done_emit_remainder_encodeBlockAsm:
|
|
MOVD dst_base+0(FP), R0
|
|
SUB R0, R1, R1
|
|
MOVD R1, ret+56(FP)
|
|
RET
|
|
|
|
// func encodeBlockAsm4MB(dst []byte, src []byte, tmp *[65536]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeBlockAsm4MB(SB), $24-64
|
|
MOVD tmp+48(FP), R0
|
|
MOVD dst_base+0(FP), R1
|
|
MOVD $0x00000200, R2
|
|
MOVD R0, R3
|
|
VEOR V0.B16, V0.B16, V0.B16
|
|
|
|
zero_loop_encodeBlockAsm4MB:
|
|
FMOVQ F0, (R3)
|
|
FMOVQ F0, 16(R3)
|
|
FMOVQ F0, 32(R3)
|
|
FMOVQ F0, 48(R3)
|
|
FMOVQ F0, 64(R3)
|
|
FMOVQ F0, 80(R3)
|
|
FMOVQ F0, 96(R3)
|
|
FMOVQ F0, 112(R3)
|
|
ADD $0x80, R3, R3
|
|
SUBS $1, R2, R2
|
|
BNE zero_loop_encodeBlockAsm4MB
|
|
MOVD $0x00000000, R16
|
|
MOVW R16, 20(RSP)
|
|
MOVD src_len+32(FP), R2
|
|
SUB $9, R2, R3
|
|
SUB $8, R2, R5
|
|
MOVW R5, 16(RSP)
|
|
LSR $0x05, R2, R2
|
|
SUBW R2, R3, R3
|
|
ADD R3, R1, R3
|
|
MOVD R3, 8(RSP)
|
|
MOVD $0x00000001, R2
|
|
MOVW R2, 24(RSP)
|
|
MOVD src_base+24(FP), R3
|
|
|
|
search_loop_encodeBlockAsm4MB:
|
|
MOVWU R2, R5
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R5, R5
|
|
LSRW $0x06, R5, R5
|
|
ADD R5, R2, R5
|
|
ADD $4, R5, R5
|
|
MOVWU R5, R5
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R5
|
|
BHS emit_remainder_encodeBlockAsm4MB
|
|
MOVD (R3)(R2), R6
|
|
MOVW R5, 28(RSP)
|
|
MOVD $0x0000cf1bbcdcbf9b, R8
|
|
MOVD R6, R9
|
|
MOVD R6, R10
|
|
LSR $0x08, R10, R10
|
|
LSL $0x10, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x32, R9, R9
|
|
LSL $0x10, R10, R10
|
|
MUL R8, R10, R10
|
|
LSR $0x32, R10, R10
|
|
MOVWU (R0)(R9<<2), R5
|
|
MOVWU (R0)(R10<<2), R7
|
|
MOVW R2, (R0)(R9<<2)
|
|
ADD $1, R2, R9
|
|
MOVWU R9, R9
|
|
MOVW R9, (R0)(R10<<2)
|
|
MOVD R6, R9
|
|
LSR $0x10, R9, R9
|
|
LSL $0x10, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x32, R9, R9
|
|
MOVWU R2, R8
|
|
MOVWU 24(RSP), R16
|
|
SUBW R16, R8, R8
|
|
ADD R8, R3, R15
|
|
MOVWU 1(R15), R10
|
|
MOVD R6, R8
|
|
LSR $0x08, R8, R8
|
|
CMPW R10, R8
|
|
BNE no_repeat_found_encodeBlockAsm4MB
|
|
ADD $1, R2, R6
|
|
MOVWU R6, R6
|
|
MOVWU 20(RSP), R7
|
|
MOVWU R6, R5
|
|
MOVWU 24(RSP), R16
|
|
SUBSW R16, R5, R5
|
|
BEQ repeat_extend_back_end_encodeBlockAsm4MB
|
|
|
|
repeat_extend_back_loop_encodeBlockAsm4MB:
|
|
CMPW R7, R6
|
|
BLS repeat_extend_back_end_encodeBlockAsm4MB
|
|
ADD R5, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R8
|
|
ADD R6, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R9
|
|
AND $0xff, R9, R16
|
|
AND $0xff, R8, R15
|
|
CMP R16, R15
|
|
BNE repeat_extend_back_end_encodeBlockAsm4MB
|
|
SUB $1, R6, R6
|
|
MOVWU R6, R6
|
|
SUBSW $1, R5, R5
|
|
BNE repeat_extend_back_loop_encodeBlockAsm4MB
|
|
|
|
repeat_extend_back_end_encodeBlockAsm4MB:
|
|
MOVWU R6, R5
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R5, R5
|
|
ADD R5, R1, R5
|
|
ADD $4, R5, R5
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R5
|
|
BLO repeat_dst_size_check_encodeBlockAsm4MB
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
repeat_dst_size_check_encodeBlockAsm4MB:
|
|
MOVWU 20(RSP), R5
|
|
CMPW R6, R5
|
|
BEQ emit_literal_done_repeat_emit_encodeBlockAsm4MB
|
|
MOVWU R6, R8
|
|
MOVW R6, 20(RSP)
|
|
ADD R5, R3, R9
|
|
SUBW R5, R8, R8
|
|
SUB $1, R8, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x3c, R5
|
|
BLO one_byte_repeat_emit_encodeBlockAsm4MB
|
|
CMPW $0x00000100, R5
|
|
BLO two_bytes_repeat_emit_encodeBlockAsm4MB
|
|
CMPW $0x00010000, R5
|
|
BLO three_bytes_repeat_emit_encodeBlockAsm4MB
|
|
MOVWU R5, R10
|
|
LSRW $0x10, R10, R10
|
|
MOVD $0xf8, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
MOVB R10, 3(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP memmove_long_repeat_emit_encodeBlockAsm4MB
|
|
|
|
three_bytes_repeat_emit_encodeBlockAsm4MB:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_repeat_emit_encodeBlockAsm4MB
|
|
|
|
two_bytes_repeat_emit_encodeBlockAsm4MB:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R5, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R5
|
|
BLO memmove_repeat_emit_encodeBlockAsm4MB
|
|
JMP memmove_long_repeat_emit_encodeBlockAsm4MB
|
|
|
|
one_byte_repeat_emit_encodeBlockAsm4MB:
|
|
LSLW $0x02, R5, R16
|
|
BFI $0, R16, $8, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_repeat_emit_encodeBlockAsm4MB:
|
|
ADD R8, R1, R5
|
|
|
|
// genMemMoveShort
|
|
CMP $0x08, R8
|
|
BLS emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_8
|
|
CMP $0x10, R8
|
|
BLS emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_8through16
|
|
CMP $0x20, R8
|
|
BLS emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_17through32
|
|
JMP emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_33through64
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_8:
|
|
MOVD (R9), R10
|
|
MOVD R10, (R1)
|
|
JMP memmove_end_copy_repeat_emit_encodeBlockAsm4MB
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_8through16:
|
|
MOVD (R9), R10
|
|
ADD R8, R9, R15
|
|
MOVD -8(R15), R9
|
|
MOVD R10, (R1)
|
|
ADD R8, R1, R15
|
|
MOVD R9, -8(R15)
|
|
JMP memmove_end_copy_repeat_emit_encodeBlockAsm4MB
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_17through32:
|
|
FMOVQ (R9), F0
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_repeat_emit_encodeBlockAsm4MB
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_33through64:
|
|
FMOVQ (R9), F0
|
|
FMOVQ 16(R9), F1
|
|
ADD R8, R9, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_repeat_emit_encodeBlockAsm4MB:
|
|
MOVD R5, R1
|
|
JMP emit_literal_done_repeat_emit_encodeBlockAsm4MB
|
|
|
|
memmove_long_repeat_emit_encodeBlockAsm4MB:
|
|
ADD R8, R1, R5
|
|
|
|
// genMemMoveLong
|
|
MOVD R9, R10
|
|
MOVD R1, R11
|
|
MOVD R8, R12
|
|
|
|
emit_lit_memmove_long_repeat_emit_encodeBlockAsm4MBlarge_big_loop_back:
|
|
FMOVQ (R10), F0
|
|
FMOVQ 16(R10), F1
|
|
FMOVQ F0, (R11)
|
|
FMOVQ F1, 16(R11)
|
|
ADD $0x20, R10, R10
|
|
ADD $0x20, R11, R11
|
|
SUB $0x20, R12, R12
|
|
CMP $0x20, R12
|
|
BHS emit_lit_memmove_long_repeat_emit_encodeBlockAsm4MBlarge_big_loop_back
|
|
ADD R8, R9, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R8, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R5, R1
|
|
|
|
emit_literal_done_repeat_emit_encodeBlockAsm4MB:
|
|
ADDW $0x05, R2, R2
|
|
MOVWU R2, R5
|
|
MOVWU 24(RSP), R16
|
|
SUBW R16, R5, R5
|
|
MOVD src_len+32(FP), R8
|
|
SUBW R2, R8, R8
|
|
ADD R2, R3, R9
|
|
ADD R5, R3, R5
|
|
|
|
// matchLen
|
|
MOVD $0, R11
|
|
|
|
matchlen_loopback_16_repeat_extend_encodeBlockAsm4MB:
|
|
CMPW $0x10, R8
|
|
BLO matchlen_match8_repeat_extend_encodeBlockAsm4MB
|
|
MOVD (R9)(R11), R10
|
|
ADD R11, R9, R15
|
|
MOVD 8(R15), R12
|
|
MOVD (R5)(R11), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_8_repeat_extend_encodeBlockAsm4MB
|
|
ADD R11, R5, R15
|
|
MOVD 8(R15), R16
|
|
EOR R16, R12, R12
|
|
TST R12, R12
|
|
BNE matchlen_bsf_16repeat_extend_encodeBlockAsm4MB
|
|
SUB $16, R8, R8
|
|
MOVWU R8, R8
|
|
ADD $16, R11, R11
|
|
MOVWU R11, R11
|
|
JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm4MB
|
|
|
|
matchlen_bsf_16repeat_extend_encodeBlockAsm4MB:
|
|
RBIT R12, R12
|
|
CLZ R12, R12
|
|
ASR $0x03, R12, R12
|
|
ADD R12, R11, R11
|
|
ADD $8, R11, R11
|
|
MOVWU R11, R11
|
|
JMP repeat_extend_forward_end_encodeBlockAsm4MB
|
|
|
|
matchlen_match8_repeat_extend_encodeBlockAsm4MB:
|
|
CMPW $0x08, R8
|
|
BLO matchlen_match4_repeat_extend_encodeBlockAsm4MB
|
|
MOVD (R9)(R11), R10
|
|
MOVD (R5)(R11), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_8_repeat_extend_encodeBlockAsm4MB
|
|
SUB $8, R8, R8
|
|
MOVWU R8, R8
|
|
ADD $8, R11, R11
|
|
MOVWU R11, R11
|
|
JMP matchlen_match4_repeat_extend_encodeBlockAsm4MB
|
|
|
|
matchlen_bsf_8_repeat_extend_encodeBlockAsm4MB:
|
|
RBIT R10, R10
|
|
CLZ R10, R10
|
|
ASR $0x03, R10, R10
|
|
ADD R10, R11, R11
|
|
MOVWU R11, R11
|
|
JMP repeat_extend_forward_end_encodeBlockAsm4MB
|
|
|
|
matchlen_match4_repeat_extend_encodeBlockAsm4MB:
|
|
CMPW $0x04, R8
|
|
BLO matchlen_match2_repeat_extend_encodeBlockAsm4MB
|
|
MOVWU (R9)(R11), R10
|
|
MOVWU (R5)(R11), R16
|
|
CMPW R10, R16
|
|
BNE matchlen_match2_repeat_extend_encodeBlockAsm4MB
|
|
SUB $4, R8, R8
|
|
MOVWU R8, R8
|
|
ADD $4, R11, R11
|
|
MOVWU R11, R11
|
|
|
|
matchlen_match2_repeat_extend_encodeBlockAsm4MB:
|
|
CMPW $0x01, R8
|
|
BEQ matchlen_match1_repeat_extend_encodeBlockAsm4MB
|
|
BLO repeat_extend_forward_end_encodeBlockAsm4MB
|
|
MOVHU (R9)(R11), R16
|
|
BFI $0, R16, $16, R10
|
|
ADD R11, R5, R15
|
|
MOVHU (R15), R15
|
|
AND $0xffff, R10, R16
|
|
CMP R16, R15
|
|
BNE matchlen_match1_repeat_extend_encodeBlockAsm4MB
|
|
ADD $2, R11, R11
|
|
MOVWU R11, R11
|
|
SUBSW $0x02, R8, R8
|
|
BEQ repeat_extend_forward_end_encodeBlockAsm4MB
|
|
|
|
matchlen_match1_repeat_extend_encodeBlockAsm4MB:
|
|
MOVBU (R9)(R11), R16
|
|
BFI $0, R16, $8, R10
|
|
ADD R11, R5, R15
|
|
MOVBU (R15), R15
|
|
AND $0xff, R10, R16
|
|
CMP R16, R15
|
|
BNE repeat_extend_forward_end_encodeBlockAsm4MB
|
|
ADD $1, R11, R11
|
|
MOVWU R11, R11
|
|
|
|
repeat_extend_forward_end_encodeBlockAsm4MB:
|
|
ADDW R11, R2, R2
|
|
MOVWU R2, R5
|
|
SUBW R6, R5, R5
|
|
MOVWU 24(RSP), R6
|
|
TSTW R7, R7
|
|
BEQ repeat_as_copy_encodeBlockAsm4MB
|
|
|
|
// emitRepeat
|
|
MOVWU R5, R7
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x08, R7
|
|
BLS repeat_two_match_repeat_encodeBlockAsm4MB
|
|
CMPW $0x0c, R7
|
|
BHS cant_repeat_two_offset_match_repeat_encodeBlockAsm4MB
|
|
CMPW $0x00000800, R6
|
|
BLO repeat_two_offset_match_repeat_encodeBlockAsm4MB
|
|
|
|
cant_repeat_two_offset_match_repeat_encodeBlockAsm4MB:
|
|
CMPW $0x00000104, R5
|
|
BLO repeat_three_match_repeat_encodeBlockAsm4MB
|
|
CMPW $0x00010100, R5
|
|
BLO repeat_four_match_repeat_encodeBlockAsm4MB
|
|
SUB $65536, R5, R5
|
|
MOVWU R5, R5
|
|
MOVWU R5, R6
|
|
MOVD $0x001d, R16
|
|
MOVH R16, (R1)
|
|
MOVH R5, 2(R1)
|
|
ASRW $0x10, R6, R6
|
|
MOVB R6, 4(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
repeat_four_match_repeat_encodeBlockAsm4MB:
|
|
SUB $256, R5, R5
|
|
MOVWU R5, R5
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R5, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
repeat_three_match_repeat_encodeBlockAsm4MB:
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R5, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
repeat_two_match_repeat_encodeBlockAsm4MB:
|
|
LSLW $0x02, R5, R5
|
|
ORRW $0x01, R5, R5
|
|
MOVH R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
repeat_two_offset_match_repeat_encodeBlockAsm4MB:
|
|
MOVD $0, R7
|
|
ADD R5<<2, R7, R5
|
|
ADD $1, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R6, 1(R1)
|
|
ASRW $0x08, R6, R6
|
|
LSLW $0x05, R6, R6
|
|
ORRW R6, R5, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
repeat_as_copy_encodeBlockAsm4MB:
|
|
// emitCopy
|
|
CMPW $0x00010000, R6
|
|
BLO two_byte_offset_repeat_as_copy_encodeBlockAsm4MB
|
|
CMPW $0x40, R5
|
|
BLS four_bytes_remain_repeat_as_copy_encodeBlockAsm4MB
|
|
MOVD $0xff, R16
|
|
MOVB R16, (R1)
|
|
MOVW R6, 1(R1)
|
|
SUB $64, R5, R5
|
|
MOVWU R5, R5
|
|
ADD $0x05, R1, R1
|
|
CMPW $0x04, R5
|
|
BLO four_bytes_remain_repeat_as_copy_encodeBlockAsm4MB
|
|
|
|
// emitRepeat
|
|
MOVWU R5, R7
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x08, R7
|
|
BLS repeat_two_repeat_as_copy_encodeBlockAsm4MB_emit_copy
|
|
CMPW $0x0c, R7
|
|
BHS cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy
|
|
CMPW $0x00000800, R6
|
|
BLO repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy
|
|
|
|
cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy:
|
|
CMPW $0x00000104, R5
|
|
BLO repeat_three_repeat_as_copy_encodeBlockAsm4MB_emit_copy
|
|
CMPW $0x00010100, R5
|
|
BLO repeat_four_repeat_as_copy_encodeBlockAsm4MB_emit_copy
|
|
SUB $65536, R5, R5
|
|
MOVWU R5, R5
|
|
MOVWU R5, R6
|
|
MOVD $0x001d, R16
|
|
MOVH R16, (R1)
|
|
MOVH R5, 2(R1)
|
|
ASRW $0x10, R6, R6
|
|
MOVB R6, 4(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
repeat_four_repeat_as_copy_encodeBlockAsm4MB_emit_copy:
|
|
SUB $256, R5, R5
|
|
MOVWU R5, R5
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R5, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
repeat_three_repeat_as_copy_encodeBlockAsm4MB_emit_copy:
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R5, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
repeat_two_repeat_as_copy_encodeBlockAsm4MB_emit_copy:
|
|
LSLW $0x02, R5, R5
|
|
ORRW $0x01, R5, R5
|
|
MOVH R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy:
|
|
MOVD $0, R7
|
|
ADD R5<<2, R7, R5
|
|
ADD $1, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R6, 1(R1)
|
|
ASRW $0x08, R6, R6
|
|
LSLW $0x05, R6, R6
|
|
ORRW R6, R5, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
four_bytes_remain_repeat_as_copy_encodeBlockAsm4MB:
|
|
TSTW R5, R5
|
|
BEQ repeat_end_emit_encodeBlockAsm4MB
|
|
MOVD $0, R7
|
|
ADD R5<<2, R7, R5
|
|
SUB $1, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R5, (R1)
|
|
MOVW R6, 1(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
two_byte_offset_repeat_as_copy_encodeBlockAsm4MB:
|
|
CMPW $0x40, R5
|
|
BLS two_byte_offset_short_repeat_as_copy_encodeBlockAsm4MB
|
|
CMPW $0x00000800, R6
|
|
BHS long_offset_short_repeat_as_copy_encodeBlockAsm4MB
|
|
MOVD $0x00000001, R7
|
|
ADD $16, R7, R7
|
|
MOVWU R7, R7
|
|
MOVB R6, 1(R1)
|
|
LSRW $0x08, R6, R6
|
|
LSLW $0x05, R6, R6
|
|
ORRW R6, R7, R7
|
|
MOVB R7, (R1)
|
|
ADD $0x02, R1, R1
|
|
SUBW $0x08, R5, R5
|
|
|
|
// emitRepeat
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
JMP cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b
|
|
MOVWU R5, R7
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x08, R7
|
|
BLS repeat_two_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b
|
|
CMPW $0x0c, R7
|
|
BHS cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b
|
|
CMPW $0x00000800, R6
|
|
BLO repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b
|
|
|
|
cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b:
|
|
CMPW $0x00000104, R5
|
|
BLO repeat_three_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b
|
|
CMPW $0x00010100, R5
|
|
BLO repeat_four_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b
|
|
SUB $65536, R5, R5
|
|
MOVWU R5, R5
|
|
MOVWU R5, R6
|
|
MOVD $0x001d, R16
|
|
MOVH R16, (R1)
|
|
MOVH R5, 2(R1)
|
|
ASRW $0x10, R6, R6
|
|
MOVB R6, 4(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
repeat_four_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b:
|
|
SUB $256, R5, R5
|
|
MOVWU R5, R5
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R5, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
repeat_three_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b:
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R5, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
repeat_two_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b:
|
|
LSLW $0x02, R5, R5
|
|
ORRW $0x01, R5, R5
|
|
MOVH R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b:
|
|
MOVD $0, R7
|
|
ADD R5<<2, R7, R5
|
|
ADD $1, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R6, 1(R1)
|
|
ASRW $0x08, R6, R6
|
|
LSLW $0x05, R6, R6
|
|
ORRW R6, R5, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
long_offset_short_repeat_as_copy_encodeBlockAsm4MB:
|
|
MOVD $0xee, R16
|
|
MOVB R16, (R1)
|
|
MOVH R6, 1(R1)
|
|
SUB $60, R5, R5
|
|
MOVWU R5, R5
|
|
ADD $0x03, R1, R1
|
|
|
|
// emitRepeat
|
|
MOVWU R5, R7
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x08, R7
|
|
BLS repeat_two_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short
|
|
CMPW $0x0c, R7
|
|
BHS cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short
|
|
CMPW $0x00000800, R6
|
|
BLO repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short
|
|
|
|
cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short:
|
|
CMPW $0x00000104, R5
|
|
BLO repeat_three_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short
|
|
CMPW $0x00010100, R5
|
|
BLO repeat_four_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short
|
|
SUB $65536, R5, R5
|
|
MOVWU R5, R5
|
|
MOVWU R5, R6
|
|
MOVD $0x001d, R16
|
|
MOVH R16, (R1)
|
|
MOVH R5, 2(R1)
|
|
ASRW $0x10, R6, R6
|
|
MOVB R6, 4(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
repeat_four_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short:
|
|
SUB $256, R5, R5
|
|
MOVWU R5, R5
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R5, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
repeat_three_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short:
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R5, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
repeat_two_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short:
|
|
LSLW $0x02, R5, R5
|
|
ORRW $0x01, R5, R5
|
|
MOVH R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short:
|
|
MOVD $0, R7
|
|
ADD R5<<2, R7, R5
|
|
ADD $1, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R6, 1(R1)
|
|
ASRW $0x08, R6, R6
|
|
LSLW $0x05, R6, R6
|
|
ORRW R6, R5, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
two_byte_offset_short_repeat_as_copy_encodeBlockAsm4MB:
|
|
MOVWU R5, R7
|
|
LSLW $0x02, R7, R7
|
|
CMPW $0x0c, R5
|
|
BHS emit_copy_three_repeat_as_copy_encodeBlockAsm4MB
|
|
CMPW $0x00000800, R6
|
|
BHS emit_copy_three_repeat_as_copy_encodeBlockAsm4MB
|
|
SUB $15, R7, R7
|
|
MOVWU R7, R7
|
|
MOVB R6, 1(R1)
|
|
LSRW $0x08, R6, R6
|
|
LSLW $0x05, R6, R6
|
|
ORRW R6, R7, R7
|
|
MOVB R7, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm4MB
|
|
|
|
emit_copy_three_repeat_as_copy_encodeBlockAsm4MB:
|
|
SUB $2, R7, R7
|
|
MOVWU R7, R7
|
|
MOVB R7, (R1)
|
|
MOVH R6, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
|
|
repeat_end_emit_encodeBlockAsm4MB:
|
|
MOVW R2, 20(RSP)
|
|
JMP search_loop_encodeBlockAsm4MB
|
|
|
|
no_repeat_found_encodeBlockAsm4MB:
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ candidate_match_encodeBlockAsm4MB
|
|
LSR $0x08, R6, R6
|
|
MOVWU (R0)(R9<<2), R5
|
|
ADD $2, R2, R8
|
|
MOVWU R8, R8
|
|
MOVWU (R3)(R7), R16
|
|
CMPW R6, R16
|
|
BEQ candidate2_match_encodeBlockAsm4MB
|
|
MOVW R8, (R0)(R9<<2)
|
|
LSR $0x08, R6, R6
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ candidate3_match_encodeBlockAsm4MB
|
|
MOVWU 28(RSP), R2
|
|
JMP search_loop_encodeBlockAsm4MB
|
|
|
|
candidate3_match_encodeBlockAsm4MB:
|
|
ADDW $0x02, R2, R2
|
|
JMP candidate_match_encodeBlockAsm4MB
|
|
|
|
candidate2_match_encodeBlockAsm4MB:
|
|
MOVW R8, (R0)(R9<<2)
|
|
ADDW $1, R2, R2
|
|
MOVWU R7, R5
|
|
|
|
candidate_match_encodeBlockAsm4MB:
|
|
MOVWU 20(RSP), R6
|
|
TSTW R5, R5
|
|
BEQ match_extend_back_end_encodeBlockAsm4MB
|
|
|
|
match_extend_back_loop_encodeBlockAsm4MB:
|
|
CMPW R6, R2
|
|
BLS match_extend_back_end_encodeBlockAsm4MB
|
|
ADD R5, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R7
|
|
ADD R2, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R8
|
|
AND $0xff, R8, R16
|
|
AND $0xff, R7, R15
|
|
CMP R16, R15
|
|
BNE match_extend_back_end_encodeBlockAsm4MB
|
|
SUB $1, R2, R2
|
|
MOVWU R2, R2
|
|
SUBSW $1, R5, R5
|
|
BEQ match_extend_back_end_encodeBlockAsm4MB
|
|
JMP match_extend_back_loop_encodeBlockAsm4MB
|
|
|
|
match_extend_back_end_encodeBlockAsm4MB:
|
|
MOVWU R2, R6
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R6, R6
|
|
ADD R6, R1, R6
|
|
ADD $4, R6, R6
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R6
|
|
BLO match_dst_size_check_encodeBlockAsm4MB
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeBlockAsm4MB:
|
|
MOVWU R2, R6
|
|
MOVWU 20(RSP), R7
|
|
CMPW R6, R7
|
|
BEQ emit_literal_done_match_emit_encodeBlockAsm4MB
|
|
MOVWU R6, R8
|
|
MOVW R6, 20(RSP)
|
|
ADD R7, R3, R6
|
|
SUBW R7, R8, R8
|
|
SUB $1, R8, R7
|
|
MOVWU R7, R7
|
|
CMPW $0x3c, R7
|
|
BLO one_byte_match_emit_encodeBlockAsm4MB
|
|
CMPW $0x00000100, R7
|
|
BLO two_bytes_match_emit_encodeBlockAsm4MB
|
|
CMPW $0x00010000, R7
|
|
BLO three_bytes_match_emit_encodeBlockAsm4MB
|
|
MOVWU R7, R9
|
|
LSRW $0x10, R9, R9
|
|
MOVD $0xf8, R16
|
|
MOVB R16, (R1)
|
|
MOVH R7, 1(R1)
|
|
MOVB R9, 3(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP memmove_long_match_emit_encodeBlockAsm4MB
|
|
|
|
three_bytes_match_emit_encodeBlockAsm4MB:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R7, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_match_emit_encodeBlockAsm4MB
|
|
|
|
two_bytes_match_emit_encodeBlockAsm4MB:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R7, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R7
|
|
BLO memmove_match_emit_encodeBlockAsm4MB
|
|
JMP memmove_long_match_emit_encodeBlockAsm4MB
|
|
|
|
one_byte_match_emit_encodeBlockAsm4MB:
|
|
LSLW $0x02, R7, R16
|
|
BFI $0, R16, $8, R7
|
|
MOVB R7, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_match_emit_encodeBlockAsm4MB:
|
|
ADD R8, R1, R7
|
|
|
|
// genMemMoveShort
|
|
CMP $0x08, R8
|
|
BLS emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_8
|
|
CMP $0x10, R8
|
|
BLS emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_8through16
|
|
CMP $0x20, R8
|
|
BLS emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_8:
|
|
MOVD (R6), R9
|
|
MOVD R9, (R1)
|
|
JMP memmove_end_copy_match_emit_encodeBlockAsm4MB
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_8through16:
|
|
MOVD (R6), R9
|
|
ADD R8, R6, R15
|
|
MOVD -8(R15), R6
|
|
MOVD R9, (R1)
|
|
ADD R8, R1, R15
|
|
MOVD R6, -8(R15)
|
|
JMP memmove_end_copy_match_emit_encodeBlockAsm4MB
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_17through32:
|
|
FMOVQ (R6), F0
|
|
ADD R8, R6, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_match_emit_encodeBlockAsm4MB
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_33through64:
|
|
FMOVQ (R6), F0
|
|
FMOVQ 16(R6), F1
|
|
ADD R8, R6, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R8, R6, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_match_emit_encodeBlockAsm4MB:
|
|
MOVD R7, R1
|
|
JMP emit_literal_done_match_emit_encodeBlockAsm4MB
|
|
|
|
memmove_long_match_emit_encodeBlockAsm4MB:
|
|
ADD R8, R1, R7
|
|
|
|
// genMemMoveLong
|
|
MOVD R6, R9
|
|
MOVD R1, R10
|
|
MOVD R8, R11
|
|
|
|
emit_lit_memmove_long_match_emit_encodeBlockAsm4MBlarge_big_loop_back:
|
|
FMOVQ (R9), F0
|
|
FMOVQ 16(R9), F1
|
|
FMOVQ F0, (R10)
|
|
FMOVQ F1, 16(R10)
|
|
ADD $0x20, R9, R9
|
|
ADD $0x20, R10, R10
|
|
SUB $0x20, R11, R11
|
|
CMP $0x20, R11
|
|
BHS emit_lit_memmove_long_match_emit_encodeBlockAsm4MBlarge_big_loop_back
|
|
ADD R8, R6, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R8, R6, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R8, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R7, R1
|
|
|
|
emit_literal_done_match_emit_encodeBlockAsm4MB:
|
|
match_nolit_loop_encodeBlockAsm4MB:
|
|
MOVWU R2, R6
|
|
SUBW R5, R6, R6
|
|
MOVW R6, 24(RSP)
|
|
ADDW $0x04, R2, R2
|
|
ADDW $0x04, R5, R5
|
|
MOVD src_len+32(FP), R6
|
|
SUBW R2, R6, R6
|
|
ADD R2, R3, R7
|
|
ADD R5, R3, R5
|
|
|
|
// matchLen
|
|
MOVD $0, R9
|
|
|
|
matchlen_loopback_16_match_nolit_encodeBlockAsm4MB:
|
|
CMPW $0x10, R6
|
|
BLO matchlen_match8_match_nolit_encodeBlockAsm4MB
|
|
MOVD (R7)(R9), R8
|
|
ADD R9, R7, R15
|
|
MOVD 8(R15), R10
|
|
MOVD (R5)(R9), R16
|
|
EOR R16, R8, R8
|
|
TST R8, R8
|
|
BNE matchlen_bsf_8_match_nolit_encodeBlockAsm4MB
|
|
ADD R9, R5, R15
|
|
MOVD 8(R15), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_16match_nolit_encodeBlockAsm4MB
|
|
SUB $16, R6, R6
|
|
MOVWU R6, R6
|
|
ADD $16, R9, R9
|
|
MOVWU R9, R9
|
|
JMP matchlen_loopback_16_match_nolit_encodeBlockAsm4MB
|
|
|
|
matchlen_bsf_16match_nolit_encodeBlockAsm4MB:
|
|
RBIT R10, R10
|
|
CLZ R10, R10
|
|
ASR $0x03, R10, R10
|
|
ADD R10, R9, R9
|
|
ADD $8, R9, R9
|
|
MOVWU R9, R9
|
|
JMP match_nolit_end_encodeBlockAsm4MB
|
|
|
|
matchlen_match8_match_nolit_encodeBlockAsm4MB:
|
|
CMPW $0x08, R6
|
|
BLO matchlen_match4_match_nolit_encodeBlockAsm4MB
|
|
MOVD (R7)(R9), R8
|
|
MOVD (R5)(R9), R16
|
|
EOR R16, R8, R8
|
|
TST R8, R8
|
|
BNE matchlen_bsf_8_match_nolit_encodeBlockAsm4MB
|
|
SUB $8, R6, R6
|
|
MOVWU R6, R6
|
|
ADD $8, R9, R9
|
|
MOVWU R9, R9
|
|
JMP matchlen_match4_match_nolit_encodeBlockAsm4MB
|
|
|
|
matchlen_bsf_8_match_nolit_encodeBlockAsm4MB:
|
|
RBIT R8, R8
|
|
CLZ R8, R8
|
|
ASR $0x03, R8, R8
|
|
ADD R8, R9, R9
|
|
MOVWU R9, R9
|
|
JMP match_nolit_end_encodeBlockAsm4MB
|
|
|
|
matchlen_match4_match_nolit_encodeBlockAsm4MB:
|
|
CMPW $0x04, R6
|
|
BLO matchlen_match2_match_nolit_encodeBlockAsm4MB
|
|
MOVWU (R7)(R9), R8
|
|
MOVWU (R5)(R9), R16
|
|
CMPW R8, R16
|
|
BNE matchlen_match2_match_nolit_encodeBlockAsm4MB
|
|
SUB $4, R6, R6
|
|
MOVWU R6, R6
|
|
ADD $4, R9, R9
|
|
MOVWU R9, R9
|
|
|
|
matchlen_match2_match_nolit_encodeBlockAsm4MB:
|
|
CMPW $0x01, R6
|
|
BEQ matchlen_match1_match_nolit_encodeBlockAsm4MB
|
|
BLO match_nolit_end_encodeBlockAsm4MB
|
|
MOVHU (R7)(R9), R16
|
|
BFI $0, R16, $16, R8
|
|
ADD R9, R5, R15
|
|
MOVHU (R15), R15
|
|
AND $0xffff, R8, R16
|
|
CMP R16, R15
|
|
BNE matchlen_match1_match_nolit_encodeBlockAsm4MB
|
|
ADD $2, R9, R9
|
|
MOVWU R9, R9
|
|
SUBSW $0x02, R6, R6
|
|
BEQ match_nolit_end_encodeBlockAsm4MB
|
|
|
|
matchlen_match1_match_nolit_encodeBlockAsm4MB:
|
|
MOVBU (R7)(R9), R16
|
|
BFI $0, R16, $8, R8
|
|
ADD R9, R5, R15
|
|
MOVBU (R15), R15
|
|
AND $0xff, R8, R16
|
|
CMP R16, R15
|
|
BNE match_nolit_end_encodeBlockAsm4MB
|
|
ADD $1, R9, R9
|
|
MOVWU R9, R9
|
|
|
|
match_nolit_end_encodeBlockAsm4MB:
|
|
ADDW R9, R2, R2
|
|
MOVWU 24(RSP), R5
|
|
ADDW $0x04, R9, R9
|
|
MOVW R2, 20(RSP)
|
|
|
|
// emitCopy
|
|
CMPW $0x00010000, R5
|
|
BLO two_byte_offset_match_nolit_encodeBlockAsm4MB
|
|
CMPW $0x40, R9
|
|
BLS four_bytes_remain_match_nolit_encodeBlockAsm4MB
|
|
MOVD $0xff, R16
|
|
MOVB R16, (R1)
|
|
MOVW R5, 1(R1)
|
|
SUB $64, R9, R9
|
|
MOVWU R9, R9
|
|
ADD $0x05, R1, R1
|
|
CMPW $0x04, R9
|
|
BLO four_bytes_remain_match_nolit_encodeBlockAsm4MB
|
|
|
|
// emitRepeat
|
|
MOVWU R9, R6
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
CMPW $0x08, R6
|
|
BLS repeat_two_match_nolit_encodeBlockAsm4MB_emit_copy
|
|
CMPW $0x0c, R6
|
|
BHS cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy
|
|
CMPW $0x00000800, R5
|
|
BLO repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy:
|
|
CMPW $0x00000104, R9
|
|
BLO repeat_three_match_nolit_encodeBlockAsm4MB_emit_copy
|
|
CMPW $0x00010100, R9
|
|
BLO repeat_four_match_nolit_encodeBlockAsm4MB_emit_copy
|
|
SUB $65536, R9, R9
|
|
MOVWU R9, R9
|
|
MOVWU R9, R5
|
|
MOVD $0x001d, R16
|
|
MOVH R16, (R1)
|
|
MOVH R9, 2(R1)
|
|
ASRW $0x10, R5, R5
|
|
MOVB R5, 4(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm4MB
|
|
|
|
repeat_four_match_nolit_encodeBlockAsm4MB_emit_copy:
|
|
SUB $256, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R9, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm4MB
|
|
|
|
repeat_three_match_nolit_encodeBlockAsm4MB_emit_copy:
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R9, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm4MB
|
|
|
|
repeat_two_match_nolit_encodeBlockAsm4MB_emit_copy:
|
|
LSLW $0x02, R9, R9
|
|
ORRW $0x01, R9, R9
|
|
MOVH R9, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm4MB
|
|
|
|
repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy:
|
|
MOVD $0, R6
|
|
ADD R9<<2, R6, R9
|
|
ADD $1, R9, R9
|
|
MOVWU R9, R9
|
|
MOVB R5, 1(R1)
|
|
ASRW $0x08, R5, R5
|
|
LSLW $0x05, R5, R5
|
|
ORRW R5, R9, R9
|
|
MOVB R9, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm4MB
|
|
|
|
four_bytes_remain_match_nolit_encodeBlockAsm4MB:
|
|
TSTW R9, R9
|
|
BEQ match_nolit_emitcopy_end_encodeBlockAsm4MB
|
|
MOVD $0, R6
|
|
ADD R9<<2, R6, R9
|
|
SUB $1, R9, R9
|
|
MOVWU R9, R9
|
|
MOVB R9, (R1)
|
|
MOVW R5, 1(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm4MB
|
|
|
|
two_byte_offset_match_nolit_encodeBlockAsm4MB:
|
|
CMPW $0x40, R9
|
|
BLS two_byte_offset_short_match_nolit_encodeBlockAsm4MB
|
|
CMPW $0x00000800, R5
|
|
BHS long_offset_short_match_nolit_encodeBlockAsm4MB
|
|
MOVD $0x00000001, R6
|
|
ADD $16, R6, R6
|
|
MOVWU R6, R6
|
|
MOVB R5, 1(R1)
|
|
LSRW $0x08, R5, R5
|
|
LSLW $0x05, R5, R5
|
|
ORRW R5, R6, R6
|
|
MOVB R6, (R1)
|
|
ADD $0x02, R1, R1
|
|
SUBW $0x08, R9, R9
|
|
|
|
// emitRepeat
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
JMP cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b
|
|
MOVWU R9, R6
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
CMPW $0x08, R6
|
|
BLS repeat_two_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b
|
|
CMPW $0x0c, R6
|
|
BHS cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b
|
|
CMPW $0x00000800, R5
|
|
BLO repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b:
|
|
CMPW $0x00000104, R9
|
|
BLO repeat_three_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b
|
|
CMPW $0x00010100, R9
|
|
BLO repeat_four_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b
|
|
SUB $65536, R9, R9
|
|
MOVWU R9, R9
|
|
MOVWU R9, R5
|
|
MOVD $0x001d, R16
|
|
MOVH R16, (R1)
|
|
MOVH R9, 2(R1)
|
|
ASRW $0x10, R5, R5
|
|
MOVB R5, 4(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm4MB
|
|
|
|
repeat_four_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b:
|
|
SUB $256, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R9, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm4MB
|
|
|
|
repeat_three_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b:
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R9, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm4MB
|
|
|
|
repeat_two_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b:
|
|
LSLW $0x02, R9, R9
|
|
ORRW $0x01, R9, R9
|
|
MOVH R9, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm4MB
|
|
|
|
repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b:
|
|
MOVD $0, R6
|
|
ADD R9<<2, R6, R9
|
|
ADD $1, R9, R9
|
|
MOVWU R9, R9
|
|
MOVB R5, 1(R1)
|
|
ASRW $0x08, R5, R5
|
|
LSLW $0x05, R5, R5
|
|
ORRW R5, R9, R9
|
|
MOVB R9, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm4MB
|
|
|
|
long_offset_short_match_nolit_encodeBlockAsm4MB:
|
|
MOVD $0xee, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
SUB $60, R9, R9
|
|
MOVWU R9, R9
|
|
ADD $0x03, R1, R1
|
|
|
|
// emitRepeat
|
|
MOVWU R9, R6
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
CMPW $0x08, R6
|
|
BLS repeat_two_match_nolit_encodeBlockAsm4MB_emit_copy_short
|
|
CMPW $0x0c, R6
|
|
BHS cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short
|
|
CMPW $0x00000800, R5
|
|
BLO repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short:
|
|
CMPW $0x00000104, R9
|
|
BLO repeat_three_match_nolit_encodeBlockAsm4MB_emit_copy_short
|
|
CMPW $0x00010100, R9
|
|
BLO repeat_four_match_nolit_encodeBlockAsm4MB_emit_copy_short
|
|
SUB $65536, R9, R9
|
|
MOVWU R9, R9
|
|
MOVWU R9, R5
|
|
MOVD $0x001d, R16
|
|
MOVH R16, (R1)
|
|
MOVH R9, 2(R1)
|
|
ASRW $0x10, R5, R5
|
|
MOVB R5, 4(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm4MB
|
|
|
|
repeat_four_match_nolit_encodeBlockAsm4MB_emit_copy_short:
|
|
SUB $256, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R9, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm4MB
|
|
|
|
repeat_three_match_nolit_encodeBlockAsm4MB_emit_copy_short:
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R9, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm4MB
|
|
|
|
repeat_two_match_nolit_encodeBlockAsm4MB_emit_copy_short:
|
|
LSLW $0x02, R9, R9
|
|
ORRW $0x01, R9, R9
|
|
MOVH R9, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm4MB
|
|
|
|
repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short:
|
|
MOVD $0, R6
|
|
ADD R9<<2, R6, R9
|
|
ADD $1, R9, R9
|
|
MOVWU R9, R9
|
|
MOVB R5, 1(R1)
|
|
ASRW $0x08, R5, R5
|
|
LSLW $0x05, R5, R5
|
|
ORRW R5, R9, R9
|
|
MOVB R9, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm4MB
|
|
|
|
two_byte_offset_short_match_nolit_encodeBlockAsm4MB:
|
|
MOVWU R9, R6
|
|
LSLW $0x02, R6, R6
|
|
CMPW $0x0c, R9
|
|
BHS emit_copy_three_match_nolit_encodeBlockAsm4MB
|
|
CMPW $0x00000800, R5
|
|
BHS emit_copy_three_match_nolit_encodeBlockAsm4MB
|
|
SUB $15, R6, R6
|
|
MOVWU R6, R6
|
|
MOVB R5, 1(R1)
|
|
LSRW $0x08, R5, R5
|
|
LSLW $0x05, R5, R5
|
|
ORRW R5, R6, R6
|
|
MOVB R6, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm4MB
|
|
|
|
emit_copy_three_match_nolit_encodeBlockAsm4MB:
|
|
SUB $2, R6, R6
|
|
MOVWU R6, R6
|
|
MOVB R6, (R1)
|
|
MOVH R5, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
|
|
match_nolit_emitcopy_end_encodeBlockAsm4MB:
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R2
|
|
BHS emit_remainder_encodeBlockAsm4MB
|
|
ADD R2, R3, R15
|
|
MOVD -2(R15), R6
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R1
|
|
BLO match_nolit_dst_ok_encodeBlockAsm4MB
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeBlockAsm4MB:
|
|
MOVD $0x0000cf1bbcdcbf9b, R8
|
|
MOVD R6, R7
|
|
LSR $0x10, R6, R6
|
|
MOVD R6, R5
|
|
LSL $0x10, R7, R7
|
|
MUL R8, R7, R7
|
|
LSR $0x32, R7, R7
|
|
LSL $0x10, R5, R5
|
|
MUL R8, R5, R5
|
|
LSR $0x32, R5, R5
|
|
SUB $2, R2, R8
|
|
MOVWU R8, R8
|
|
ADD R5<<2, R0, R9
|
|
MOVWU (R9), R5
|
|
MOVW R8, (R0)(R7<<2)
|
|
MOVW R2, (R9)
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ match_nolit_loop_encodeBlockAsm4MB
|
|
ADDW $1, R2, R2
|
|
JMP search_loop_encodeBlockAsm4MB
|
|
|
|
emit_remainder_encodeBlockAsm4MB:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R0, R0
|
|
ADD R0, R1, R0
|
|
ADD $4, R0, R0
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R0
|
|
BLO emit_remainder_ok_encodeBlockAsm4MB
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeBlockAsm4MB:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R2
|
|
CMPW R0, R2
|
|
BEQ emit_literal_done_emit_remainder_encodeBlockAsm4MB
|
|
MOVWU R0, R5
|
|
MOVW R0, 20(RSP)
|
|
ADD R2, R3, R0
|
|
SUBW R2, R5, R5
|
|
SUB $1, R5, R2
|
|
MOVWU R2, R2
|
|
CMPW $0x3c, R2
|
|
BLO one_byte_emit_remainder_encodeBlockAsm4MB
|
|
CMPW $0x00000100, R2
|
|
BLO two_bytes_emit_remainder_encodeBlockAsm4MB
|
|
CMPW $0x00010000, R2
|
|
BLO three_bytes_emit_remainder_encodeBlockAsm4MB
|
|
MOVWU R2, R3
|
|
LSRW $0x10, R3, R3
|
|
MOVD $0xf8, R16
|
|
MOVB R16, (R1)
|
|
MOVH R2, 1(R1)
|
|
MOVB R3, 3(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP memmove_long_emit_remainder_encodeBlockAsm4MB
|
|
|
|
three_bytes_emit_remainder_encodeBlockAsm4MB:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R2, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_emit_remainder_encodeBlockAsm4MB
|
|
|
|
two_bytes_emit_remainder_encodeBlockAsm4MB:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R2, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R2
|
|
BLO memmove_emit_remainder_encodeBlockAsm4MB
|
|
JMP memmove_long_emit_remainder_encodeBlockAsm4MB
|
|
|
|
one_byte_emit_remainder_encodeBlockAsm4MB:
|
|
LSLW $0x02, R2, R16
|
|
BFI $0, R16, $8, R2
|
|
MOVB R2, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_emit_remainder_encodeBlockAsm4MB:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveShort
|
|
CMP $0x03, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_1or2
|
|
BEQ emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_3
|
|
CMP $0x08, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_4through7
|
|
CMP $0x10, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_8through16
|
|
CMP $0x20, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_1or2:
|
|
MOVBU (R0), R16
|
|
BFI $0, R16, $8, R5
|
|
ADD R3, R0, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVB R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVB R0, -1(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm4MB
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_3:
|
|
MOVHU (R0), R16
|
|
BFI $0, R16, $16, R5
|
|
MOVBU 2(R0), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVH R5, (R1)
|
|
MOVB R0, 2(R1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm4MB
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_4through7:
|
|
MOVWU (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVWU -4(R15), R0
|
|
MOVW R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVW R0, -4(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm4MB
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_8through16:
|
|
MOVD (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVD -8(R15), R0
|
|
MOVD R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVD R0, -8(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm4MB
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_17through32:
|
|
FMOVQ (R0), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm4MB
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_33through64:
|
|
FMOVQ (R0), F0
|
|
FMOVQ 16(R0), F1
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_emit_remainder_encodeBlockAsm4MB:
|
|
MOVD R2, R1
|
|
JMP emit_literal_done_emit_remainder_encodeBlockAsm4MB
|
|
|
|
memmove_long_emit_remainder_encodeBlockAsm4MB:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveLong
|
|
MOVD R0, R5
|
|
MOVD R1, R6
|
|
MOVD R3, R7
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeBlockAsm4MBlarge_big_loop_back:
|
|
FMOVQ (R5), F0
|
|
FMOVQ 16(R5), F1
|
|
FMOVQ F0, (R6)
|
|
FMOVQ F1, 16(R6)
|
|
ADD $0x20, R5, R5
|
|
ADD $0x20, R6, R6
|
|
SUB $0x20, R7, R7
|
|
CMP $0x20, R7
|
|
BHS emit_lit_memmove_long_emit_remainder_encodeBlockAsm4MBlarge_big_loop_back
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R3, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R2, R1
|
|
|
|
emit_literal_done_emit_remainder_encodeBlockAsm4MB:
|
|
MOVD dst_base+0(FP), R0
|
|
SUB R0, R1, R1
|
|
MOVD R1, ret+56(FP)
|
|
RET
|
|
|
|
// func encodeBlockAsm12B(dst []byte, src []byte, tmp *[16384]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeBlockAsm12B(SB), $24-64
|
|
MOVD tmp+48(FP), R0
|
|
MOVD dst_base+0(FP), R1
|
|
MOVD $0x00000080, R2
|
|
MOVD R0, R3
|
|
VEOR V0.B16, V0.B16, V0.B16
|
|
|
|
zero_loop_encodeBlockAsm12B:
|
|
FMOVQ F0, (R3)
|
|
FMOVQ F0, 16(R3)
|
|
FMOVQ F0, 32(R3)
|
|
FMOVQ F0, 48(R3)
|
|
FMOVQ F0, 64(R3)
|
|
FMOVQ F0, 80(R3)
|
|
FMOVQ F0, 96(R3)
|
|
FMOVQ F0, 112(R3)
|
|
ADD $0x80, R3, R3
|
|
SUBS $1, R2, R2
|
|
BNE zero_loop_encodeBlockAsm12B
|
|
MOVD $0x00000000, R16
|
|
MOVW R16, 20(RSP)
|
|
MOVD src_len+32(FP), R2
|
|
SUB $9, R2, R3
|
|
SUB $8, R2, R5
|
|
MOVW R5, 16(RSP)
|
|
LSR $0x05, R2, R2
|
|
SUBW R2, R3, R3
|
|
ADD R3, R1, R3
|
|
MOVD R3, 8(RSP)
|
|
MOVD $0x00000001, R2
|
|
MOVW R2, 24(RSP)
|
|
MOVD src_base+24(FP), R3
|
|
|
|
search_loop_encodeBlockAsm12B:
|
|
MOVWU R2, R5
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R5, R5
|
|
LSRW $0x05, R5, R5
|
|
ADD R5, R2, R5
|
|
ADD $4, R5, R5
|
|
MOVWU R5, R5
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R5
|
|
BHS emit_remainder_encodeBlockAsm12B
|
|
MOVD (R3)(R2), R6
|
|
MOVW R5, 28(RSP)
|
|
MOVD $0x000000cf1bbcdcbb, R8
|
|
MOVD R6, R9
|
|
MOVD R6, R10
|
|
LSR $0x08, R10, R10
|
|
LSL $0x18, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x34, R9, R9
|
|
LSL $0x18, R10, R10
|
|
MUL R8, R10, R10
|
|
LSR $0x34, R10, R10
|
|
MOVWU (R0)(R9<<2), R5
|
|
MOVWU (R0)(R10<<2), R7
|
|
MOVW R2, (R0)(R9<<2)
|
|
ADD $1, R2, R9
|
|
MOVWU R9, R9
|
|
MOVW R9, (R0)(R10<<2)
|
|
MOVD R6, R9
|
|
LSR $0x10, R9, R9
|
|
LSL $0x18, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x34, R9, R9
|
|
MOVWU R2, R8
|
|
MOVWU 24(RSP), R16
|
|
SUBW R16, R8, R8
|
|
ADD R8, R3, R15
|
|
MOVWU 1(R15), R10
|
|
MOVD R6, R8
|
|
LSR $0x08, R8, R8
|
|
CMPW R10, R8
|
|
BNE no_repeat_found_encodeBlockAsm12B
|
|
ADD $1, R2, R6
|
|
MOVWU R6, R6
|
|
MOVWU 20(RSP), R7
|
|
MOVWU R6, R5
|
|
MOVWU 24(RSP), R16
|
|
SUBSW R16, R5, R5
|
|
BEQ repeat_extend_back_end_encodeBlockAsm12B
|
|
|
|
repeat_extend_back_loop_encodeBlockAsm12B:
|
|
CMPW R7, R6
|
|
BLS repeat_extend_back_end_encodeBlockAsm12B
|
|
ADD R5, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R8
|
|
ADD R6, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R9
|
|
AND $0xff, R9, R16
|
|
AND $0xff, R8, R15
|
|
CMP R16, R15
|
|
BNE repeat_extend_back_end_encodeBlockAsm12B
|
|
SUB $1, R6, R6
|
|
MOVWU R6, R6
|
|
SUBSW $1, R5, R5
|
|
BNE repeat_extend_back_loop_encodeBlockAsm12B
|
|
|
|
repeat_extend_back_end_encodeBlockAsm12B:
|
|
MOVWU R6, R5
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R5, R5
|
|
ADD R5, R1, R5
|
|
ADD $3, R5, R5
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R5
|
|
BLO repeat_dst_size_check_encodeBlockAsm12B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
repeat_dst_size_check_encodeBlockAsm12B:
|
|
MOVWU 20(RSP), R5
|
|
CMPW R6, R5
|
|
BEQ emit_literal_done_repeat_emit_encodeBlockAsm12B
|
|
MOVWU R6, R8
|
|
MOVW R6, 20(RSP)
|
|
ADD R5, R3, R9
|
|
SUBW R5, R8, R8
|
|
SUB $1, R8, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x3c, R5
|
|
BLO one_byte_repeat_emit_encodeBlockAsm12B
|
|
CMPW $0x00000100, R5
|
|
BLO two_bytes_repeat_emit_encodeBlockAsm12B
|
|
BLO three_bytes_repeat_emit_encodeBlockAsm12B
|
|
|
|
three_bytes_repeat_emit_encodeBlockAsm12B:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_repeat_emit_encodeBlockAsm12B
|
|
|
|
two_bytes_repeat_emit_encodeBlockAsm12B:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R5, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R5
|
|
BLO memmove_repeat_emit_encodeBlockAsm12B
|
|
JMP memmove_long_repeat_emit_encodeBlockAsm12B
|
|
|
|
one_byte_repeat_emit_encodeBlockAsm12B:
|
|
LSLW $0x02, R5, R16
|
|
BFI $0, R16, $8, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_repeat_emit_encodeBlockAsm12B:
|
|
ADD R8, R1, R5
|
|
|
|
// genMemMoveShort
|
|
CMP $0x08, R8
|
|
BLS emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_8
|
|
CMP $0x10, R8
|
|
BLS emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_8through16
|
|
CMP $0x20, R8
|
|
BLS emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_17through32
|
|
JMP emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_8:
|
|
MOVD (R9), R10
|
|
MOVD R10, (R1)
|
|
JMP memmove_end_copy_repeat_emit_encodeBlockAsm12B
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_8through16:
|
|
MOVD (R9), R10
|
|
ADD R8, R9, R15
|
|
MOVD -8(R15), R9
|
|
MOVD R10, (R1)
|
|
ADD R8, R1, R15
|
|
MOVD R9, -8(R15)
|
|
JMP memmove_end_copy_repeat_emit_encodeBlockAsm12B
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_17through32:
|
|
FMOVQ (R9), F0
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_repeat_emit_encodeBlockAsm12B
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_33through64:
|
|
FMOVQ (R9), F0
|
|
FMOVQ 16(R9), F1
|
|
ADD R8, R9, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_repeat_emit_encodeBlockAsm12B:
|
|
MOVD R5, R1
|
|
JMP emit_literal_done_repeat_emit_encodeBlockAsm12B
|
|
|
|
memmove_long_repeat_emit_encodeBlockAsm12B:
|
|
ADD R8, R1, R5
|
|
|
|
// genMemMoveLong
|
|
MOVD R9, R10
|
|
MOVD R1, R11
|
|
MOVD R8, R12
|
|
|
|
emit_lit_memmove_long_repeat_emit_encodeBlockAsm12Blarge_big_loop_back:
|
|
FMOVQ (R10), F0
|
|
FMOVQ 16(R10), F1
|
|
FMOVQ F0, (R11)
|
|
FMOVQ F1, 16(R11)
|
|
ADD $0x20, R10, R10
|
|
ADD $0x20, R11, R11
|
|
SUB $0x20, R12, R12
|
|
CMP $0x20, R12
|
|
BHS emit_lit_memmove_long_repeat_emit_encodeBlockAsm12Blarge_big_loop_back
|
|
ADD R8, R9, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R8, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R5, R1
|
|
|
|
emit_literal_done_repeat_emit_encodeBlockAsm12B:
|
|
ADDW $0x05, R2, R2
|
|
MOVWU R2, R5
|
|
MOVWU 24(RSP), R16
|
|
SUBW R16, R5, R5
|
|
MOVD src_len+32(FP), R8
|
|
SUBW R2, R8, R8
|
|
ADD R2, R3, R9
|
|
ADD R5, R3, R5
|
|
|
|
// matchLen
|
|
MOVD $0, R11
|
|
|
|
matchlen_loopback_16_repeat_extend_encodeBlockAsm12B:
|
|
CMPW $0x10, R8
|
|
BLO matchlen_match8_repeat_extend_encodeBlockAsm12B
|
|
MOVD (R9)(R11), R10
|
|
ADD R11, R9, R15
|
|
MOVD 8(R15), R12
|
|
MOVD (R5)(R11), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_8_repeat_extend_encodeBlockAsm12B
|
|
ADD R11, R5, R15
|
|
MOVD 8(R15), R16
|
|
EOR R16, R12, R12
|
|
TST R12, R12
|
|
BNE matchlen_bsf_16repeat_extend_encodeBlockAsm12B
|
|
SUB $16, R8, R8
|
|
MOVWU R8, R8
|
|
ADD $16, R11, R11
|
|
MOVWU R11, R11
|
|
JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm12B
|
|
|
|
matchlen_bsf_16repeat_extend_encodeBlockAsm12B:
|
|
RBIT R12, R12
|
|
CLZ R12, R12
|
|
ASR $0x03, R12, R12
|
|
ADD R12, R11, R11
|
|
ADD $8, R11, R11
|
|
MOVWU R11, R11
|
|
JMP repeat_extend_forward_end_encodeBlockAsm12B
|
|
|
|
matchlen_match8_repeat_extend_encodeBlockAsm12B:
|
|
CMPW $0x08, R8
|
|
BLO matchlen_match4_repeat_extend_encodeBlockAsm12B
|
|
MOVD (R9)(R11), R10
|
|
MOVD (R5)(R11), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_8_repeat_extend_encodeBlockAsm12B
|
|
SUB $8, R8, R8
|
|
MOVWU R8, R8
|
|
ADD $8, R11, R11
|
|
MOVWU R11, R11
|
|
JMP matchlen_match4_repeat_extend_encodeBlockAsm12B
|
|
|
|
matchlen_bsf_8_repeat_extend_encodeBlockAsm12B:
|
|
RBIT R10, R10
|
|
CLZ R10, R10
|
|
ASR $0x03, R10, R10
|
|
ADD R10, R11, R11
|
|
MOVWU R11, R11
|
|
JMP repeat_extend_forward_end_encodeBlockAsm12B
|
|
|
|
matchlen_match4_repeat_extend_encodeBlockAsm12B:
|
|
CMPW $0x04, R8
|
|
BLO matchlen_match2_repeat_extend_encodeBlockAsm12B
|
|
MOVWU (R9)(R11), R10
|
|
MOVWU (R5)(R11), R16
|
|
CMPW R10, R16
|
|
BNE matchlen_match2_repeat_extend_encodeBlockAsm12B
|
|
SUB $4, R8, R8
|
|
MOVWU R8, R8
|
|
ADD $4, R11, R11
|
|
MOVWU R11, R11
|
|
|
|
matchlen_match2_repeat_extend_encodeBlockAsm12B:
|
|
CMPW $0x01, R8
|
|
BEQ matchlen_match1_repeat_extend_encodeBlockAsm12B
|
|
BLO repeat_extend_forward_end_encodeBlockAsm12B
|
|
MOVHU (R9)(R11), R16
|
|
BFI $0, R16, $16, R10
|
|
ADD R11, R5, R15
|
|
MOVHU (R15), R15
|
|
AND $0xffff, R10, R16
|
|
CMP R16, R15
|
|
BNE matchlen_match1_repeat_extend_encodeBlockAsm12B
|
|
ADD $2, R11, R11
|
|
MOVWU R11, R11
|
|
SUBSW $0x02, R8, R8
|
|
BEQ repeat_extend_forward_end_encodeBlockAsm12B
|
|
|
|
matchlen_match1_repeat_extend_encodeBlockAsm12B:
|
|
MOVBU (R9)(R11), R16
|
|
BFI $0, R16, $8, R10
|
|
ADD R11, R5, R15
|
|
MOVBU (R15), R15
|
|
AND $0xff, R10, R16
|
|
CMP R16, R15
|
|
BNE repeat_extend_forward_end_encodeBlockAsm12B
|
|
ADD $1, R11, R11
|
|
MOVWU R11, R11
|
|
|
|
repeat_extend_forward_end_encodeBlockAsm12B:
|
|
ADDW R11, R2, R2
|
|
MOVWU R2, R5
|
|
SUBW R6, R5, R5
|
|
MOVWU 24(RSP), R6
|
|
TSTW R7, R7
|
|
BEQ repeat_as_copy_encodeBlockAsm12B
|
|
|
|
// emitRepeat
|
|
MOVWU R5, R7
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x08, R7
|
|
BLS repeat_two_match_repeat_encodeBlockAsm12B
|
|
CMPW $0x0c, R7
|
|
BHS cant_repeat_two_offset_match_repeat_encodeBlockAsm12B
|
|
CMPW $0x00000800, R6
|
|
BLO repeat_two_offset_match_repeat_encodeBlockAsm12B
|
|
|
|
cant_repeat_two_offset_match_repeat_encodeBlockAsm12B:
|
|
CMPW $0x00000104, R5
|
|
BLO repeat_three_match_repeat_encodeBlockAsm12B
|
|
SUB $256, R5, R5
|
|
MOVWU R5, R5
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R5, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm12B
|
|
|
|
repeat_three_match_repeat_encodeBlockAsm12B:
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R5, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm12B
|
|
|
|
repeat_two_match_repeat_encodeBlockAsm12B:
|
|
LSLW $0x02, R5, R5
|
|
ORRW $0x01, R5, R5
|
|
MOVH R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm12B
|
|
|
|
repeat_two_offset_match_repeat_encodeBlockAsm12B:
|
|
MOVD $0, R7
|
|
ADD R5<<2, R7, R5
|
|
ADD $1, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R6, 1(R1)
|
|
ASRW $0x08, R6, R6
|
|
LSLW $0x05, R6, R6
|
|
ORRW R6, R5, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm12B
|
|
|
|
repeat_as_copy_encodeBlockAsm12B:
|
|
// emitCopy
|
|
CMPW $0x40, R5
|
|
BLS two_byte_offset_short_repeat_as_copy_encodeBlockAsm12B
|
|
CMPW $0x00000800, R6
|
|
BHS long_offset_short_repeat_as_copy_encodeBlockAsm12B
|
|
MOVD $0x00000001, R7
|
|
ADD $16, R7, R7
|
|
MOVWU R7, R7
|
|
MOVB R6, 1(R1)
|
|
LSRW $0x08, R6, R6
|
|
LSLW $0x05, R6, R6
|
|
ORRW R6, R7, R7
|
|
MOVB R7, (R1)
|
|
ADD $0x02, R1, R1
|
|
SUBW $0x08, R5, R5
|
|
|
|
// emitRepeat
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
JMP cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b
|
|
MOVWU R5, R7
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x08, R7
|
|
BLS repeat_two_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b
|
|
CMPW $0x0c, R7
|
|
BHS cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b
|
|
CMPW $0x00000800, R6
|
|
BLO repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b
|
|
|
|
cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b:
|
|
CMPW $0x00000104, R5
|
|
BLO repeat_three_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b
|
|
SUB $256, R5, R5
|
|
MOVWU R5, R5
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R5, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm12B
|
|
|
|
repeat_three_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b:
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R5, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm12B
|
|
|
|
repeat_two_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b:
|
|
LSLW $0x02, R5, R5
|
|
ORRW $0x01, R5, R5
|
|
MOVH R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm12B
|
|
|
|
repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b:
|
|
MOVD $0, R7
|
|
ADD R5<<2, R7, R5
|
|
ADD $1, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R6, 1(R1)
|
|
ASRW $0x08, R6, R6
|
|
LSLW $0x05, R6, R6
|
|
ORRW R6, R5, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm12B
|
|
|
|
long_offset_short_repeat_as_copy_encodeBlockAsm12B:
|
|
MOVD $0xee, R16
|
|
MOVB R16, (R1)
|
|
MOVH R6, 1(R1)
|
|
SUB $60, R5, R5
|
|
MOVWU R5, R5
|
|
ADD $0x03, R1, R1
|
|
|
|
// emitRepeat
|
|
MOVWU R5, R7
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x08, R7
|
|
BLS repeat_two_repeat_as_copy_encodeBlockAsm12B_emit_copy_short
|
|
CMPW $0x0c, R7
|
|
BHS cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short
|
|
CMPW $0x00000800, R6
|
|
BLO repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short
|
|
|
|
cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short:
|
|
CMPW $0x00000104, R5
|
|
BLO repeat_three_repeat_as_copy_encodeBlockAsm12B_emit_copy_short
|
|
SUB $256, R5, R5
|
|
MOVWU R5, R5
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R5, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm12B
|
|
|
|
repeat_three_repeat_as_copy_encodeBlockAsm12B_emit_copy_short:
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R5, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm12B
|
|
|
|
repeat_two_repeat_as_copy_encodeBlockAsm12B_emit_copy_short:
|
|
LSLW $0x02, R5, R5
|
|
ORRW $0x01, R5, R5
|
|
MOVH R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm12B
|
|
|
|
repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short:
|
|
MOVD $0, R7
|
|
ADD R5<<2, R7, R5
|
|
ADD $1, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R6, 1(R1)
|
|
ASRW $0x08, R6, R6
|
|
LSLW $0x05, R6, R6
|
|
ORRW R6, R5, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm12B
|
|
|
|
two_byte_offset_short_repeat_as_copy_encodeBlockAsm12B:
|
|
MOVWU R5, R7
|
|
LSLW $0x02, R7, R7
|
|
CMPW $0x0c, R5
|
|
BHS emit_copy_three_repeat_as_copy_encodeBlockAsm12B
|
|
CMPW $0x00000800, R6
|
|
BHS emit_copy_three_repeat_as_copy_encodeBlockAsm12B
|
|
SUB $15, R7, R7
|
|
MOVWU R7, R7
|
|
MOVB R6, 1(R1)
|
|
LSRW $0x08, R6, R6
|
|
LSLW $0x05, R6, R6
|
|
ORRW R6, R7, R7
|
|
MOVB R7, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm12B
|
|
|
|
emit_copy_three_repeat_as_copy_encodeBlockAsm12B:
|
|
SUB $2, R7, R7
|
|
MOVWU R7, R7
|
|
MOVB R7, (R1)
|
|
MOVH R6, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
|
|
repeat_end_emit_encodeBlockAsm12B:
|
|
MOVW R2, 20(RSP)
|
|
JMP search_loop_encodeBlockAsm12B
|
|
|
|
no_repeat_found_encodeBlockAsm12B:
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ candidate_match_encodeBlockAsm12B
|
|
LSR $0x08, R6, R6
|
|
MOVWU (R0)(R9<<2), R5
|
|
ADD $2, R2, R8
|
|
MOVWU R8, R8
|
|
MOVWU (R3)(R7), R16
|
|
CMPW R6, R16
|
|
BEQ candidate2_match_encodeBlockAsm12B
|
|
MOVW R8, (R0)(R9<<2)
|
|
LSR $0x08, R6, R6
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ candidate3_match_encodeBlockAsm12B
|
|
MOVWU 28(RSP), R2
|
|
JMP search_loop_encodeBlockAsm12B
|
|
|
|
candidate3_match_encodeBlockAsm12B:
|
|
ADDW $0x02, R2, R2
|
|
JMP candidate_match_encodeBlockAsm12B
|
|
|
|
candidate2_match_encodeBlockAsm12B:
|
|
MOVW R8, (R0)(R9<<2)
|
|
ADDW $1, R2, R2
|
|
MOVWU R7, R5
|
|
|
|
candidate_match_encodeBlockAsm12B:
|
|
MOVWU 20(RSP), R6
|
|
TSTW R5, R5
|
|
BEQ match_extend_back_end_encodeBlockAsm12B
|
|
|
|
match_extend_back_loop_encodeBlockAsm12B:
|
|
CMPW R6, R2
|
|
BLS match_extend_back_end_encodeBlockAsm12B
|
|
ADD R5, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R7
|
|
ADD R2, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R8
|
|
AND $0xff, R8, R16
|
|
AND $0xff, R7, R15
|
|
CMP R16, R15
|
|
BNE match_extend_back_end_encodeBlockAsm12B
|
|
SUB $1, R2, R2
|
|
MOVWU R2, R2
|
|
SUBSW $1, R5, R5
|
|
BEQ match_extend_back_end_encodeBlockAsm12B
|
|
JMP match_extend_back_loop_encodeBlockAsm12B
|
|
|
|
match_extend_back_end_encodeBlockAsm12B:
|
|
MOVWU R2, R6
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R6, R6
|
|
ADD R6, R1, R6
|
|
ADD $3, R6, R6
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R6
|
|
BLO match_dst_size_check_encodeBlockAsm12B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeBlockAsm12B:
|
|
MOVWU R2, R6
|
|
MOVWU 20(RSP), R7
|
|
CMPW R6, R7
|
|
BEQ emit_literal_done_match_emit_encodeBlockAsm12B
|
|
MOVWU R6, R8
|
|
MOVW R6, 20(RSP)
|
|
ADD R7, R3, R6
|
|
SUBW R7, R8, R8
|
|
SUB $1, R8, R7
|
|
MOVWU R7, R7
|
|
CMPW $0x3c, R7
|
|
BLO one_byte_match_emit_encodeBlockAsm12B
|
|
CMPW $0x00000100, R7
|
|
BLO two_bytes_match_emit_encodeBlockAsm12B
|
|
BLO three_bytes_match_emit_encodeBlockAsm12B
|
|
|
|
three_bytes_match_emit_encodeBlockAsm12B:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R7, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_match_emit_encodeBlockAsm12B
|
|
|
|
two_bytes_match_emit_encodeBlockAsm12B:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R7, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R7
|
|
BLO memmove_match_emit_encodeBlockAsm12B
|
|
JMP memmove_long_match_emit_encodeBlockAsm12B
|
|
|
|
one_byte_match_emit_encodeBlockAsm12B:
|
|
LSLW $0x02, R7, R16
|
|
BFI $0, R16, $8, R7
|
|
MOVB R7, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_match_emit_encodeBlockAsm12B:
|
|
ADD R8, R1, R7
|
|
|
|
// genMemMoveShort
|
|
CMP $0x08, R8
|
|
BLS emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_8
|
|
CMP $0x10, R8
|
|
BLS emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_8through16
|
|
CMP $0x20, R8
|
|
BLS emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_8:
|
|
MOVD (R6), R9
|
|
MOVD R9, (R1)
|
|
JMP memmove_end_copy_match_emit_encodeBlockAsm12B
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_8through16:
|
|
MOVD (R6), R9
|
|
ADD R8, R6, R15
|
|
MOVD -8(R15), R6
|
|
MOVD R9, (R1)
|
|
ADD R8, R1, R15
|
|
MOVD R6, -8(R15)
|
|
JMP memmove_end_copy_match_emit_encodeBlockAsm12B
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_17through32:
|
|
FMOVQ (R6), F0
|
|
ADD R8, R6, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_match_emit_encodeBlockAsm12B
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_33through64:
|
|
FMOVQ (R6), F0
|
|
FMOVQ 16(R6), F1
|
|
ADD R8, R6, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R8, R6, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_match_emit_encodeBlockAsm12B:
|
|
MOVD R7, R1
|
|
JMP emit_literal_done_match_emit_encodeBlockAsm12B
|
|
|
|
memmove_long_match_emit_encodeBlockAsm12B:
|
|
ADD R8, R1, R7
|
|
|
|
// genMemMoveLong
|
|
MOVD R6, R9
|
|
MOVD R1, R10
|
|
MOVD R8, R11
|
|
|
|
emit_lit_memmove_long_match_emit_encodeBlockAsm12Blarge_big_loop_back:
|
|
FMOVQ (R9), F0
|
|
FMOVQ 16(R9), F1
|
|
FMOVQ F0, (R10)
|
|
FMOVQ F1, 16(R10)
|
|
ADD $0x20, R9, R9
|
|
ADD $0x20, R10, R10
|
|
SUB $0x20, R11, R11
|
|
CMP $0x20, R11
|
|
BHS emit_lit_memmove_long_match_emit_encodeBlockAsm12Blarge_big_loop_back
|
|
ADD R8, R6, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R8, R6, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R8, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R7, R1
|
|
|
|
emit_literal_done_match_emit_encodeBlockAsm12B:
|
|
match_nolit_loop_encodeBlockAsm12B:
|
|
MOVWU R2, R6
|
|
SUBW R5, R6, R6
|
|
MOVW R6, 24(RSP)
|
|
ADDW $0x04, R2, R2
|
|
ADDW $0x04, R5, R5
|
|
MOVD src_len+32(FP), R6
|
|
SUBW R2, R6, R6
|
|
ADD R2, R3, R7
|
|
ADD R5, R3, R5
|
|
|
|
// matchLen
|
|
MOVD $0, R9
|
|
|
|
matchlen_loopback_16_match_nolit_encodeBlockAsm12B:
|
|
CMPW $0x10, R6
|
|
BLO matchlen_match8_match_nolit_encodeBlockAsm12B
|
|
MOVD (R7)(R9), R8
|
|
ADD R9, R7, R15
|
|
MOVD 8(R15), R10
|
|
MOVD (R5)(R9), R16
|
|
EOR R16, R8, R8
|
|
TST R8, R8
|
|
BNE matchlen_bsf_8_match_nolit_encodeBlockAsm12B
|
|
ADD R9, R5, R15
|
|
MOVD 8(R15), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_16match_nolit_encodeBlockAsm12B
|
|
SUB $16, R6, R6
|
|
MOVWU R6, R6
|
|
ADD $16, R9, R9
|
|
MOVWU R9, R9
|
|
JMP matchlen_loopback_16_match_nolit_encodeBlockAsm12B
|
|
|
|
matchlen_bsf_16match_nolit_encodeBlockAsm12B:
|
|
RBIT R10, R10
|
|
CLZ R10, R10
|
|
ASR $0x03, R10, R10
|
|
ADD R10, R9, R9
|
|
ADD $8, R9, R9
|
|
MOVWU R9, R9
|
|
JMP match_nolit_end_encodeBlockAsm12B
|
|
|
|
matchlen_match8_match_nolit_encodeBlockAsm12B:
|
|
CMPW $0x08, R6
|
|
BLO matchlen_match4_match_nolit_encodeBlockAsm12B
|
|
MOVD (R7)(R9), R8
|
|
MOVD (R5)(R9), R16
|
|
EOR R16, R8, R8
|
|
TST R8, R8
|
|
BNE matchlen_bsf_8_match_nolit_encodeBlockAsm12B
|
|
SUB $8, R6, R6
|
|
MOVWU R6, R6
|
|
ADD $8, R9, R9
|
|
MOVWU R9, R9
|
|
JMP matchlen_match4_match_nolit_encodeBlockAsm12B
|
|
|
|
matchlen_bsf_8_match_nolit_encodeBlockAsm12B:
|
|
RBIT R8, R8
|
|
CLZ R8, R8
|
|
ASR $0x03, R8, R8
|
|
ADD R8, R9, R9
|
|
MOVWU R9, R9
|
|
JMP match_nolit_end_encodeBlockAsm12B
|
|
|
|
matchlen_match4_match_nolit_encodeBlockAsm12B:
|
|
CMPW $0x04, R6
|
|
BLO matchlen_match2_match_nolit_encodeBlockAsm12B
|
|
MOVWU (R7)(R9), R8
|
|
MOVWU (R5)(R9), R16
|
|
CMPW R8, R16
|
|
BNE matchlen_match2_match_nolit_encodeBlockAsm12B
|
|
SUB $4, R6, R6
|
|
MOVWU R6, R6
|
|
ADD $4, R9, R9
|
|
MOVWU R9, R9
|
|
|
|
matchlen_match2_match_nolit_encodeBlockAsm12B:
|
|
CMPW $0x01, R6
|
|
BEQ matchlen_match1_match_nolit_encodeBlockAsm12B
|
|
BLO match_nolit_end_encodeBlockAsm12B
|
|
MOVHU (R7)(R9), R16
|
|
BFI $0, R16, $16, R8
|
|
ADD R9, R5, R15
|
|
MOVHU (R15), R15
|
|
AND $0xffff, R8, R16
|
|
CMP R16, R15
|
|
BNE matchlen_match1_match_nolit_encodeBlockAsm12B
|
|
ADD $2, R9, R9
|
|
MOVWU R9, R9
|
|
SUBSW $0x02, R6, R6
|
|
BEQ match_nolit_end_encodeBlockAsm12B
|
|
|
|
matchlen_match1_match_nolit_encodeBlockAsm12B:
|
|
MOVBU (R7)(R9), R16
|
|
BFI $0, R16, $8, R8
|
|
ADD R9, R5, R15
|
|
MOVBU (R15), R15
|
|
AND $0xff, R8, R16
|
|
CMP R16, R15
|
|
BNE match_nolit_end_encodeBlockAsm12B
|
|
ADD $1, R9, R9
|
|
MOVWU R9, R9
|
|
|
|
match_nolit_end_encodeBlockAsm12B:
|
|
ADDW R9, R2, R2
|
|
MOVWU 24(RSP), R5
|
|
ADDW $0x04, R9, R9
|
|
MOVW R2, 20(RSP)
|
|
|
|
// emitCopy
|
|
CMPW $0x40, R9
|
|
BLS two_byte_offset_short_match_nolit_encodeBlockAsm12B
|
|
CMPW $0x00000800, R5
|
|
BHS long_offset_short_match_nolit_encodeBlockAsm12B
|
|
MOVD $0x00000001, R6
|
|
ADD $16, R6, R6
|
|
MOVWU R6, R6
|
|
MOVB R5, 1(R1)
|
|
LSRW $0x08, R5, R5
|
|
LSLW $0x05, R5, R5
|
|
ORRW R5, R6, R6
|
|
MOVB R6, (R1)
|
|
ADD $0x02, R1, R1
|
|
SUBW $0x08, R9, R9
|
|
|
|
// emitRepeat
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
JMP cant_repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short_2b
|
|
MOVWU R9, R6
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
CMPW $0x08, R6
|
|
BLS repeat_two_match_nolit_encodeBlockAsm12B_emit_copy_short_2b
|
|
CMPW $0x0c, R6
|
|
BHS cant_repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short_2b
|
|
CMPW $0x00000800, R5
|
|
BLO repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short_2b
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short_2b:
|
|
CMPW $0x00000104, R9
|
|
BLO repeat_three_match_nolit_encodeBlockAsm12B_emit_copy_short_2b
|
|
SUB $256, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R9, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm12B
|
|
|
|
repeat_three_match_nolit_encodeBlockAsm12B_emit_copy_short_2b:
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R9, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm12B
|
|
|
|
repeat_two_match_nolit_encodeBlockAsm12B_emit_copy_short_2b:
|
|
LSLW $0x02, R9, R9
|
|
ORRW $0x01, R9, R9
|
|
MOVH R9, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm12B
|
|
|
|
repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short_2b:
|
|
MOVD $0, R6
|
|
ADD R9<<2, R6, R9
|
|
ADD $1, R9, R9
|
|
MOVWU R9, R9
|
|
MOVB R5, 1(R1)
|
|
ASRW $0x08, R5, R5
|
|
LSLW $0x05, R5, R5
|
|
ORRW R5, R9, R9
|
|
MOVB R9, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm12B
|
|
|
|
long_offset_short_match_nolit_encodeBlockAsm12B:
|
|
MOVD $0xee, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
SUB $60, R9, R9
|
|
MOVWU R9, R9
|
|
ADD $0x03, R1, R1
|
|
|
|
// emitRepeat
|
|
MOVWU R9, R6
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
CMPW $0x08, R6
|
|
BLS repeat_two_match_nolit_encodeBlockAsm12B_emit_copy_short
|
|
CMPW $0x0c, R6
|
|
BHS cant_repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short
|
|
CMPW $0x00000800, R5
|
|
BLO repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short:
|
|
CMPW $0x00000104, R9
|
|
BLO repeat_three_match_nolit_encodeBlockAsm12B_emit_copy_short
|
|
SUB $256, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R9, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm12B
|
|
|
|
repeat_three_match_nolit_encodeBlockAsm12B_emit_copy_short:
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R9, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm12B
|
|
|
|
repeat_two_match_nolit_encodeBlockAsm12B_emit_copy_short:
|
|
LSLW $0x02, R9, R9
|
|
ORRW $0x01, R9, R9
|
|
MOVH R9, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm12B
|
|
|
|
repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short:
|
|
MOVD $0, R6
|
|
ADD R9<<2, R6, R9
|
|
ADD $1, R9, R9
|
|
MOVWU R9, R9
|
|
MOVB R5, 1(R1)
|
|
ASRW $0x08, R5, R5
|
|
LSLW $0x05, R5, R5
|
|
ORRW R5, R9, R9
|
|
MOVB R9, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm12B
|
|
|
|
two_byte_offset_short_match_nolit_encodeBlockAsm12B:
|
|
MOVWU R9, R6
|
|
LSLW $0x02, R6, R6
|
|
CMPW $0x0c, R9
|
|
BHS emit_copy_three_match_nolit_encodeBlockAsm12B
|
|
CMPW $0x00000800, R5
|
|
BHS emit_copy_three_match_nolit_encodeBlockAsm12B
|
|
SUB $15, R6, R6
|
|
MOVWU R6, R6
|
|
MOVB R5, 1(R1)
|
|
LSRW $0x08, R5, R5
|
|
LSLW $0x05, R5, R5
|
|
ORRW R5, R6, R6
|
|
MOVB R6, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm12B
|
|
|
|
emit_copy_three_match_nolit_encodeBlockAsm12B:
|
|
SUB $2, R6, R6
|
|
MOVWU R6, R6
|
|
MOVB R6, (R1)
|
|
MOVH R5, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
|
|
match_nolit_emitcopy_end_encodeBlockAsm12B:
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R2
|
|
BHS emit_remainder_encodeBlockAsm12B
|
|
ADD R2, R3, R15
|
|
MOVD -2(R15), R6
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R1
|
|
BLO match_nolit_dst_ok_encodeBlockAsm12B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeBlockAsm12B:
|
|
MOVD $0x000000cf1bbcdcbb, R8
|
|
MOVD R6, R7
|
|
LSR $0x10, R6, R6
|
|
MOVD R6, R5
|
|
LSL $0x18, R7, R7
|
|
MUL R8, R7, R7
|
|
LSR $0x34, R7, R7
|
|
LSL $0x18, R5, R5
|
|
MUL R8, R5, R5
|
|
LSR $0x34, R5, R5
|
|
SUB $2, R2, R8
|
|
MOVWU R8, R8
|
|
ADD R5<<2, R0, R9
|
|
MOVWU (R9), R5
|
|
MOVW R8, (R0)(R7<<2)
|
|
MOVW R2, (R9)
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ match_nolit_loop_encodeBlockAsm12B
|
|
ADDW $1, R2, R2
|
|
JMP search_loop_encodeBlockAsm12B
|
|
|
|
emit_remainder_encodeBlockAsm12B:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R0, R0
|
|
ADD R0, R1, R0
|
|
ADD $3, R0, R0
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R0
|
|
BLO emit_remainder_ok_encodeBlockAsm12B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeBlockAsm12B:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R2
|
|
CMPW R0, R2
|
|
BEQ emit_literal_done_emit_remainder_encodeBlockAsm12B
|
|
MOVWU R0, R5
|
|
MOVW R0, 20(RSP)
|
|
ADD R2, R3, R0
|
|
SUBW R2, R5, R5
|
|
SUB $1, R5, R2
|
|
MOVWU R2, R2
|
|
CMPW $0x3c, R2
|
|
BLO one_byte_emit_remainder_encodeBlockAsm12B
|
|
CMPW $0x00000100, R2
|
|
BLO two_bytes_emit_remainder_encodeBlockAsm12B
|
|
BLO three_bytes_emit_remainder_encodeBlockAsm12B
|
|
|
|
three_bytes_emit_remainder_encodeBlockAsm12B:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R2, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_emit_remainder_encodeBlockAsm12B
|
|
|
|
two_bytes_emit_remainder_encodeBlockAsm12B:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R2, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R2
|
|
BLO memmove_emit_remainder_encodeBlockAsm12B
|
|
JMP memmove_long_emit_remainder_encodeBlockAsm12B
|
|
|
|
one_byte_emit_remainder_encodeBlockAsm12B:
|
|
LSLW $0x02, R2, R16
|
|
BFI $0, R16, $8, R2
|
|
MOVB R2, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_emit_remainder_encodeBlockAsm12B:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveShort
|
|
CMP $0x03, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_1or2
|
|
BEQ emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_3
|
|
CMP $0x08, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_4through7
|
|
CMP $0x10, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_8through16
|
|
CMP $0x20, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_1or2:
|
|
MOVBU (R0), R16
|
|
BFI $0, R16, $8, R5
|
|
ADD R3, R0, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVB R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVB R0, -1(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_3:
|
|
MOVHU (R0), R16
|
|
BFI $0, R16, $16, R5
|
|
MOVBU 2(R0), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVH R5, (R1)
|
|
MOVB R0, 2(R1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_4through7:
|
|
MOVWU (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVWU -4(R15), R0
|
|
MOVW R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVW R0, -4(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_8through16:
|
|
MOVD (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVD -8(R15), R0
|
|
MOVD R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVD R0, -8(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_17through32:
|
|
FMOVQ (R0), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_33through64:
|
|
FMOVQ (R0), F0
|
|
FMOVQ 16(R0), F1
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_emit_remainder_encodeBlockAsm12B:
|
|
MOVD R2, R1
|
|
JMP emit_literal_done_emit_remainder_encodeBlockAsm12B
|
|
|
|
memmove_long_emit_remainder_encodeBlockAsm12B:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveLong
|
|
MOVD R0, R5
|
|
MOVD R1, R6
|
|
MOVD R3, R7
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeBlockAsm12Blarge_big_loop_back:
|
|
FMOVQ (R5), F0
|
|
FMOVQ 16(R5), F1
|
|
FMOVQ F0, (R6)
|
|
FMOVQ F1, 16(R6)
|
|
ADD $0x20, R5, R5
|
|
ADD $0x20, R6, R6
|
|
SUB $0x20, R7, R7
|
|
CMP $0x20, R7
|
|
BHS emit_lit_memmove_long_emit_remainder_encodeBlockAsm12Blarge_big_loop_back
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R3, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R2, R1
|
|
|
|
emit_literal_done_emit_remainder_encodeBlockAsm12B:
|
|
MOVD dst_base+0(FP), R0
|
|
SUB R0, R1, R1
|
|
MOVD R1, ret+56(FP)
|
|
RET
|
|
|
|
// func encodeBlockAsm10B(dst []byte, src []byte, tmp *[4096]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeBlockAsm10B(SB), $24-64
|
|
MOVD tmp+48(FP), R0
|
|
MOVD dst_base+0(FP), R1
|
|
MOVD $0x00000020, R2
|
|
MOVD R0, R3
|
|
VEOR V0.B16, V0.B16, V0.B16
|
|
|
|
zero_loop_encodeBlockAsm10B:
|
|
FMOVQ F0, (R3)
|
|
FMOVQ F0, 16(R3)
|
|
FMOVQ F0, 32(R3)
|
|
FMOVQ F0, 48(R3)
|
|
FMOVQ F0, 64(R3)
|
|
FMOVQ F0, 80(R3)
|
|
FMOVQ F0, 96(R3)
|
|
FMOVQ F0, 112(R3)
|
|
ADD $0x80, R3, R3
|
|
SUBS $1, R2, R2
|
|
BNE zero_loop_encodeBlockAsm10B
|
|
MOVD $0x00000000, R16
|
|
MOVW R16, 20(RSP)
|
|
MOVD src_len+32(FP), R2
|
|
SUB $9, R2, R3
|
|
SUB $8, R2, R5
|
|
MOVW R5, 16(RSP)
|
|
LSR $0x05, R2, R2
|
|
SUBW R2, R3, R3
|
|
ADD R3, R1, R3
|
|
MOVD R3, 8(RSP)
|
|
MOVD $0x00000001, R2
|
|
MOVW R2, 24(RSP)
|
|
MOVD src_base+24(FP), R3
|
|
|
|
search_loop_encodeBlockAsm10B:
|
|
MOVWU R2, R5
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R5, R5
|
|
LSRW $0x05, R5, R5
|
|
ADD R5, R2, R5
|
|
ADD $4, R5, R5
|
|
MOVWU R5, R5
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R5
|
|
BHS emit_remainder_encodeBlockAsm10B
|
|
MOVD (R3)(R2), R6
|
|
MOVW R5, 28(RSP)
|
|
MOVD $0x9e3779b1, R8
|
|
MOVD R6, R9
|
|
MOVD R6, R10
|
|
LSR $0x08, R10, R10
|
|
LSL $0x20, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x36, R9, R9
|
|
LSL $0x20, R10, R10
|
|
MUL R8, R10, R10
|
|
LSR $0x36, R10, R10
|
|
MOVWU (R0)(R9<<2), R5
|
|
MOVWU (R0)(R10<<2), R7
|
|
MOVW R2, (R0)(R9<<2)
|
|
ADD $1, R2, R9
|
|
MOVWU R9, R9
|
|
MOVW R9, (R0)(R10<<2)
|
|
MOVD R6, R9
|
|
LSR $0x10, R9, R9
|
|
LSL $0x20, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x36, R9, R9
|
|
MOVWU R2, R8
|
|
MOVWU 24(RSP), R16
|
|
SUBW R16, R8, R8
|
|
ADD R8, R3, R15
|
|
MOVWU 1(R15), R10
|
|
MOVD R6, R8
|
|
LSR $0x08, R8, R8
|
|
CMPW R10, R8
|
|
BNE no_repeat_found_encodeBlockAsm10B
|
|
ADD $1, R2, R6
|
|
MOVWU R6, R6
|
|
MOVWU 20(RSP), R7
|
|
MOVWU R6, R5
|
|
MOVWU 24(RSP), R16
|
|
SUBSW R16, R5, R5
|
|
BEQ repeat_extend_back_end_encodeBlockAsm10B
|
|
|
|
repeat_extend_back_loop_encodeBlockAsm10B:
|
|
CMPW R7, R6
|
|
BLS repeat_extend_back_end_encodeBlockAsm10B
|
|
ADD R5, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R8
|
|
ADD R6, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R9
|
|
AND $0xff, R9, R16
|
|
AND $0xff, R8, R15
|
|
CMP R16, R15
|
|
BNE repeat_extend_back_end_encodeBlockAsm10B
|
|
SUB $1, R6, R6
|
|
MOVWU R6, R6
|
|
SUBSW $1, R5, R5
|
|
BNE repeat_extend_back_loop_encodeBlockAsm10B
|
|
|
|
repeat_extend_back_end_encodeBlockAsm10B:
|
|
MOVWU R6, R5
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R5, R5
|
|
ADD R5, R1, R5
|
|
ADD $3, R5, R5
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R5
|
|
BLO repeat_dst_size_check_encodeBlockAsm10B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
repeat_dst_size_check_encodeBlockAsm10B:
|
|
MOVWU 20(RSP), R5
|
|
CMPW R6, R5
|
|
BEQ emit_literal_done_repeat_emit_encodeBlockAsm10B
|
|
MOVWU R6, R8
|
|
MOVW R6, 20(RSP)
|
|
ADD R5, R3, R9
|
|
SUBW R5, R8, R8
|
|
SUB $1, R8, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x3c, R5
|
|
BLO one_byte_repeat_emit_encodeBlockAsm10B
|
|
CMPW $0x00000100, R5
|
|
BLO two_bytes_repeat_emit_encodeBlockAsm10B
|
|
BLO three_bytes_repeat_emit_encodeBlockAsm10B
|
|
|
|
three_bytes_repeat_emit_encodeBlockAsm10B:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_repeat_emit_encodeBlockAsm10B
|
|
|
|
two_bytes_repeat_emit_encodeBlockAsm10B:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R5, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R5
|
|
BLO memmove_repeat_emit_encodeBlockAsm10B
|
|
JMP memmove_long_repeat_emit_encodeBlockAsm10B
|
|
|
|
one_byte_repeat_emit_encodeBlockAsm10B:
|
|
LSLW $0x02, R5, R16
|
|
BFI $0, R16, $8, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_repeat_emit_encodeBlockAsm10B:
|
|
ADD R8, R1, R5
|
|
|
|
// genMemMoveShort
|
|
CMP $0x08, R8
|
|
BLS emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_8
|
|
CMP $0x10, R8
|
|
BLS emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_8through16
|
|
CMP $0x20, R8
|
|
BLS emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_17through32
|
|
JMP emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_8:
|
|
MOVD (R9), R10
|
|
MOVD R10, (R1)
|
|
JMP memmove_end_copy_repeat_emit_encodeBlockAsm10B
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_8through16:
|
|
MOVD (R9), R10
|
|
ADD R8, R9, R15
|
|
MOVD -8(R15), R9
|
|
MOVD R10, (R1)
|
|
ADD R8, R1, R15
|
|
MOVD R9, -8(R15)
|
|
JMP memmove_end_copy_repeat_emit_encodeBlockAsm10B
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_17through32:
|
|
FMOVQ (R9), F0
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_repeat_emit_encodeBlockAsm10B
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_33through64:
|
|
FMOVQ (R9), F0
|
|
FMOVQ 16(R9), F1
|
|
ADD R8, R9, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_repeat_emit_encodeBlockAsm10B:
|
|
MOVD R5, R1
|
|
JMP emit_literal_done_repeat_emit_encodeBlockAsm10B
|
|
|
|
memmove_long_repeat_emit_encodeBlockAsm10B:
|
|
ADD R8, R1, R5
|
|
|
|
// genMemMoveLong
|
|
MOVD R9, R10
|
|
MOVD R1, R11
|
|
MOVD R8, R12
|
|
|
|
emit_lit_memmove_long_repeat_emit_encodeBlockAsm10Blarge_big_loop_back:
|
|
FMOVQ (R10), F0
|
|
FMOVQ 16(R10), F1
|
|
FMOVQ F0, (R11)
|
|
FMOVQ F1, 16(R11)
|
|
ADD $0x20, R10, R10
|
|
ADD $0x20, R11, R11
|
|
SUB $0x20, R12, R12
|
|
CMP $0x20, R12
|
|
BHS emit_lit_memmove_long_repeat_emit_encodeBlockAsm10Blarge_big_loop_back
|
|
ADD R8, R9, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R8, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R5, R1
|
|
|
|
emit_literal_done_repeat_emit_encodeBlockAsm10B:
|
|
ADDW $0x05, R2, R2
|
|
MOVWU R2, R5
|
|
MOVWU 24(RSP), R16
|
|
SUBW R16, R5, R5
|
|
MOVD src_len+32(FP), R8
|
|
SUBW R2, R8, R8
|
|
ADD R2, R3, R9
|
|
ADD R5, R3, R5
|
|
|
|
// matchLen
|
|
MOVD $0, R11
|
|
|
|
matchlen_loopback_16_repeat_extend_encodeBlockAsm10B:
|
|
CMPW $0x10, R8
|
|
BLO matchlen_match8_repeat_extend_encodeBlockAsm10B
|
|
MOVD (R9)(R11), R10
|
|
ADD R11, R9, R15
|
|
MOVD 8(R15), R12
|
|
MOVD (R5)(R11), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_8_repeat_extend_encodeBlockAsm10B
|
|
ADD R11, R5, R15
|
|
MOVD 8(R15), R16
|
|
EOR R16, R12, R12
|
|
TST R12, R12
|
|
BNE matchlen_bsf_16repeat_extend_encodeBlockAsm10B
|
|
SUB $16, R8, R8
|
|
MOVWU R8, R8
|
|
ADD $16, R11, R11
|
|
MOVWU R11, R11
|
|
JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm10B
|
|
|
|
matchlen_bsf_16repeat_extend_encodeBlockAsm10B:
|
|
RBIT R12, R12
|
|
CLZ R12, R12
|
|
ASR $0x03, R12, R12
|
|
ADD R12, R11, R11
|
|
ADD $8, R11, R11
|
|
MOVWU R11, R11
|
|
JMP repeat_extend_forward_end_encodeBlockAsm10B
|
|
|
|
matchlen_match8_repeat_extend_encodeBlockAsm10B:
|
|
CMPW $0x08, R8
|
|
BLO matchlen_match4_repeat_extend_encodeBlockAsm10B
|
|
MOVD (R9)(R11), R10
|
|
MOVD (R5)(R11), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_8_repeat_extend_encodeBlockAsm10B
|
|
SUB $8, R8, R8
|
|
MOVWU R8, R8
|
|
ADD $8, R11, R11
|
|
MOVWU R11, R11
|
|
JMP matchlen_match4_repeat_extend_encodeBlockAsm10B
|
|
|
|
matchlen_bsf_8_repeat_extend_encodeBlockAsm10B:
|
|
RBIT R10, R10
|
|
CLZ R10, R10
|
|
ASR $0x03, R10, R10
|
|
ADD R10, R11, R11
|
|
MOVWU R11, R11
|
|
JMP repeat_extend_forward_end_encodeBlockAsm10B
|
|
|
|
matchlen_match4_repeat_extend_encodeBlockAsm10B:
|
|
CMPW $0x04, R8
|
|
BLO matchlen_match2_repeat_extend_encodeBlockAsm10B
|
|
MOVWU (R9)(R11), R10
|
|
MOVWU (R5)(R11), R16
|
|
CMPW R10, R16
|
|
BNE matchlen_match2_repeat_extend_encodeBlockAsm10B
|
|
SUB $4, R8, R8
|
|
MOVWU R8, R8
|
|
ADD $4, R11, R11
|
|
MOVWU R11, R11
|
|
|
|
matchlen_match2_repeat_extend_encodeBlockAsm10B:
|
|
CMPW $0x01, R8
|
|
BEQ matchlen_match1_repeat_extend_encodeBlockAsm10B
|
|
BLO repeat_extend_forward_end_encodeBlockAsm10B
|
|
MOVHU (R9)(R11), R16
|
|
BFI $0, R16, $16, R10
|
|
ADD R11, R5, R15
|
|
MOVHU (R15), R15
|
|
AND $0xffff, R10, R16
|
|
CMP R16, R15
|
|
BNE matchlen_match1_repeat_extend_encodeBlockAsm10B
|
|
ADD $2, R11, R11
|
|
MOVWU R11, R11
|
|
SUBSW $0x02, R8, R8
|
|
BEQ repeat_extend_forward_end_encodeBlockAsm10B
|
|
|
|
matchlen_match1_repeat_extend_encodeBlockAsm10B:
|
|
MOVBU (R9)(R11), R16
|
|
BFI $0, R16, $8, R10
|
|
ADD R11, R5, R15
|
|
MOVBU (R15), R15
|
|
AND $0xff, R10, R16
|
|
CMP R16, R15
|
|
BNE repeat_extend_forward_end_encodeBlockAsm10B
|
|
ADD $1, R11, R11
|
|
MOVWU R11, R11
|
|
|
|
repeat_extend_forward_end_encodeBlockAsm10B:
|
|
ADDW R11, R2, R2
|
|
MOVWU R2, R5
|
|
SUBW R6, R5, R5
|
|
MOVWU 24(RSP), R6
|
|
TSTW R7, R7
|
|
BEQ repeat_as_copy_encodeBlockAsm10B
|
|
|
|
// emitRepeat
|
|
MOVWU R5, R7
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x08, R7
|
|
BLS repeat_two_match_repeat_encodeBlockAsm10B
|
|
CMPW $0x0c, R7
|
|
BHS cant_repeat_two_offset_match_repeat_encodeBlockAsm10B
|
|
CMPW $0x00000800, R6
|
|
BLO repeat_two_offset_match_repeat_encodeBlockAsm10B
|
|
|
|
cant_repeat_two_offset_match_repeat_encodeBlockAsm10B:
|
|
CMPW $0x00000104, R5
|
|
BLO repeat_three_match_repeat_encodeBlockAsm10B
|
|
SUB $256, R5, R5
|
|
MOVWU R5, R5
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R5, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm10B
|
|
|
|
repeat_three_match_repeat_encodeBlockAsm10B:
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R5, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm10B
|
|
|
|
repeat_two_match_repeat_encodeBlockAsm10B:
|
|
LSLW $0x02, R5, R5
|
|
ORRW $0x01, R5, R5
|
|
MOVH R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm10B
|
|
|
|
repeat_two_offset_match_repeat_encodeBlockAsm10B:
|
|
MOVD $0, R7
|
|
ADD R5<<2, R7, R5
|
|
ADD $1, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R6, 1(R1)
|
|
ASRW $0x08, R6, R6
|
|
LSLW $0x05, R6, R6
|
|
ORRW R6, R5, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm10B
|
|
|
|
repeat_as_copy_encodeBlockAsm10B:
|
|
// emitCopy
|
|
CMPW $0x40, R5
|
|
BLS two_byte_offset_short_repeat_as_copy_encodeBlockAsm10B
|
|
CMPW $0x00000800, R6
|
|
BHS long_offset_short_repeat_as_copy_encodeBlockAsm10B
|
|
MOVD $0x00000001, R7
|
|
ADD $16, R7, R7
|
|
MOVWU R7, R7
|
|
MOVB R6, 1(R1)
|
|
LSRW $0x08, R6, R6
|
|
LSLW $0x05, R6, R6
|
|
ORRW R6, R7, R7
|
|
MOVB R7, (R1)
|
|
ADD $0x02, R1, R1
|
|
SUBW $0x08, R5, R5
|
|
|
|
// emitRepeat
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
JMP cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b
|
|
MOVWU R5, R7
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x08, R7
|
|
BLS repeat_two_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b
|
|
CMPW $0x0c, R7
|
|
BHS cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b
|
|
CMPW $0x00000800, R6
|
|
BLO repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b
|
|
|
|
cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b:
|
|
CMPW $0x00000104, R5
|
|
BLO repeat_three_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b
|
|
SUB $256, R5, R5
|
|
MOVWU R5, R5
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R5, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm10B
|
|
|
|
repeat_three_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b:
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R5, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm10B
|
|
|
|
repeat_two_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b:
|
|
LSLW $0x02, R5, R5
|
|
ORRW $0x01, R5, R5
|
|
MOVH R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm10B
|
|
|
|
repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b:
|
|
MOVD $0, R7
|
|
ADD R5<<2, R7, R5
|
|
ADD $1, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R6, 1(R1)
|
|
ASRW $0x08, R6, R6
|
|
LSLW $0x05, R6, R6
|
|
ORRW R6, R5, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm10B
|
|
|
|
long_offset_short_repeat_as_copy_encodeBlockAsm10B:
|
|
MOVD $0xee, R16
|
|
MOVB R16, (R1)
|
|
MOVH R6, 1(R1)
|
|
SUB $60, R5, R5
|
|
MOVWU R5, R5
|
|
ADD $0x03, R1, R1
|
|
|
|
// emitRepeat
|
|
MOVWU R5, R7
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x08, R7
|
|
BLS repeat_two_repeat_as_copy_encodeBlockAsm10B_emit_copy_short
|
|
CMPW $0x0c, R7
|
|
BHS cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short
|
|
CMPW $0x00000800, R6
|
|
BLO repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short
|
|
|
|
cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short:
|
|
CMPW $0x00000104, R5
|
|
BLO repeat_three_repeat_as_copy_encodeBlockAsm10B_emit_copy_short
|
|
SUB $256, R5, R5
|
|
MOVWU R5, R5
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R5, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm10B
|
|
|
|
repeat_three_repeat_as_copy_encodeBlockAsm10B_emit_copy_short:
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R5, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm10B
|
|
|
|
repeat_two_repeat_as_copy_encodeBlockAsm10B_emit_copy_short:
|
|
LSLW $0x02, R5, R5
|
|
ORRW $0x01, R5, R5
|
|
MOVH R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm10B
|
|
|
|
repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short:
|
|
MOVD $0, R7
|
|
ADD R5<<2, R7, R5
|
|
ADD $1, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R6, 1(R1)
|
|
ASRW $0x08, R6, R6
|
|
LSLW $0x05, R6, R6
|
|
ORRW R6, R5, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm10B
|
|
|
|
two_byte_offset_short_repeat_as_copy_encodeBlockAsm10B:
|
|
MOVWU R5, R7
|
|
LSLW $0x02, R7, R7
|
|
CMPW $0x0c, R5
|
|
BHS emit_copy_three_repeat_as_copy_encodeBlockAsm10B
|
|
CMPW $0x00000800, R6
|
|
BHS emit_copy_three_repeat_as_copy_encodeBlockAsm10B
|
|
SUB $15, R7, R7
|
|
MOVWU R7, R7
|
|
MOVB R6, 1(R1)
|
|
LSRW $0x08, R6, R6
|
|
LSLW $0x05, R6, R6
|
|
ORRW R6, R7, R7
|
|
MOVB R7, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm10B
|
|
|
|
emit_copy_three_repeat_as_copy_encodeBlockAsm10B:
|
|
SUB $2, R7, R7
|
|
MOVWU R7, R7
|
|
MOVB R7, (R1)
|
|
MOVH R6, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
|
|
repeat_end_emit_encodeBlockAsm10B:
|
|
MOVW R2, 20(RSP)
|
|
JMP search_loop_encodeBlockAsm10B
|
|
|
|
no_repeat_found_encodeBlockAsm10B:
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ candidate_match_encodeBlockAsm10B
|
|
LSR $0x08, R6, R6
|
|
MOVWU (R0)(R9<<2), R5
|
|
ADD $2, R2, R8
|
|
MOVWU R8, R8
|
|
MOVWU (R3)(R7), R16
|
|
CMPW R6, R16
|
|
BEQ candidate2_match_encodeBlockAsm10B
|
|
MOVW R8, (R0)(R9<<2)
|
|
LSR $0x08, R6, R6
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ candidate3_match_encodeBlockAsm10B
|
|
MOVWU 28(RSP), R2
|
|
JMP search_loop_encodeBlockAsm10B
|
|
|
|
candidate3_match_encodeBlockAsm10B:
|
|
ADDW $0x02, R2, R2
|
|
JMP candidate_match_encodeBlockAsm10B
|
|
|
|
candidate2_match_encodeBlockAsm10B:
|
|
MOVW R8, (R0)(R9<<2)
|
|
ADDW $1, R2, R2
|
|
MOVWU R7, R5
|
|
|
|
candidate_match_encodeBlockAsm10B:
|
|
MOVWU 20(RSP), R6
|
|
TSTW R5, R5
|
|
BEQ match_extend_back_end_encodeBlockAsm10B
|
|
|
|
match_extend_back_loop_encodeBlockAsm10B:
|
|
CMPW R6, R2
|
|
BLS match_extend_back_end_encodeBlockAsm10B
|
|
ADD R5, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R7
|
|
ADD R2, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R8
|
|
AND $0xff, R8, R16
|
|
AND $0xff, R7, R15
|
|
CMP R16, R15
|
|
BNE match_extend_back_end_encodeBlockAsm10B
|
|
SUB $1, R2, R2
|
|
MOVWU R2, R2
|
|
SUBSW $1, R5, R5
|
|
BEQ match_extend_back_end_encodeBlockAsm10B
|
|
JMP match_extend_back_loop_encodeBlockAsm10B
|
|
|
|
match_extend_back_end_encodeBlockAsm10B:
|
|
MOVWU R2, R6
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R6, R6
|
|
ADD R6, R1, R6
|
|
ADD $3, R6, R6
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R6
|
|
BLO match_dst_size_check_encodeBlockAsm10B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeBlockAsm10B:
|
|
MOVWU R2, R6
|
|
MOVWU 20(RSP), R7
|
|
CMPW R6, R7
|
|
BEQ emit_literal_done_match_emit_encodeBlockAsm10B
|
|
MOVWU R6, R8
|
|
MOVW R6, 20(RSP)
|
|
ADD R7, R3, R6
|
|
SUBW R7, R8, R8
|
|
SUB $1, R8, R7
|
|
MOVWU R7, R7
|
|
CMPW $0x3c, R7
|
|
BLO one_byte_match_emit_encodeBlockAsm10B
|
|
CMPW $0x00000100, R7
|
|
BLO two_bytes_match_emit_encodeBlockAsm10B
|
|
BLO three_bytes_match_emit_encodeBlockAsm10B
|
|
|
|
three_bytes_match_emit_encodeBlockAsm10B:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R7, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_match_emit_encodeBlockAsm10B
|
|
|
|
two_bytes_match_emit_encodeBlockAsm10B:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R7, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R7
|
|
BLO memmove_match_emit_encodeBlockAsm10B
|
|
JMP memmove_long_match_emit_encodeBlockAsm10B
|
|
|
|
one_byte_match_emit_encodeBlockAsm10B:
|
|
LSLW $0x02, R7, R16
|
|
BFI $0, R16, $8, R7
|
|
MOVB R7, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_match_emit_encodeBlockAsm10B:
|
|
ADD R8, R1, R7
|
|
|
|
// genMemMoveShort
|
|
CMP $0x08, R8
|
|
BLS emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_8
|
|
CMP $0x10, R8
|
|
BLS emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_8through16
|
|
CMP $0x20, R8
|
|
BLS emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_8:
|
|
MOVD (R6), R9
|
|
MOVD R9, (R1)
|
|
JMP memmove_end_copy_match_emit_encodeBlockAsm10B
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_8through16:
|
|
MOVD (R6), R9
|
|
ADD R8, R6, R15
|
|
MOVD -8(R15), R6
|
|
MOVD R9, (R1)
|
|
ADD R8, R1, R15
|
|
MOVD R6, -8(R15)
|
|
JMP memmove_end_copy_match_emit_encodeBlockAsm10B
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_17through32:
|
|
FMOVQ (R6), F0
|
|
ADD R8, R6, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_match_emit_encodeBlockAsm10B
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_33through64:
|
|
FMOVQ (R6), F0
|
|
FMOVQ 16(R6), F1
|
|
ADD R8, R6, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R8, R6, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_match_emit_encodeBlockAsm10B:
|
|
MOVD R7, R1
|
|
JMP emit_literal_done_match_emit_encodeBlockAsm10B
|
|
|
|
memmove_long_match_emit_encodeBlockAsm10B:
|
|
ADD R8, R1, R7
|
|
|
|
// genMemMoveLong
|
|
MOVD R6, R9
|
|
MOVD R1, R10
|
|
MOVD R8, R11
|
|
|
|
emit_lit_memmove_long_match_emit_encodeBlockAsm10Blarge_big_loop_back:
|
|
FMOVQ (R9), F0
|
|
FMOVQ 16(R9), F1
|
|
FMOVQ F0, (R10)
|
|
FMOVQ F1, 16(R10)
|
|
ADD $0x20, R9, R9
|
|
ADD $0x20, R10, R10
|
|
SUB $0x20, R11, R11
|
|
CMP $0x20, R11
|
|
BHS emit_lit_memmove_long_match_emit_encodeBlockAsm10Blarge_big_loop_back
|
|
ADD R8, R6, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R8, R6, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R8, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R7, R1
|
|
|
|
emit_literal_done_match_emit_encodeBlockAsm10B:
|
|
match_nolit_loop_encodeBlockAsm10B:
|
|
MOVWU R2, R6
|
|
SUBW R5, R6, R6
|
|
MOVW R6, 24(RSP)
|
|
ADDW $0x04, R2, R2
|
|
ADDW $0x04, R5, R5
|
|
MOVD src_len+32(FP), R6
|
|
SUBW R2, R6, R6
|
|
ADD R2, R3, R7
|
|
ADD R5, R3, R5
|
|
|
|
// matchLen
|
|
MOVD $0, R9
|
|
|
|
matchlen_loopback_16_match_nolit_encodeBlockAsm10B:
|
|
CMPW $0x10, R6
|
|
BLO matchlen_match8_match_nolit_encodeBlockAsm10B
|
|
MOVD (R7)(R9), R8
|
|
ADD R9, R7, R15
|
|
MOVD 8(R15), R10
|
|
MOVD (R5)(R9), R16
|
|
EOR R16, R8, R8
|
|
TST R8, R8
|
|
BNE matchlen_bsf_8_match_nolit_encodeBlockAsm10B
|
|
ADD R9, R5, R15
|
|
MOVD 8(R15), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_16match_nolit_encodeBlockAsm10B
|
|
SUB $16, R6, R6
|
|
MOVWU R6, R6
|
|
ADD $16, R9, R9
|
|
MOVWU R9, R9
|
|
JMP matchlen_loopback_16_match_nolit_encodeBlockAsm10B
|
|
|
|
matchlen_bsf_16match_nolit_encodeBlockAsm10B:
|
|
RBIT R10, R10
|
|
CLZ R10, R10
|
|
ASR $0x03, R10, R10
|
|
ADD R10, R9, R9
|
|
ADD $8, R9, R9
|
|
MOVWU R9, R9
|
|
JMP match_nolit_end_encodeBlockAsm10B
|
|
|
|
matchlen_match8_match_nolit_encodeBlockAsm10B:
|
|
CMPW $0x08, R6
|
|
BLO matchlen_match4_match_nolit_encodeBlockAsm10B
|
|
MOVD (R7)(R9), R8
|
|
MOVD (R5)(R9), R16
|
|
EOR R16, R8, R8
|
|
TST R8, R8
|
|
BNE matchlen_bsf_8_match_nolit_encodeBlockAsm10B
|
|
SUB $8, R6, R6
|
|
MOVWU R6, R6
|
|
ADD $8, R9, R9
|
|
MOVWU R9, R9
|
|
JMP matchlen_match4_match_nolit_encodeBlockAsm10B
|
|
|
|
matchlen_bsf_8_match_nolit_encodeBlockAsm10B:
|
|
RBIT R8, R8
|
|
CLZ R8, R8
|
|
ASR $0x03, R8, R8
|
|
ADD R8, R9, R9
|
|
MOVWU R9, R9
|
|
JMP match_nolit_end_encodeBlockAsm10B
|
|
|
|
matchlen_match4_match_nolit_encodeBlockAsm10B:
|
|
CMPW $0x04, R6
|
|
BLO matchlen_match2_match_nolit_encodeBlockAsm10B
|
|
MOVWU (R7)(R9), R8
|
|
MOVWU (R5)(R9), R16
|
|
CMPW R8, R16
|
|
BNE matchlen_match2_match_nolit_encodeBlockAsm10B
|
|
SUB $4, R6, R6
|
|
MOVWU R6, R6
|
|
ADD $4, R9, R9
|
|
MOVWU R9, R9
|
|
|
|
matchlen_match2_match_nolit_encodeBlockAsm10B:
|
|
CMPW $0x01, R6
|
|
BEQ matchlen_match1_match_nolit_encodeBlockAsm10B
|
|
BLO match_nolit_end_encodeBlockAsm10B
|
|
MOVHU (R7)(R9), R16
|
|
BFI $0, R16, $16, R8
|
|
ADD R9, R5, R15
|
|
MOVHU (R15), R15
|
|
AND $0xffff, R8, R16
|
|
CMP R16, R15
|
|
BNE matchlen_match1_match_nolit_encodeBlockAsm10B
|
|
ADD $2, R9, R9
|
|
MOVWU R9, R9
|
|
SUBSW $0x02, R6, R6
|
|
BEQ match_nolit_end_encodeBlockAsm10B
|
|
|
|
matchlen_match1_match_nolit_encodeBlockAsm10B:
|
|
MOVBU (R7)(R9), R16
|
|
BFI $0, R16, $8, R8
|
|
ADD R9, R5, R15
|
|
MOVBU (R15), R15
|
|
AND $0xff, R8, R16
|
|
CMP R16, R15
|
|
BNE match_nolit_end_encodeBlockAsm10B
|
|
ADD $1, R9, R9
|
|
MOVWU R9, R9
|
|
|
|
match_nolit_end_encodeBlockAsm10B:
|
|
ADDW R9, R2, R2
|
|
MOVWU 24(RSP), R5
|
|
ADDW $0x04, R9, R9
|
|
MOVW R2, 20(RSP)
|
|
|
|
// emitCopy
|
|
CMPW $0x40, R9
|
|
BLS two_byte_offset_short_match_nolit_encodeBlockAsm10B
|
|
CMPW $0x00000800, R5
|
|
BHS long_offset_short_match_nolit_encodeBlockAsm10B
|
|
MOVD $0x00000001, R6
|
|
ADD $16, R6, R6
|
|
MOVWU R6, R6
|
|
MOVB R5, 1(R1)
|
|
LSRW $0x08, R5, R5
|
|
LSLW $0x05, R5, R5
|
|
ORRW R5, R6, R6
|
|
MOVB R6, (R1)
|
|
ADD $0x02, R1, R1
|
|
SUBW $0x08, R9, R9
|
|
|
|
// emitRepeat
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
JMP cant_repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short_2b
|
|
MOVWU R9, R6
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
CMPW $0x08, R6
|
|
BLS repeat_two_match_nolit_encodeBlockAsm10B_emit_copy_short_2b
|
|
CMPW $0x0c, R6
|
|
BHS cant_repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short_2b
|
|
CMPW $0x00000800, R5
|
|
BLO repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short_2b
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short_2b:
|
|
CMPW $0x00000104, R9
|
|
BLO repeat_three_match_nolit_encodeBlockAsm10B_emit_copy_short_2b
|
|
SUB $256, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R9, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm10B
|
|
|
|
repeat_three_match_nolit_encodeBlockAsm10B_emit_copy_short_2b:
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R9, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm10B
|
|
|
|
repeat_two_match_nolit_encodeBlockAsm10B_emit_copy_short_2b:
|
|
LSLW $0x02, R9, R9
|
|
ORRW $0x01, R9, R9
|
|
MOVH R9, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm10B
|
|
|
|
repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short_2b:
|
|
MOVD $0, R6
|
|
ADD R9<<2, R6, R9
|
|
ADD $1, R9, R9
|
|
MOVWU R9, R9
|
|
MOVB R5, 1(R1)
|
|
ASRW $0x08, R5, R5
|
|
LSLW $0x05, R5, R5
|
|
ORRW R5, R9, R9
|
|
MOVB R9, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm10B
|
|
|
|
long_offset_short_match_nolit_encodeBlockAsm10B:
|
|
MOVD $0xee, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
SUB $60, R9, R9
|
|
MOVWU R9, R9
|
|
ADD $0x03, R1, R1
|
|
|
|
// emitRepeat
|
|
MOVWU R9, R6
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
CMPW $0x08, R6
|
|
BLS repeat_two_match_nolit_encodeBlockAsm10B_emit_copy_short
|
|
CMPW $0x0c, R6
|
|
BHS cant_repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short
|
|
CMPW $0x00000800, R5
|
|
BLO repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short:
|
|
CMPW $0x00000104, R9
|
|
BLO repeat_three_match_nolit_encodeBlockAsm10B_emit_copy_short
|
|
SUB $256, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R9, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm10B
|
|
|
|
repeat_three_match_nolit_encodeBlockAsm10B_emit_copy_short:
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R9, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm10B
|
|
|
|
repeat_two_match_nolit_encodeBlockAsm10B_emit_copy_short:
|
|
LSLW $0x02, R9, R9
|
|
ORRW $0x01, R9, R9
|
|
MOVH R9, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm10B
|
|
|
|
repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short:
|
|
MOVD $0, R6
|
|
ADD R9<<2, R6, R9
|
|
ADD $1, R9, R9
|
|
MOVWU R9, R9
|
|
MOVB R5, 1(R1)
|
|
ASRW $0x08, R5, R5
|
|
LSLW $0x05, R5, R5
|
|
ORRW R5, R9, R9
|
|
MOVB R9, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm10B
|
|
|
|
two_byte_offset_short_match_nolit_encodeBlockAsm10B:
|
|
MOVWU R9, R6
|
|
LSLW $0x02, R6, R6
|
|
CMPW $0x0c, R9
|
|
BHS emit_copy_three_match_nolit_encodeBlockAsm10B
|
|
CMPW $0x00000800, R5
|
|
BHS emit_copy_three_match_nolit_encodeBlockAsm10B
|
|
SUB $15, R6, R6
|
|
MOVWU R6, R6
|
|
MOVB R5, 1(R1)
|
|
LSRW $0x08, R5, R5
|
|
LSLW $0x05, R5, R5
|
|
ORRW R5, R6, R6
|
|
MOVB R6, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm10B
|
|
|
|
emit_copy_three_match_nolit_encodeBlockAsm10B:
|
|
SUB $2, R6, R6
|
|
MOVWU R6, R6
|
|
MOVB R6, (R1)
|
|
MOVH R5, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
|
|
match_nolit_emitcopy_end_encodeBlockAsm10B:
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R2
|
|
BHS emit_remainder_encodeBlockAsm10B
|
|
ADD R2, R3, R15
|
|
MOVD -2(R15), R6
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R1
|
|
BLO match_nolit_dst_ok_encodeBlockAsm10B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeBlockAsm10B:
|
|
MOVD $0x9e3779b1, R8
|
|
MOVD R6, R7
|
|
LSR $0x10, R6, R6
|
|
MOVD R6, R5
|
|
LSL $0x20, R7, R7
|
|
MUL R8, R7, R7
|
|
LSR $0x36, R7, R7
|
|
LSL $0x20, R5, R5
|
|
MUL R8, R5, R5
|
|
LSR $0x36, R5, R5
|
|
SUB $2, R2, R8
|
|
MOVWU R8, R8
|
|
ADD R5<<2, R0, R9
|
|
MOVWU (R9), R5
|
|
MOVW R8, (R0)(R7<<2)
|
|
MOVW R2, (R9)
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ match_nolit_loop_encodeBlockAsm10B
|
|
ADDW $1, R2, R2
|
|
JMP search_loop_encodeBlockAsm10B
|
|
|
|
emit_remainder_encodeBlockAsm10B:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R0, R0
|
|
ADD R0, R1, R0
|
|
ADD $3, R0, R0
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R0
|
|
BLO emit_remainder_ok_encodeBlockAsm10B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeBlockAsm10B:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R2
|
|
CMPW R0, R2
|
|
BEQ emit_literal_done_emit_remainder_encodeBlockAsm10B
|
|
MOVWU R0, R5
|
|
MOVW R0, 20(RSP)
|
|
ADD R2, R3, R0
|
|
SUBW R2, R5, R5
|
|
SUB $1, R5, R2
|
|
MOVWU R2, R2
|
|
CMPW $0x3c, R2
|
|
BLO one_byte_emit_remainder_encodeBlockAsm10B
|
|
CMPW $0x00000100, R2
|
|
BLO two_bytes_emit_remainder_encodeBlockAsm10B
|
|
BLO three_bytes_emit_remainder_encodeBlockAsm10B
|
|
|
|
three_bytes_emit_remainder_encodeBlockAsm10B:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R2, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_emit_remainder_encodeBlockAsm10B
|
|
|
|
two_bytes_emit_remainder_encodeBlockAsm10B:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R2, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R2
|
|
BLO memmove_emit_remainder_encodeBlockAsm10B
|
|
JMP memmove_long_emit_remainder_encodeBlockAsm10B
|
|
|
|
one_byte_emit_remainder_encodeBlockAsm10B:
|
|
LSLW $0x02, R2, R16
|
|
BFI $0, R16, $8, R2
|
|
MOVB R2, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_emit_remainder_encodeBlockAsm10B:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveShort
|
|
CMP $0x03, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_1or2
|
|
BEQ emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_3
|
|
CMP $0x08, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_4through7
|
|
CMP $0x10, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_8through16
|
|
CMP $0x20, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_1or2:
|
|
MOVBU (R0), R16
|
|
BFI $0, R16, $8, R5
|
|
ADD R3, R0, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVB R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVB R0, -1(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_3:
|
|
MOVHU (R0), R16
|
|
BFI $0, R16, $16, R5
|
|
MOVBU 2(R0), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVH R5, (R1)
|
|
MOVB R0, 2(R1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_4through7:
|
|
MOVWU (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVWU -4(R15), R0
|
|
MOVW R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVW R0, -4(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_8through16:
|
|
MOVD (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVD -8(R15), R0
|
|
MOVD R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVD R0, -8(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_17through32:
|
|
FMOVQ (R0), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_33through64:
|
|
FMOVQ (R0), F0
|
|
FMOVQ 16(R0), F1
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_emit_remainder_encodeBlockAsm10B:
|
|
MOVD R2, R1
|
|
JMP emit_literal_done_emit_remainder_encodeBlockAsm10B
|
|
|
|
memmove_long_emit_remainder_encodeBlockAsm10B:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveLong
|
|
MOVD R0, R5
|
|
MOVD R1, R6
|
|
MOVD R3, R7
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeBlockAsm10Blarge_big_loop_back:
|
|
FMOVQ (R5), F0
|
|
FMOVQ 16(R5), F1
|
|
FMOVQ F0, (R6)
|
|
FMOVQ F1, 16(R6)
|
|
ADD $0x20, R5, R5
|
|
ADD $0x20, R6, R6
|
|
SUB $0x20, R7, R7
|
|
CMP $0x20, R7
|
|
BHS emit_lit_memmove_long_emit_remainder_encodeBlockAsm10Blarge_big_loop_back
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R3, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R2, R1
|
|
|
|
emit_literal_done_emit_remainder_encodeBlockAsm10B:
|
|
MOVD dst_base+0(FP), R0
|
|
SUB R0, R1, R1
|
|
MOVD R1, ret+56(FP)
|
|
RET
|
|
|
|
// func encodeBlockAsm8B(dst []byte, src []byte, tmp *[1024]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeBlockAsm8B(SB), $24-64
|
|
MOVD tmp+48(FP), R0
|
|
MOVD dst_base+0(FP), R1
|
|
MOVD $0x00000008, R2
|
|
MOVD R0, R3
|
|
VEOR V0.B16, V0.B16, V0.B16
|
|
|
|
zero_loop_encodeBlockAsm8B:
|
|
FMOVQ F0, (R3)
|
|
FMOVQ F0, 16(R3)
|
|
FMOVQ F0, 32(R3)
|
|
FMOVQ F0, 48(R3)
|
|
FMOVQ F0, 64(R3)
|
|
FMOVQ F0, 80(R3)
|
|
FMOVQ F0, 96(R3)
|
|
FMOVQ F0, 112(R3)
|
|
ADD $0x80, R3, R3
|
|
SUBS $1, R2, R2
|
|
BNE zero_loop_encodeBlockAsm8B
|
|
MOVD $0x00000000, R16
|
|
MOVW R16, 20(RSP)
|
|
MOVD src_len+32(FP), R2
|
|
SUB $9, R2, R3
|
|
SUB $8, R2, R5
|
|
MOVW R5, 16(RSP)
|
|
LSR $0x05, R2, R2
|
|
SUBW R2, R3, R3
|
|
ADD R3, R1, R3
|
|
MOVD R3, 8(RSP)
|
|
MOVD $0x00000001, R2
|
|
MOVW R2, 24(RSP)
|
|
MOVD src_base+24(FP), R3
|
|
|
|
search_loop_encodeBlockAsm8B:
|
|
MOVWU R2, R5
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R5, R5
|
|
LSRW $0x04, R5, R5
|
|
ADD R5, R2, R5
|
|
ADD $4, R5, R5
|
|
MOVWU R5, R5
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R5
|
|
BHS emit_remainder_encodeBlockAsm8B
|
|
MOVD (R3)(R2), R6
|
|
MOVW R5, 28(RSP)
|
|
MOVD $0x9e3779b1, R8
|
|
MOVD R6, R9
|
|
MOVD R6, R10
|
|
LSR $0x08, R10, R10
|
|
LSL $0x20, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x38, R9, R9
|
|
LSL $0x20, R10, R10
|
|
MUL R8, R10, R10
|
|
LSR $0x38, R10, R10
|
|
MOVWU (R0)(R9<<2), R5
|
|
MOVWU (R0)(R10<<2), R7
|
|
MOVW R2, (R0)(R9<<2)
|
|
ADD $1, R2, R9
|
|
MOVWU R9, R9
|
|
MOVW R9, (R0)(R10<<2)
|
|
MOVD R6, R9
|
|
LSR $0x10, R9, R9
|
|
LSL $0x20, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x38, R9, R9
|
|
MOVWU R2, R8
|
|
MOVWU 24(RSP), R16
|
|
SUBW R16, R8, R8
|
|
ADD R8, R3, R15
|
|
MOVWU 1(R15), R10
|
|
MOVD R6, R8
|
|
LSR $0x08, R8, R8
|
|
CMPW R10, R8
|
|
BNE no_repeat_found_encodeBlockAsm8B
|
|
ADD $1, R2, R6
|
|
MOVWU R6, R6
|
|
MOVWU 20(RSP), R7
|
|
MOVWU R6, R5
|
|
MOVWU 24(RSP), R16
|
|
SUBSW R16, R5, R5
|
|
BEQ repeat_extend_back_end_encodeBlockAsm8B
|
|
|
|
repeat_extend_back_loop_encodeBlockAsm8B:
|
|
CMPW R7, R6
|
|
BLS repeat_extend_back_end_encodeBlockAsm8B
|
|
ADD R5, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R8
|
|
ADD R6, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R9
|
|
AND $0xff, R9, R16
|
|
AND $0xff, R8, R15
|
|
CMP R16, R15
|
|
BNE repeat_extend_back_end_encodeBlockAsm8B
|
|
SUB $1, R6, R6
|
|
MOVWU R6, R6
|
|
SUBSW $1, R5, R5
|
|
BNE repeat_extend_back_loop_encodeBlockAsm8B
|
|
|
|
repeat_extend_back_end_encodeBlockAsm8B:
|
|
MOVWU R6, R5
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R5, R5
|
|
ADD R5, R1, R5
|
|
ADD $3, R5, R5
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R5
|
|
BLO repeat_dst_size_check_encodeBlockAsm8B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
repeat_dst_size_check_encodeBlockAsm8B:
|
|
MOVWU 20(RSP), R5
|
|
CMPW R6, R5
|
|
BEQ emit_literal_done_repeat_emit_encodeBlockAsm8B
|
|
MOVWU R6, R8
|
|
MOVW R6, 20(RSP)
|
|
ADD R5, R3, R9
|
|
SUBW R5, R8, R8
|
|
SUB $1, R8, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x3c, R5
|
|
BLO one_byte_repeat_emit_encodeBlockAsm8B
|
|
CMPW $0x00000100, R5
|
|
BLO two_bytes_repeat_emit_encodeBlockAsm8B
|
|
BLO three_bytes_repeat_emit_encodeBlockAsm8B
|
|
|
|
three_bytes_repeat_emit_encodeBlockAsm8B:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_repeat_emit_encodeBlockAsm8B
|
|
|
|
two_bytes_repeat_emit_encodeBlockAsm8B:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R5, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R5
|
|
BLO memmove_repeat_emit_encodeBlockAsm8B
|
|
JMP memmove_long_repeat_emit_encodeBlockAsm8B
|
|
|
|
one_byte_repeat_emit_encodeBlockAsm8B:
|
|
LSLW $0x02, R5, R16
|
|
BFI $0, R16, $8, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_repeat_emit_encodeBlockAsm8B:
|
|
ADD R8, R1, R5
|
|
|
|
// genMemMoveShort
|
|
CMP $0x08, R8
|
|
BLS emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_8
|
|
CMP $0x10, R8
|
|
BLS emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_8through16
|
|
CMP $0x20, R8
|
|
BLS emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_17through32
|
|
JMP emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_8:
|
|
MOVD (R9), R10
|
|
MOVD R10, (R1)
|
|
JMP memmove_end_copy_repeat_emit_encodeBlockAsm8B
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_8through16:
|
|
MOVD (R9), R10
|
|
ADD R8, R9, R15
|
|
MOVD -8(R15), R9
|
|
MOVD R10, (R1)
|
|
ADD R8, R1, R15
|
|
MOVD R9, -8(R15)
|
|
JMP memmove_end_copy_repeat_emit_encodeBlockAsm8B
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_17through32:
|
|
FMOVQ (R9), F0
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_repeat_emit_encodeBlockAsm8B
|
|
|
|
emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_33through64:
|
|
FMOVQ (R9), F0
|
|
FMOVQ 16(R9), F1
|
|
ADD R8, R9, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_repeat_emit_encodeBlockAsm8B:
|
|
MOVD R5, R1
|
|
JMP emit_literal_done_repeat_emit_encodeBlockAsm8B
|
|
|
|
memmove_long_repeat_emit_encodeBlockAsm8B:
|
|
ADD R8, R1, R5
|
|
|
|
// genMemMoveLong
|
|
MOVD R9, R10
|
|
MOVD R1, R11
|
|
MOVD R8, R12
|
|
|
|
emit_lit_memmove_long_repeat_emit_encodeBlockAsm8Blarge_big_loop_back:
|
|
FMOVQ (R10), F0
|
|
FMOVQ 16(R10), F1
|
|
FMOVQ F0, (R11)
|
|
FMOVQ F1, 16(R11)
|
|
ADD $0x20, R10, R10
|
|
ADD $0x20, R11, R11
|
|
SUB $0x20, R12, R12
|
|
CMP $0x20, R12
|
|
BHS emit_lit_memmove_long_repeat_emit_encodeBlockAsm8Blarge_big_loop_back
|
|
ADD R8, R9, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R8, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R5, R1
|
|
|
|
emit_literal_done_repeat_emit_encodeBlockAsm8B:
|
|
ADDW $0x05, R2, R2
|
|
MOVWU R2, R5
|
|
MOVWU 24(RSP), R16
|
|
SUBW R16, R5, R5
|
|
MOVD src_len+32(FP), R8
|
|
SUBW R2, R8, R8
|
|
ADD R2, R3, R9
|
|
ADD R5, R3, R5
|
|
|
|
// matchLen
|
|
MOVD $0, R11
|
|
|
|
matchlen_loopback_16_repeat_extend_encodeBlockAsm8B:
|
|
CMPW $0x10, R8
|
|
BLO matchlen_match8_repeat_extend_encodeBlockAsm8B
|
|
MOVD (R9)(R11), R10
|
|
ADD R11, R9, R15
|
|
MOVD 8(R15), R12
|
|
MOVD (R5)(R11), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_8_repeat_extend_encodeBlockAsm8B
|
|
ADD R11, R5, R15
|
|
MOVD 8(R15), R16
|
|
EOR R16, R12, R12
|
|
TST R12, R12
|
|
BNE matchlen_bsf_16repeat_extend_encodeBlockAsm8B
|
|
SUB $16, R8, R8
|
|
MOVWU R8, R8
|
|
ADD $16, R11, R11
|
|
MOVWU R11, R11
|
|
JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm8B
|
|
|
|
matchlen_bsf_16repeat_extend_encodeBlockAsm8B:
|
|
RBIT R12, R12
|
|
CLZ R12, R12
|
|
ASR $0x03, R12, R12
|
|
ADD R12, R11, R11
|
|
ADD $8, R11, R11
|
|
MOVWU R11, R11
|
|
JMP repeat_extend_forward_end_encodeBlockAsm8B
|
|
|
|
matchlen_match8_repeat_extend_encodeBlockAsm8B:
|
|
CMPW $0x08, R8
|
|
BLO matchlen_match4_repeat_extend_encodeBlockAsm8B
|
|
MOVD (R9)(R11), R10
|
|
MOVD (R5)(R11), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_8_repeat_extend_encodeBlockAsm8B
|
|
SUB $8, R8, R8
|
|
MOVWU R8, R8
|
|
ADD $8, R11, R11
|
|
MOVWU R11, R11
|
|
JMP matchlen_match4_repeat_extend_encodeBlockAsm8B
|
|
|
|
matchlen_bsf_8_repeat_extend_encodeBlockAsm8B:
|
|
RBIT R10, R10
|
|
CLZ R10, R10
|
|
ASR $0x03, R10, R10
|
|
ADD R10, R11, R11
|
|
MOVWU R11, R11
|
|
JMP repeat_extend_forward_end_encodeBlockAsm8B
|
|
|
|
matchlen_match4_repeat_extend_encodeBlockAsm8B:
|
|
CMPW $0x04, R8
|
|
BLO matchlen_match2_repeat_extend_encodeBlockAsm8B
|
|
MOVWU (R9)(R11), R10
|
|
MOVWU (R5)(R11), R16
|
|
CMPW R10, R16
|
|
BNE matchlen_match2_repeat_extend_encodeBlockAsm8B
|
|
SUB $4, R8, R8
|
|
MOVWU R8, R8
|
|
ADD $4, R11, R11
|
|
MOVWU R11, R11
|
|
|
|
matchlen_match2_repeat_extend_encodeBlockAsm8B:
|
|
CMPW $0x01, R8
|
|
BEQ matchlen_match1_repeat_extend_encodeBlockAsm8B
|
|
BLO repeat_extend_forward_end_encodeBlockAsm8B
|
|
MOVHU (R9)(R11), R16
|
|
BFI $0, R16, $16, R10
|
|
ADD R11, R5, R15
|
|
MOVHU (R15), R15
|
|
AND $0xffff, R10, R16
|
|
CMP R16, R15
|
|
BNE matchlen_match1_repeat_extend_encodeBlockAsm8B
|
|
ADD $2, R11, R11
|
|
MOVWU R11, R11
|
|
SUBSW $0x02, R8, R8
|
|
BEQ repeat_extend_forward_end_encodeBlockAsm8B
|
|
|
|
matchlen_match1_repeat_extend_encodeBlockAsm8B:
|
|
MOVBU (R9)(R11), R16
|
|
BFI $0, R16, $8, R10
|
|
ADD R11, R5, R15
|
|
MOVBU (R15), R15
|
|
AND $0xff, R10, R16
|
|
CMP R16, R15
|
|
BNE repeat_extend_forward_end_encodeBlockAsm8B
|
|
ADD $1, R11, R11
|
|
MOVWU R11, R11
|
|
|
|
repeat_extend_forward_end_encodeBlockAsm8B:
|
|
ADDW R11, R2, R2
|
|
MOVWU R2, R5
|
|
SUBW R6, R5, R5
|
|
MOVWU 24(RSP), R6
|
|
TSTW R7, R7
|
|
BEQ repeat_as_copy_encodeBlockAsm8B
|
|
|
|
// emitRepeat
|
|
MOVWU R5, R6
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x08, R6
|
|
BLS repeat_two_match_repeat_encodeBlockAsm8B
|
|
CMPW $0x0c, R6
|
|
BHS cant_repeat_two_offset_match_repeat_encodeBlockAsm8B
|
|
|
|
cant_repeat_two_offset_match_repeat_encodeBlockAsm8B:
|
|
CMPW $0x00000104, R5
|
|
BLO repeat_three_match_repeat_encodeBlockAsm8B
|
|
SUB $256, R5, R5
|
|
MOVWU R5, R5
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R5, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm8B
|
|
|
|
repeat_three_match_repeat_encodeBlockAsm8B:
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R5, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm8B
|
|
|
|
repeat_two_match_repeat_encodeBlockAsm8B:
|
|
LSLW $0x02, R5, R5
|
|
ORRW $0x01, R5, R5
|
|
MOVH R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm8B
|
|
MOVD $0, R7
|
|
ADD R5<<2, R7, R5
|
|
ADD $1, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R6, 1(R1)
|
|
ASRW $0x08, R6, R6
|
|
LSLW $0x05, R6, R6
|
|
ORRW R6, R5, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm8B
|
|
|
|
repeat_as_copy_encodeBlockAsm8B:
|
|
// emitCopy
|
|
CMPW $0x40, R5
|
|
BLS two_byte_offset_short_repeat_as_copy_encodeBlockAsm8B
|
|
CMPW $0x00000800, R6
|
|
BHS long_offset_short_repeat_as_copy_encodeBlockAsm8B
|
|
MOVD $0x00000001, R7
|
|
ADD $16, R7, R7
|
|
MOVWU R7, R7
|
|
MOVB R6, 1(R1)
|
|
LSRW $0x08, R6, R6
|
|
LSLW $0x05, R6, R6
|
|
ORRW R6, R7, R7
|
|
MOVB R7, (R1)
|
|
ADD $0x02, R1, R1
|
|
SUBW $0x08, R5, R5
|
|
|
|
// emitRepeat
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
JMP cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b
|
|
MOVWU R5, R6
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x08, R6
|
|
BLS repeat_two_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b
|
|
CMPW $0x0c, R6
|
|
BHS cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b
|
|
|
|
cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b:
|
|
CMPW $0x00000104, R5
|
|
BLO repeat_three_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b
|
|
SUB $256, R5, R5
|
|
MOVWU R5, R5
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R5, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm8B
|
|
|
|
repeat_three_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b:
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R5, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm8B
|
|
|
|
repeat_two_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b:
|
|
LSLW $0x02, R5, R5
|
|
ORRW $0x01, R5, R5
|
|
MOVH R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm8B
|
|
MOVD $0, R7
|
|
ADD R5<<2, R7, R5
|
|
ADD $1, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R6, 1(R1)
|
|
ASRW $0x08, R6, R6
|
|
LSLW $0x05, R6, R6
|
|
ORRW R6, R5, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm8B
|
|
|
|
long_offset_short_repeat_as_copy_encodeBlockAsm8B:
|
|
MOVD $0xee, R16
|
|
MOVB R16, (R1)
|
|
MOVH R6, 1(R1)
|
|
SUB $60, R5, R5
|
|
MOVWU R5, R5
|
|
ADD $0x03, R1, R1
|
|
|
|
// emitRepeat
|
|
MOVWU R5, R6
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x08, R6
|
|
BLS repeat_two_repeat_as_copy_encodeBlockAsm8B_emit_copy_short
|
|
CMPW $0x0c, R6
|
|
BHS cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm8B_emit_copy_short
|
|
|
|
cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm8B_emit_copy_short:
|
|
CMPW $0x00000104, R5
|
|
BLO repeat_three_repeat_as_copy_encodeBlockAsm8B_emit_copy_short
|
|
SUB $256, R5, R5
|
|
MOVWU R5, R5
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R5, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm8B
|
|
|
|
repeat_three_repeat_as_copy_encodeBlockAsm8B_emit_copy_short:
|
|
SUB $4, R5, R5
|
|
MOVWU R5, R5
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R5, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm8B
|
|
|
|
repeat_two_repeat_as_copy_encodeBlockAsm8B_emit_copy_short:
|
|
LSLW $0x02, R5, R5
|
|
ORRW $0x01, R5, R5
|
|
MOVH R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm8B
|
|
MOVD $0, R7
|
|
ADD R5<<2, R7, R5
|
|
ADD $1, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R6, 1(R1)
|
|
ASRW $0x08, R6, R6
|
|
LSLW $0x05, R6, R6
|
|
ORRW R6, R5, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm8B
|
|
|
|
two_byte_offset_short_repeat_as_copy_encodeBlockAsm8B:
|
|
MOVWU R5, R7
|
|
LSLW $0x02, R7, R7
|
|
CMPW $0x0c, R5
|
|
BHS emit_copy_three_repeat_as_copy_encodeBlockAsm8B
|
|
SUB $15, R7, R7
|
|
MOVWU R7, R7
|
|
MOVB R6, 1(R1)
|
|
LSRW $0x08, R6, R6
|
|
LSLW $0x05, R6, R6
|
|
ORRW R6, R7, R7
|
|
MOVB R7, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeBlockAsm8B
|
|
|
|
emit_copy_three_repeat_as_copy_encodeBlockAsm8B:
|
|
SUB $2, R7, R7
|
|
MOVWU R7, R7
|
|
MOVB R7, (R1)
|
|
MOVH R6, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
|
|
repeat_end_emit_encodeBlockAsm8B:
|
|
MOVW R2, 20(RSP)
|
|
JMP search_loop_encodeBlockAsm8B
|
|
|
|
no_repeat_found_encodeBlockAsm8B:
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ candidate_match_encodeBlockAsm8B
|
|
LSR $0x08, R6, R6
|
|
MOVWU (R0)(R9<<2), R5
|
|
ADD $2, R2, R8
|
|
MOVWU R8, R8
|
|
MOVWU (R3)(R7), R16
|
|
CMPW R6, R16
|
|
BEQ candidate2_match_encodeBlockAsm8B
|
|
MOVW R8, (R0)(R9<<2)
|
|
LSR $0x08, R6, R6
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ candidate3_match_encodeBlockAsm8B
|
|
MOVWU 28(RSP), R2
|
|
JMP search_loop_encodeBlockAsm8B
|
|
|
|
candidate3_match_encodeBlockAsm8B:
|
|
ADDW $0x02, R2, R2
|
|
JMP candidate_match_encodeBlockAsm8B
|
|
|
|
candidate2_match_encodeBlockAsm8B:
|
|
MOVW R8, (R0)(R9<<2)
|
|
ADDW $1, R2, R2
|
|
MOVWU R7, R5
|
|
|
|
candidate_match_encodeBlockAsm8B:
|
|
MOVWU 20(RSP), R6
|
|
TSTW R5, R5
|
|
BEQ match_extend_back_end_encodeBlockAsm8B
|
|
|
|
match_extend_back_loop_encodeBlockAsm8B:
|
|
CMPW R6, R2
|
|
BLS match_extend_back_end_encodeBlockAsm8B
|
|
ADD R5, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R7
|
|
ADD R2, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R8
|
|
AND $0xff, R8, R16
|
|
AND $0xff, R7, R15
|
|
CMP R16, R15
|
|
BNE match_extend_back_end_encodeBlockAsm8B
|
|
SUB $1, R2, R2
|
|
MOVWU R2, R2
|
|
SUBSW $1, R5, R5
|
|
BEQ match_extend_back_end_encodeBlockAsm8B
|
|
JMP match_extend_back_loop_encodeBlockAsm8B
|
|
|
|
match_extend_back_end_encodeBlockAsm8B:
|
|
MOVWU R2, R6
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R6, R6
|
|
ADD R6, R1, R6
|
|
ADD $3, R6, R6
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R6
|
|
BLO match_dst_size_check_encodeBlockAsm8B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeBlockAsm8B:
|
|
MOVWU R2, R6
|
|
MOVWU 20(RSP), R7
|
|
CMPW R6, R7
|
|
BEQ emit_literal_done_match_emit_encodeBlockAsm8B
|
|
MOVWU R6, R8
|
|
MOVW R6, 20(RSP)
|
|
ADD R7, R3, R6
|
|
SUBW R7, R8, R8
|
|
SUB $1, R8, R7
|
|
MOVWU R7, R7
|
|
CMPW $0x3c, R7
|
|
BLO one_byte_match_emit_encodeBlockAsm8B
|
|
CMPW $0x00000100, R7
|
|
BLO two_bytes_match_emit_encodeBlockAsm8B
|
|
BLO three_bytes_match_emit_encodeBlockAsm8B
|
|
|
|
three_bytes_match_emit_encodeBlockAsm8B:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R7, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_match_emit_encodeBlockAsm8B
|
|
|
|
two_bytes_match_emit_encodeBlockAsm8B:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R7, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R7
|
|
BLO memmove_match_emit_encodeBlockAsm8B
|
|
JMP memmove_long_match_emit_encodeBlockAsm8B
|
|
|
|
one_byte_match_emit_encodeBlockAsm8B:
|
|
LSLW $0x02, R7, R16
|
|
BFI $0, R16, $8, R7
|
|
MOVB R7, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_match_emit_encodeBlockAsm8B:
|
|
ADD R8, R1, R7
|
|
|
|
// genMemMoveShort
|
|
CMP $0x08, R8
|
|
BLS emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_8
|
|
CMP $0x10, R8
|
|
BLS emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_8through16
|
|
CMP $0x20, R8
|
|
BLS emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_8:
|
|
MOVD (R6), R9
|
|
MOVD R9, (R1)
|
|
JMP memmove_end_copy_match_emit_encodeBlockAsm8B
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_8through16:
|
|
MOVD (R6), R9
|
|
ADD R8, R6, R15
|
|
MOVD -8(R15), R6
|
|
MOVD R9, (R1)
|
|
ADD R8, R1, R15
|
|
MOVD R6, -8(R15)
|
|
JMP memmove_end_copy_match_emit_encodeBlockAsm8B
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_17through32:
|
|
FMOVQ (R6), F0
|
|
ADD R8, R6, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_match_emit_encodeBlockAsm8B
|
|
|
|
emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_33through64:
|
|
FMOVQ (R6), F0
|
|
FMOVQ 16(R6), F1
|
|
ADD R8, R6, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R8, R6, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_match_emit_encodeBlockAsm8B:
|
|
MOVD R7, R1
|
|
JMP emit_literal_done_match_emit_encodeBlockAsm8B
|
|
|
|
memmove_long_match_emit_encodeBlockAsm8B:
|
|
ADD R8, R1, R7
|
|
|
|
// genMemMoveLong
|
|
MOVD R6, R9
|
|
MOVD R1, R10
|
|
MOVD R8, R11
|
|
|
|
emit_lit_memmove_long_match_emit_encodeBlockAsm8Blarge_big_loop_back:
|
|
FMOVQ (R9), F0
|
|
FMOVQ 16(R9), F1
|
|
FMOVQ F0, (R10)
|
|
FMOVQ F1, 16(R10)
|
|
ADD $0x20, R9, R9
|
|
ADD $0x20, R10, R10
|
|
SUB $0x20, R11, R11
|
|
CMP $0x20, R11
|
|
BHS emit_lit_memmove_long_match_emit_encodeBlockAsm8Blarge_big_loop_back
|
|
ADD R8, R6, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R8, R6, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R8, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R7, R1
|
|
|
|
emit_literal_done_match_emit_encodeBlockAsm8B:
|
|
match_nolit_loop_encodeBlockAsm8B:
|
|
MOVWU R2, R6
|
|
SUBW R5, R6, R6
|
|
MOVW R6, 24(RSP)
|
|
ADDW $0x04, R2, R2
|
|
ADDW $0x04, R5, R5
|
|
MOVD src_len+32(FP), R6
|
|
SUBW R2, R6, R6
|
|
ADD R2, R3, R7
|
|
ADD R5, R3, R5
|
|
|
|
// matchLen
|
|
MOVD $0, R9
|
|
|
|
matchlen_loopback_16_match_nolit_encodeBlockAsm8B:
|
|
CMPW $0x10, R6
|
|
BLO matchlen_match8_match_nolit_encodeBlockAsm8B
|
|
MOVD (R7)(R9), R8
|
|
ADD R9, R7, R15
|
|
MOVD 8(R15), R10
|
|
MOVD (R5)(R9), R16
|
|
EOR R16, R8, R8
|
|
TST R8, R8
|
|
BNE matchlen_bsf_8_match_nolit_encodeBlockAsm8B
|
|
ADD R9, R5, R15
|
|
MOVD 8(R15), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_16match_nolit_encodeBlockAsm8B
|
|
SUB $16, R6, R6
|
|
MOVWU R6, R6
|
|
ADD $16, R9, R9
|
|
MOVWU R9, R9
|
|
JMP matchlen_loopback_16_match_nolit_encodeBlockAsm8B
|
|
|
|
matchlen_bsf_16match_nolit_encodeBlockAsm8B:
|
|
RBIT R10, R10
|
|
CLZ R10, R10
|
|
ASR $0x03, R10, R10
|
|
ADD R10, R9, R9
|
|
ADD $8, R9, R9
|
|
MOVWU R9, R9
|
|
JMP match_nolit_end_encodeBlockAsm8B
|
|
|
|
matchlen_match8_match_nolit_encodeBlockAsm8B:
|
|
CMPW $0x08, R6
|
|
BLO matchlen_match4_match_nolit_encodeBlockAsm8B
|
|
MOVD (R7)(R9), R8
|
|
MOVD (R5)(R9), R16
|
|
EOR R16, R8, R8
|
|
TST R8, R8
|
|
BNE matchlen_bsf_8_match_nolit_encodeBlockAsm8B
|
|
SUB $8, R6, R6
|
|
MOVWU R6, R6
|
|
ADD $8, R9, R9
|
|
MOVWU R9, R9
|
|
JMP matchlen_match4_match_nolit_encodeBlockAsm8B
|
|
|
|
matchlen_bsf_8_match_nolit_encodeBlockAsm8B:
|
|
RBIT R8, R8
|
|
CLZ R8, R8
|
|
ASR $0x03, R8, R8
|
|
ADD R8, R9, R9
|
|
MOVWU R9, R9
|
|
JMP match_nolit_end_encodeBlockAsm8B
|
|
|
|
matchlen_match4_match_nolit_encodeBlockAsm8B:
|
|
CMPW $0x04, R6
|
|
BLO matchlen_match2_match_nolit_encodeBlockAsm8B
|
|
MOVWU (R7)(R9), R8
|
|
MOVWU (R5)(R9), R16
|
|
CMPW R8, R16
|
|
BNE matchlen_match2_match_nolit_encodeBlockAsm8B
|
|
SUB $4, R6, R6
|
|
MOVWU R6, R6
|
|
ADD $4, R9, R9
|
|
MOVWU R9, R9
|
|
|
|
matchlen_match2_match_nolit_encodeBlockAsm8B:
|
|
CMPW $0x01, R6
|
|
BEQ matchlen_match1_match_nolit_encodeBlockAsm8B
|
|
BLO match_nolit_end_encodeBlockAsm8B
|
|
MOVHU (R7)(R9), R16
|
|
BFI $0, R16, $16, R8
|
|
ADD R9, R5, R15
|
|
MOVHU (R15), R15
|
|
AND $0xffff, R8, R16
|
|
CMP R16, R15
|
|
BNE matchlen_match1_match_nolit_encodeBlockAsm8B
|
|
ADD $2, R9, R9
|
|
MOVWU R9, R9
|
|
SUBSW $0x02, R6, R6
|
|
BEQ match_nolit_end_encodeBlockAsm8B
|
|
|
|
matchlen_match1_match_nolit_encodeBlockAsm8B:
|
|
MOVBU (R7)(R9), R16
|
|
BFI $0, R16, $8, R8
|
|
ADD R9, R5, R15
|
|
MOVBU (R15), R15
|
|
AND $0xff, R8, R16
|
|
CMP R16, R15
|
|
BNE match_nolit_end_encodeBlockAsm8B
|
|
ADD $1, R9, R9
|
|
MOVWU R9, R9
|
|
|
|
match_nolit_end_encodeBlockAsm8B:
|
|
ADDW R9, R2, R2
|
|
MOVWU 24(RSP), R5
|
|
ADDW $0x04, R9, R9
|
|
MOVW R2, 20(RSP)
|
|
|
|
// emitCopy
|
|
CMPW $0x40, R9
|
|
BLS two_byte_offset_short_match_nolit_encodeBlockAsm8B
|
|
CMPW $0x00000800, R5
|
|
BHS long_offset_short_match_nolit_encodeBlockAsm8B
|
|
MOVD $0x00000001, R6
|
|
ADD $16, R6, R6
|
|
MOVWU R6, R6
|
|
MOVB R5, 1(R1)
|
|
LSRW $0x08, R5, R5
|
|
LSLW $0x05, R5, R5
|
|
ORRW R5, R6, R6
|
|
MOVB R6, (R1)
|
|
ADD $0x02, R1, R1
|
|
SUBW $0x08, R9, R9
|
|
|
|
// emitRepeat
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
JMP cant_repeat_two_offset_match_nolit_encodeBlockAsm8B_emit_copy_short_2b
|
|
MOVWU R9, R5
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
CMPW $0x08, R5
|
|
BLS repeat_two_match_nolit_encodeBlockAsm8B_emit_copy_short_2b
|
|
CMPW $0x0c, R5
|
|
BHS cant_repeat_two_offset_match_nolit_encodeBlockAsm8B_emit_copy_short_2b
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBlockAsm8B_emit_copy_short_2b:
|
|
CMPW $0x00000104, R9
|
|
BLO repeat_three_match_nolit_encodeBlockAsm8B_emit_copy_short_2b
|
|
SUB $256, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R9, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm8B
|
|
|
|
repeat_three_match_nolit_encodeBlockAsm8B_emit_copy_short_2b:
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R9, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm8B
|
|
|
|
repeat_two_match_nolit_encodeBlockAsm8B_emit_copy_short_2b:
|
|
LSLW $0x02, R9, R9
|
|
ORRW $0x01, R9, R9
|
|
MOVH R9, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm8B
|
|
MOVD $0, R6
|
|
ADD R9<<2, R6, R9
|
|
ADD $1, R9, R9
|
|
MOVWU R9, R9
|
|
MOVB R5, 1(R1)
|
|
ASRW $0x08, R5, R5
|
|
LSLW $0x05, R5, R5
|
|
ORRW R5, R9, R9
|
|
MOVB R9, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm8B
|
|
|
|
long_offset_short_match_nolit_encodeBlockAsm8B:
|
|
MOVD $0xee, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
SUB $60, R9, R9
|
|
MOVWU R9, R9
|
|
ADD $0x03, R1, R1
|
|
|
|
// emitRepeat
|
|
MOVWU R9, R5
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
CMPW $0x08, R5
|
|
BLS repeat_two_match_nolit_encodeBlockAsm8B_emit_copy_short
|
|
CMPW $0x0c, R5
|
|
BHS cant_repeat_two_offset_match_nolit_encodeBlockAsm8B_emit_copy_short
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBlockAsm8B_emit_copy_short:
|
|
CMPW $0x00000104, R9
|
|
BLO repeat_three_match_nolit_encodeBlockAsm8B_emit_copy_short
|
|
SUB $256, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R9, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm8B
|
|
|
|
repeat_three_match_nolit_encodeBlockAsm8B_emit_copy_short:
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R9, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm8B
|
|
|
|
repeat_two_match_nolit_encodeBlockAsm8B_emit_copy_short:
|
|
LSLW $0x02, R9, R9
|
|
ORRW $0x01, R9, R9
|
|
MOVH R9, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm8B
|
|
MOVD $0, R6
|
|
ADD R9<<2, R6, R9
|
|
ADD $1, R9, R9
|
|
MOVWU R9, R9
|
|
MOVB R5, 1(R1)
|
|
ASRW $0x08, R5, R5
|
|
LSLW $0x05, R5, R5
|
|
ORRW R5, R9, R9
|
|
MOVB R9, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm8B
|
|
|
|
two_byte_offset_short_match_nolit_encodeBlockAsm8B:
|
|
MOVWU R9, R6
|
|
LSLW $0x02, R6, R6
|
|
CMPW $0x0c, R9
|
|
BHS emit_copy_three_match_nolit_encodeBlockAsm8B
|
|
SUB $15, R6, R6
|
|
MOVWU R6, R6
|
|
MOVB R5, 1(R1)
|
|
LSRW $0x08, R5, R5
|
|
LSLW $0x05, R5, R5
|
|
ORRW R5, R6, R6
|
|
MOVB R6, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBlockAsm8B
|
|
|
|
emit_copy_three_match_nolit_encodeBlockAsm8B:
|
|
SUB $2, R6, R6
|
|
MOVWU R6, R6
|
|
MOVB R6, (R1)
|
|
MOVH R5, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
|
|
match_nolit_emitcopy_end_encodeBlockAsm8B:
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R2
|
|
BHS emit_remainder_encodeBlockAsm8B
|
|
ADD R2, R3, R15
|
|
MOVD -2(R15), R6
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R1
|
|
BLO match_nolit_dst_ok_encodeBlockAsm8B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeBlockAsm8B:
|
|
MOVD $0x9e3779b1, R8
|
|
MOVD R6, R7
|
|
LSR $0x10, R6, R6
|
|
MOVD R6, R5
|
|
LSL $0x20, R7, R7
|
|
MUL R8, R7, R7
|
|
LSR $0x38, R7, R7
|
|
LSL $0x20, R5, R5
|
|
MUL R8, R5, R5
|
|
LSR $0x38, R5, R5
|
|
SUB $2, R2, R8
|
|
MOVWU R8, R8
|
|
ADD R5<<2, R0, R9
|
|
MOVWU (R9), R5
|
|
MOVW R8, (R0)(R7<<2)
|
|
MOVW R2, (R9)
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ match_nolit_loop_encodeBlockAsm8B
|
|
ADDW $1, R2, R2
|
|
JMP search_loop_encodeBlockAsm8B
|
|
|
|
emit_remainder_encodeBlockAsm8B:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R0, R0
|
|
ADD R0, R1, R0
|
|
ADD $3, R0, R0
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R0
|
|
BLO emit_remainder_ok_encodeBlockAsm8B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeBlockAsm8B:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R2
|
|
CMPW R0, R2
|
|
BEQ emit_literal_done_emit_remainder_encodeBlockAsm8B
|
|
MOVWU R0, R5
|
|
MOVW R0, 20(RSP)
|
|
ADD R2, R3, R0
|
|
SUBW R2, R5, R5
|
|
SUB $1, R5, R2
|
|
MOVWU R2, R2
|
|
CMPW $0x3c, R2
|
|
BLO one_byte_emit_remainder_encodeBlockAsm8B
|
|
CMPW $0x00000100, R2
|
|
BLO two_bytes_emit_remainder_encodeBlockAsm8B
|
|
BLO three_bytes_emit_remainder_encodeBlockAsm8B
|
|
|
|
three_bytes_emit_remainder_encodeBlockAsm8B:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R2, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_emit_remainder_encodeBlockAsm8B
|
|
|
|
two_bytes_emit_remainder_encodeBlockAsm8B:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R2, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R2
|
|
BLO memmove_emit_remainder_encodeBlockAsm8B
|
|
JMP memmove_long_emit_remainder_encodeBlockAsm8B
|
|
|
|
one_byte_emit_remainder_encodeBlockAsm8B:
|
|
LSLW $0x02, R2, R16
|
|
BFI $0, R16, $8, R2
|
|
MOVB R2, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_emit_remainder_encodeBlockAsm8B:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveShort
|
|
CMP $0x03, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_1or2
|
|
BEQ emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_3
|
|
CMP $0x08, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_4through7
|
|
CMP $0x10, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_8through16
|
|
CMP $0x20, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_1or2:
|
|
MOVBU (R0), R16
|
|
BFI $0, R16, $8, R5
|
|
ADD R3, R0, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVB R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVB R0, -1(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_3:
|
|
MOVHU (R0), R16
|
|
BFI $0, R16, $16, R5
|
|
MOVBU 2(R0), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVH R5, (R1)
|
|
MOVB R0, 2(R1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_4through7:
|
|
MOVWU (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVWU -4(R15), R0
|
|
MOVW R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVW R0, -4(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_8through16:
|
|
MOVD (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVD -8(R15), R0
|
|
MOVD R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVD R0, -8(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_17through32:
|
|
FMOVQ (R0), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_33through64:
|
|
FMOVQ (R0), F0
|
|
FMOVQ 16(R0), F1
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_emit_remainder_encodeBlockAsm8B:
|
|
MOVD R2, R1
|
|
JMP emit_literal_done_emit_remainder_encodeBlockAsm8B
|
|
|
|
memmove_long_emit_remainder_encodeBlockAsm8B:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveLong
|
|
MOVD R0, R5
|
|
MOVD R1, R6
|
|
MOVD R3, R7
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeBlockAsm8Blarge_big_loop_back:
|
|
FMOVQ (R5), F0
|
|
FMOVQ 16(R5), F1
|
|
FMOVQ F0, (R6)
|
|
FMOVQ F1, 16(R6)
|
|
ADD $0x20, R5, R5
|
|
ADD $0x20, R6, R6
|
|
SUB $0x20, R7, R7
|
|
CMP $0x20, R7
|
|
BHS emit_lit_memmove_long_emit_remainder_encodeBlockAsm8Blarge_big_loop_back
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R3, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R2, R1
|
|
|
|
emit_literal_done_emit_remainder_encodeBlockAsm8B:
|
|
MOVD dst_base+0(FP), R0
|
|
SUB R0, R1, R1
|
|
MOVD R1, ret+56(FP)
|
|
RET
|
|
|
|
// func encodeBetterBlockAsm(dst []byte, src []byte, tmp *[589824]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeBetterBlockAsm(SB), $24-64
|
|
MOVD tmp+48(FP), R0
|
|
MOVD dst_base+0(FP), R1
|
|
MOVD $0x00001200, R2
|
|
MOVD R0, R3
|
|
VEOR V0.B16, V0.B16, V0.B16
|
|
|
|
zero_loop_encodeBetterBlockAsm:
|
|
FMOVQ F0, (R3)
|
|
FMOVQ F0, 16(R3)
|
|
FMOVQ F0, 32(R3)
|
|
FMOVQ F0, 48(R3)
|
|
FMOVQ F0, 64(R3)
|
|
FMOVQ F0, 80(R3)
|
|
FMOVQ F0, 96(R3)
|
|
FMOVQ F0, 112(R3)
|
|
ADD $0x80, R3, R3
|
|
SUBS $1, R2, R2
|
|
BNE zero_loop_encodeBetterBlockAsm
|
|
MOVD $0x00000000, R16
|
|
MOVW R16, 20(RSP)
|
|
MOVD src_len+32(FP), R2
|
|
SUB $6, R2, R3
|
|
SUB $8, R2, R5
|
|
MOVW R5, 16(RSP)
|
|
LSR $0x05, R2, R2
|
|
SUBW R2, R3, R3
|
|
ADD R3, R1, R3
|
|
MOVD R3, 8(RSP)
|
|
MOVD $0x00000001, R2
|
|
MOVD $0x00000000, R16
|
|
MOVW R16, 24(RSP)
|
|
MOVD src_base+24(FP), R3
|
|
|
|
search_loop_encodeBetterBlockAsm:
|
|
MOVWU R2, R5
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R5, R5
|
|
LSRW $0x07, R5, R5
|
|
CMPW $0x63, R5
|
|
BLS check_maxskip_ok_encodeBetterBlockAsm
|
|
ADD $100, R2, R5
|
|
MOVWU R5, R5
|
|
JMP check_maxskip_cont_encodeBetterBlockAsm
|
|
|
|
check_maxskip_ok_encodeBetterBlockAsm:
|
|
ADD R5, R2, R5
|
|
ADD $1, R5, R5
|
|
MOVWU R5, R5
|
|
|
|
check_maxskip_cont_encodeBetterBlockAsm:
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R5
|
|
BHS emit_remainder_encodeBetterBlockAsm
|
|
MOVD (R3)(R2), R6
|
|
MOVW R5, 28(RSP)
|
|
MOVD $0x00cf1bbcdcbfa563, R8
|
|
MOVD $0x9e3779b1, R5
|
|
MOVD R6, R9
|
|
MOVD R6, R10
|
|
LSL $0x08, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x2f, R9, R9
|
|
LSL $0x20, R10, R10
|
|
MUL R5, R10, R10
|
|
LSR $0x32, R10, R10
|
|
MOVWU (R0)(R9<<2), R5
|
|
ADD R10<<2, R0, R15
|
|
MOVWU 524288(R15), R7
|
|
MOVW R2, (R0)(R9<<2)
|
|
ADD R10<<2, R0, R15
|
|
MOVW R2, 524288(R15)
|
|
MOVD (R3)(R5), R9
|
|
MOVD (R3)(R7), R10
|
|
CMP R6, R9
|
|
BEQ candidate_match_encodeBetterBlockAsm
|
|
CMP R6, R10
|
|
BNE no_short_found_encodeBetterBlockAsm
|
|
MOVWU R7, R5
|
|
JMP candidate_match_encodeBetterBlockAsm
|
|
|
|
no_short_found_encodeBetterBlockAsm:
|
|
CMPW R6, R9
|
|
BEQ candidate_match_encodeBetterBlockAsm
|
|
CMPW R6, R10
|
|
BEQ candidateS_match_encodeBetterBlockAsm
|
|
MOVWU 28(RSP), R2
|
|
JMP search_loop_encodeBetterBlockAsm
|
|
|
|
candidateS_match_encodeBetterBlockAsm:
|
|
LSR $0x08, R6, R6
|
|
MOVD R6, R9
|
|
LSL $0x08, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x2f, R9, R9
|
|
MOVWU (R0)(R9<<2), R5
|
|
ADDW $1, R2, R2
|
|
MOVW R2, (R0)(R9<<2)
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ candidate_match_encodeBetterBlockAsm
|
|
SUBW $1, R2, R2
|
|
MOVWU R7, R5
|
|
|
|
candidate_match_encodeBetterBlockAsm:
|
|
MOVWU 20(RSP), R6
|
|
TSTW R5, R5
|
|
BEQ match_extend_back_end_encodeBetterBlockAsm
|
|
|
|
match_extend_back_loop_encodeBetterBlockAsm:
|
|
CMPW R6, R2
|
|
BLS match_extend_back_end_encodeBetterBlockAsm
|
|
ADD R5, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R7
|
|
ADD R2, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R8
|
|
AND $0xff, R8, R16
|
|
AND $0xff, R7, R15
|
|
CMP R16, R15
|
|
BNE match_extend_back_end_encodeBetterBlockAsm
|
|
SUB $1, R2, R2
|
|
MOVWU R2, R2
|
|
SUBSW $1, R5, R5
|
|
BEQ match_extend_back_end_encodeBetterBlockAsm
|
|
JMP match_extend_back_loop_encodeBetterBlockAsm
|
|
|
|
match_extend_back_end_encodeBetterBlockAsm:
|
|
MOVWU R2, R6
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R6, R6
|
|
ADD R6, R1, R6
|
|
ADD $5, R6, R6
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R6
|
|
BLO match_dst_size_check_encodeBetterBlockAsm
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeBetterBlockAsm:
|
|
MOVWU R2, R6
|
|
ADDW $0x04, R2, R2
|
|
ADDW $0x04, R5, R5
|
|
MOVD src_len+32(FP), R7
|
|
SUBW R2, R7, R7
|
|
ADD R2, R3, R8
|
|
ADD R5, R3, R9
|
|
|
|
// matchLen
|
|
MOVD $0, R11
|
|
|
|
matchlen_loopback_16_match_nolit_encodeBetterBlockAsm:
|
|
CMPW $0x10, R7
|
|
BLO matchlen_match8_match_nolit_encodeBetterBlockAsm
|
|
MOVD (R8)(R11), R10
|
|
ADD R11, R8, R15
|
|
MOVD 8(R15), R12
|
|
MOVD (R9)(R11), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_8_match_nolit_encodeBetterBlockAsm
|
|
ADD R11, R9, R15
|
|
MOVD 8(R15), R16
|
|
EOR R16, R12, R12
|
|
TST R12, R12
|
|
BNE matchlen_bsf_16match_nolit_encodeBetterBlockAsm
|
|
SUB $16, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $16, R11, R11
|
|
MOVWU R11, R11
|
|
JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm
|
|
|
|
matchlen_bsf_16match_nolit_encodeBetterBlockAsm:
|
|
RBIT R12, R12
|
|
CLZ R12, R12
|
|
ASR $0x03, R12, R12
|
|
ADD R12, R11, R11
|
|
ADD $8, R11, R11
|
|
MOVWU R11, R11
|
|
JMP match_nolit_end_encodeBetterBlockAsm
|
|
|
|
matchlen_match8_match_nolit_encodeBetterBlockAsm:
|
|
CMPW $0x08, R7
|
|
BLO matchlen_match4_match_nolit_encodeBetterBlockAsm
|
|
MOVD (R8)(R11), R10
|
|
MOVD (R9)(R11), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_8_match_nolit_encodeBetterBlockAsm
|
|
SUB $8, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $8, R11, R11
|
|
MOVWU R11, R11
|
|
JMP matchlen_match4_match_nolit_encodeBetterBlockAsm
|
|
|
|
matchlen_bsf_8_match_nolit_encodeBetterBlockAsm:
|
|
RBIT R10, R10
|
|
CLZ R10, R10
|
|
ASR $0x03, R10, R10
|
|
ADD R10, R11, R11
|
|
MOVWU R11, R11
|
|
JMP match_nolit_end_encodeBetterBlockAsm
|
|
|
|
matchlen_match4_match_nolit_encodeBetterBlockAsm:
|
|
CMPW $0x04, R7
|
|
BLO matchlen_match2_match_nolit_encodeBetterBlockAsm
|
|
MOVWU (R8)(R11), R10
|
|
MOVWU (R9)(R11), R16
|
|
CMPW R10, R16
|
|
BNE matchlen_match2_match_nolit_encodeBetterBlockAsm
|
|
SUB $4, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $4, R11, R11
|
|
MOVWU R11, R11
|
|
|
|
matchlen_match2_match_nolit_encodeBetterBlockAsm:
|
|
CMPW $0x01, R7
|
|
BEQ matchlen_match1_match_nolit_encodeBetterBlockAsm
|
|
BLO match_nolit_end_encodeBetterBlockAsm
|
|
MOVHU (R8)(R11), R16
|
|
BFI $0, R16, $16, R10
|
|
ADD R11, R9, R15
|
|
MOVHU (R15), R15
|
|
AND $0xffff, R10, R16
|
|
CMP R16, R15
|
|
BNE matchlen_match1_match_nolit_encodeBetterBlockAsm
|
|
ADD $2, R11, R11
|
|
MOVWU R11, R11
|
|
SUBSW $0x02, R7, R7
|
|
BEQ match_nolit_end_encodeBetterBlockAsm
|
|
|
|
matchlen_match1_match_nolit_encodeBetterBlockAsm:
|
|
MOVBU (R8)(R11), R16
|
|
BFI $0, R16, $8, R10
|
|
ADD R11, R9, R15
|
|
MOVBU (R15), R15
|
|
AND $0xff, R10, R16
|
|
CMP R16, R15
|
|
BNE match_nolit_end_encodeBetterBlockAsm
|
|
ADD $1, R11, R11
|
|
MOVWU R11, R11
|
|
|
|
match_nolit_end_encodeBetterBlockAsm:
|
|
MOVWU R2, R7
|
|
SUBW R5, R7, R7
|
|
|
|
// Check if repeat
|
|
MOVWU 24(RSP), R16
|
|
CMPW R7, R16
|
|
BEQ match_is_repeat_encodeBetterBlockAsm
|
|
CMPW $0x01, R11
|
|
BHI match_length_ok_encodeBetterBlockAsm
|
|
CMPW $0x0000ffff, R7
|
|
BLS match_length_ok_encodeBetterBlockAsm
|
|
MOVWU 28(RSP), R2
|
|
ADDW $1, R2, R2
|
|
JMP search_loop_encodeBetterBlockAsm
|
|
|
|
match_length_ok_encodeBetterBlockAsm:
|
|
MOVW R7, 24(RSP)
|
|
MOVWU 20(RSP), R5
|
|
CMPW R6, R5
|
|
BEQ emit_literal_done_match_emit_encodeBetterBlockAsm
|
|
MOVWU R6, R8
|
|
MOVW R6, 20(RSP)
|
|
ADD R5, R3, R9
|
|
SUBW R5, R8, R8
|
|
SUB $1, R8, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x3c, R5
|
|
BLO one_byte_match_emit_encodeBetterBlockAsm
|
|
CMPW $0x00000100, R5
|
|
BLO two_bytes_match_emit_encodeBetterBlockAsm
|
|
CMPW $0x00010000, R5
|
|
BLO three_bytes_match_emit_encodeBetterBlockAsm
|
|
CMPW $0x01000000, R5
|
|
BLO four_bytes_match_emit_encodeBetterBlockAsm
|
|
MOVD $0xfc, R16
|
|
MOVB R16, (R1)
|
|
MOVW R5, 1(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP memmove_long_match_emit_encodeBetterBlockAsm
|
|
|
|
four_bytes_match_emit_encodeBetterBlockAsm:
|
|
MOVWU R5, R10
|
|
LSRW $0x10, R10, R10
|
|
MOVD $0xf8, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
MOVB R10, 3(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP memmove_long_match_emit_encodeBetterBlockAsm
|
|
|
|
three_bytes_match_emit_encodeBetterBlockAsm:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_match_emit_encodeBetterBlockAsm
|
|
|
|
two_bytes_match_emit_encodeBetterBlockAsm:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R5, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R5
|
|
BLO memmove_match_emit_encodeBetterBlockAsm
|
|
JMP memmove_long_match_emit_encodeBetterBlockAsm
|
|
|
|
one_byte_match_emit_encodeBetterBlockAsm:
|
|
LSLW $0x02, R5, R16
|
|
BFI $0, R16, $8, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_match_emit_encodeBetterBlockAsm:
|
|
ADD R8, R1, R5
|
|
|
|
// genMemMoveShort
|
|
CMP $0x04, R8
|
|
BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_4
|
|
CMP $0x08, R8
|
|
BLO emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_4through7
|
|
CMP $0x10, R8
|
|
BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_8through16
|
|
CMP $0x20, R8
|
|
BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_4:
|
|
MOVWU (R9), R10
|
|
MOVW R10, (R1)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_4through7:
|
|
MOVWU (R9), R10
|
|
ADD R8, R9, R15
|
|
MOVWU -4(R15), R9
|
|
MOVW R10, (R1)
|
|
ADD R8, R1, R15
|
|
MOVW R9, -4(R15)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_8through16:
|
|
MOVD (R9), R10
|
|
ADD R8, R9, R15
|
|
MOVD -8(R15), R9
|
|
MOVD R10, (R1)
|
|
ADD R8, R1, R15
|
|
MOVD R9, -8(R15)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_17through32:
|
|
FMOVQ (R9), F0
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_33through64:
|
|
FMOVQ (R9), F0
|
|
FMOVQ 16(R9), F1
|
|
ADD R8, R9, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_match_emit_encodeBetterBlockAsm:
|
|
MOVD R5, R1
|
|
JMP emit_literal_done_match_emit_encodeBetterBlockAsm
|
|
|
|
memmove_long_match_emit_encodeBetterBlockAsm:
|
|
ADD R8, R1, R5
|
|
|
|
// genMemMoveLong
|
|
MOVD R9, R10
|
|
MOVD R1, R12
|
|
MOVD R8, R13
|
|
|
|
emit_lit_memmove_long_match_emit_encodeBetterBlockAsmlarge_big_loop_back:
|
|
FMOVQ (R10), F0
|
|
FMOVQ 16(R10), F1
|
|
FMOVQ F0, (R12)
|
|
FMOVQ F1, 16(R12)
|
|
ADD $0x20, R10, R10
|
|
ADD $0x20, R12, R12
|
|
SUB $0x20, R13, R13
|
|
CMP $0x20, R13
|
|
BHS emit_lit_memmove_long_match_emit_encodeBetterBlockAsmlarge_big_loop_back
|
|
ADD R8, R9, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R8, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R5, R1
|
|
|
|
emit_literal_done_match_emit_encodeBetterBlockAsm:
|
|
ADDW R11, R2, R2
|
|
ADDW $0x04, R11, R11
|
|
MOVW R2, 20(RSP)
|
|
|
|
// emitCopy
|
|
CMPW $0x00010000, R7
|
|
BLO two_byte_offset_match_nolit_encodeBetterBlockAsm
|
|
CMPW $0x40, R11
|
|
BLS four_bytes_remain_match_nolit_encodeBetterBlockAsm
|
|
MOVD $0xff, R16
|
|
MOVB R16, (R1)
|
|
MOVW R7, 1(R1)
|
|
SUB $64, R11, R11
|
|
MOVWU R11, R11
|
|
ADD $0x05, R1, R1
|
|
CMPW $0x04, R11
|
|
BLO four_bytes_remain_match_nolit_encodeBetterBlockAsm
|
|
|
|
// emitRepeat
|
|
emit_repeat_again_match_nolit_encodeBetterBlockAsm_emit_copy:
|
|
MOVWU R11, R5
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
CMPW $0x08, R5
|
|
BLS repeat_two_match_nolit_encodeBetterBlockAsm_emit_copy
|
|
CMPW $0x0c, R5
|
|
BHS cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy
|
|
CMPW $0x00000800, R7
|
|
BLO repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy:
|
|
CMPW $0x00000104, R11
|
|
BLO repeat_three_match_nolit_encodeBetterBlockAsm_emit_copy
|
|
CMPW $0x00010100, R11
|
|
BLO repeat_four_match_nolit_encodeBetterBlockAsm_emit_copy
|
|
CMPW $0x0100ffff, R11
|
|
BLO repeat_five_match_nolit_encodeBetterBlockAsm_emit_copy
|
|
SUB $16842747, R11, R11
|
|
MOVWU R11, R11
|
|
MOVD $0xfffb001d, R16
|
|
MOVW R16, (R1)
|
|
MOVD $0xff, R16
|
|
MOVB R16, 4(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP emit_repeat_again_match_nolit_encodeBetterBlockAsm_emit_copy
|
|
|
|
repeat_five_match_nolit_encodeBetterBlockAsm_emit_copy:
|
|
SUB $65536, R11, R11
|
|
MOVWU R11, R11
|
|
MOVWU R11, R7
|
|
MOVD $0x001d, R16
|
|
MOVH R16, (R1)
|
|
MOVH R11, 2(R1)
|
|
ASRW $0x10, R7, R7
|
|
MOVB R7, 4(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
repeat_four_match_nolit_encodeBetterBlockAsm_emit_copy:
|
|
SUB $256, R11, R11
|
|
MOVWU R11, R11
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R11, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
repeat_three_match_nolit_encodeBetterBlockAsm_emit_copy:
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R11, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
repeat_two_match_nolit_encodeBetterBlockAsm_emit_copy:
|
|
LSLW $0x02, R11, R11
|
|
ORRW $0x01, R11, R11
|
|
MOVH R11, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy:
|
|
MOVD $0, R5
|
|
ADD R11<<2, R5, R11
|
|
ADD $1, R11, R11
|
|
MOVWU R11, R11
|
|
MOVB R7, 1(R1)
|
|
ASRW $0x08, R7, R7
|
|
LSLW $0x05, R7, R7
|
|
ORRW R7, R11, R11
|
|
MOVB R11, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
four_bytes_remain_match_nolit_encodeBetterBlockAsm:
|
|
TSTW R11, R11
|
|
BEQ match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
MOVD $0, R5
|
|
ADD R11<<2, R5, R11
|
|
SUB $1, R11, R11
|
|
MOVWU R11, R11
|
|
MOVB R11, (R1)
|
|
MOVW R7, 1(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
two_byte_offset_match_nolit_encodeBetterBlockAsm:
|
|
CMPW $0x40, R11
|
|
BLS two_byte_offset_short_match_nolit_encodeBetterBlockAsm
|
|
CMPW $0x00000800, R7
|
|
BHS long_offset_short_match_nolit_encodeBetterBlockAsm
|
|
MOVD $0x00000001, R5
|
|
ADD $16, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R7, 1(R1)
|
|
MOVWU R7, R8
|
|
LSRW $0x08, R8, R8
|
|
LSLW $0x05, R8, R8
|
|
ORRW R8, R5, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
SUBW $0x08, R11, R11
|
|
|
|
// emitRepeat
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
JMP cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b
|
|
|
|
emit_repeat_again_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b:
|
|
MOVWU R11, R5
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
CMPW $0x08, R5
|
|
BLS repeat_two_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b
|
|
CMPW $0x0c, R5
|
|
BHS cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b
|
|
CMPW $0x00000800, R7
|
|
BLO repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b:
|
|
CMPW $0x00000104, R11
|
|
BLO repeat_three_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b
|
|
CMPW $0x00010100, R11
|
|
BLO repeat_four_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b
|
|
CMPW $0x0100ffff, R11
|
|
BLO repeat_five_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b
|
|
SUB $16842747, R11, R11
|
|
MOVWU R11, R11
|
|
MOVD $0xfffb001d, R16
|
|
MOVW R16, (R1)
|
|
MOVD $0xff, R16
|
|
MOVB R16, 4(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP emit_repeat_again_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b
|
|
|
|
repeat_five_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b:
|
|
SUB $65536, R11, R11
|
|
MOVWU R11, R11
|
|
MOVWU R11, R7
|
|
MOVD $0x001d, R16
|
|
MOVH R16, (R1)
|
|
MOVH R11, 2(R1)
|
|
ASRW $0x10, R7, R7
|
|
MOVB R7, 4(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
repeat_four_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b:
|
|
SUB $256, R11, R11
|
|
MOVWU R11, R11
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R11, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
repeat_three_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b:
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R11, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
repeat_two_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b:
|
|
LSLW $0x02, R11, R11
|
|
ORRW $0x01, R11, R11
|
|
MOVH R11, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b:
|
|
MOVD $0, R5
|
|
ADD R11<<2, R5, R11
|
|
ADD $1, R11, R11
|
|
MOVWU R11, R11
|
|
MOVB R7, 1(R1)
|
|
ASRW $0x08, R7, R7
|
|
LSLW $0x05, R7, R7
|
|
ORRW R7, R11, R11
|
|
MOVB R11, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
long_offset_short_match_nolit_encodeBetterBlockAsm:
|
|
MOVD $0xee, R16
|
|
MOVB R16, (R1)
|
|
MOVH R7, 1(R1)
|
|
SUB $60, R11, R11
|
|
MOVWU R11, R11
|
|
ADD $0x03, R1, R1
|
|
|
|
// emitRepeat
|
|
emit_repeat_again_match_nolit_encodeBetterBlockAsm_emit_copy_short:
|
|
MOVWU R11, R5
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
CMPW $0x08, R5
|
|
BLS repeat_two_match_nolit_encodeBetterBlockAsm_emit_copy_short
|
|
CMPW $0x0c, R5
|
|
BHS cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short
|
|
CMPW $0x00000800, R7
|
|
BLO repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short:
|
|
CMPW $0x00000104, R11
|
|
BLO repeat_three_match_nolit_encodeBetterBlockAsm_emit_copy_short
|
|
CMPW $0x00010100, R11
|
|
BLO repeat_four_match_nolit_encodeBetterBlockAsm_emit_copy_short
|
|
CMPW $0x0100ffff, R11
|
|
BLO repeat_five_match_nolit_encodeBetterBlockAsm_emit_copy_short
|
|
SUB $16842747, R11, R11
|
|
MOVWU R11, R11
|
|
MOVD $0xfffb001d, R16
|
|
MOVW R16, (R1)
|
|
MOVD $0xff, R16
|
|
MOVB R16, 4(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP emit_repeat_again_match_nolit_encodeBetterBlockAsm_emit_copy_short
|
|
|
|
repeat_five_match_nolit_encodeBetterBlockAsm_emit_copy_short:
|
|
SUB $65536, R11, R11
|
|
MOVWU R11, R11
|
|
MOVWU R11, R7
|
|
MOVD $0x001d, R16
|
|
MOVH R16, (R1)
|
|
MOVH R11, 2(R1)
|
|
ASRW $0x10, R7, R7
|
|
MOVB R7, 4(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
repeat_four_match_nolit_encodeBetterBlockAsm_emit_copy_short:
|
|
SUB $256, R11, R11
|
|
MOVWU R11, R11
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R11, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
repeat_three_match_nolit_encodeBetterBlockAsm_emit_copy_short:
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R11, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
repeat_two_match_nolit_encodeBetterBlockAsm_emit_copy_short:
|
|
LSLW $0x02, R11, R11
|
|
ORRW $0x01, R11, R11
|
|
MOVH R11, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short:
|
|
MOVD $0, R5
|
|
ADD R11<<2, R5, R11
|
|
ADD $1, R11, R11
|
|
MOVWU R11, R11
|
|
MOVB R7, 1(R1)
|
|
ASRW $0x08, R7, R7
|
|
LSLW $0x05, R7, R7
|
|
ORRW R7, R11, R11
|
|
MOVB R11, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
two_byte_offset_short_match_nolit_encodeBetterBlockAsm:
|
|
MOVWU R11, R5
|
|
LSLW $0x02, R5, R5
|
|
CMPW $0x0c, R11
|
|
BHS emit_copy_three_match_nolit_encodeBetterBlockAsm
|
|
CMPW $0x00000800, R7
|
|
BHS emit_copy_three_match_nolit_encodeBetterBlockAsm
|
|
SUB $15, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R7, 1(R1)
|
|
LSRW $0x08, R7, R7
|
|
LSLW $0x05, R7, R7
|
|
ORRW R7, R5, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
emit_copy_three_match_nolit_encodeBetterBlockAsm:
|
|
SUB $2, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R5, (R1)
|
|
MOVH R7, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
match_is_repeat_encodeBetterBlockAsm:
|
|
MOVWU 20(RSP), R5
|
|
CMPW R6, R5
|
|
BEQ emit_literal_done_match_emit_repeat_encodeBetterBlockAsm
|
|
MOVWU R6, R8
|
|
MOVW R6, 20(RSP)
|
|
ADD R5, R3, R9
|
|
SUBW R5, R8, R8
|
|
SUB $1, R8, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x3c, R5
|
|
BLO one_byte_match_emit_repeat_encodeBetterBlockAsm
|
|
CMPW $0x00000100, R5
|
|
BLO two_bytes_match_emit_repeat_encodeBetterBlockAsm
|
|
CMPW $0x00010000, R5
|
|
BLO three_bytes_match_emit_repeat_encodeBetterBlockAsm
|
|
CMPW $0x01000000, R5
|
|
BLO four_bytes_match_emit_repeat_encodeBetterBlockAsm
|
|
MOVD $0xfc, R16
|
|
MOVB R16, (R1)
|
|
MOVW R5, 1(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm
|
|
|
|
four_bytes_match_emit_repeat_encodeBetterBlockAsm:
|
|
MOVWU R5, R10
|
|
LSRW $0x10, R10, R10
|
|
MOVD $0xf8, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
MOVB R10, 3(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm
|
|
|
|
three_bytes_match_emit_repeat_encodeBetterBlockAsm:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm
|
|
|
|
two_bytes_match_emit_repeat_encodeBetterBlockAsm:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R5, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R5
|
|
BLO memmove_match_emit_repeat_encodeBetterBlockAsm
|
|
JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm
|
|
|
|
one_byte_match_emit_repeat_encodeBetterBlockAsm:
|
|
LSLW $0x02, R5, R16
|
|
BFI $0, R16, $8, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_match_emit_repeat_encodeBetterBlockAsm:
|
|
ADD R8, R1, R5
|
|
|
|
// genMemMoveShort
|
|
CMP $0x04, R8
|
|
BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_4
|
|
CMP $0x08, R8
|
|
BLO emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_4through7
|
|
CMP $0x10, R8
|
|
BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_8through16
|
|
CMP $0x20, R8
|
|
BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_4:
|
|
MOVWU (R9), R10
|
|
MOVW R10, (R1)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_4through7:
|
|
MOVWU (R9), R10
|
|
ADD R8, R9, R15
|
|
MOVWU -4(R15), R9
|
|
MOVW R10, (R1)
|
|
ADD R8, R1, R15
|
|
MOVW R9, -4(R15)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_8through16:
|
|
MOVD (R9), R10
|
|
ADD R8, R9, R15
|
|
MOVD -8(R15), R9
|
|
MOVD R10, (R1)
|
|
ADD R8, R1, R15
|
|
MOVD R9, -8(R15)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_17through32:
|
|
FMOVQ (R9), F0
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_33through64:
|
|
FMOVQ (R9), F0
|
|
FMOVQ 16(R9), F1
|
|
ADD R8, R9, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm:
|
|
MOVD R5, R1
|
|
JMP emit_literal_done_match_emit_repeat_encodeBetterBlockAsm
|
|
|
|
memmove_long_match_emit_repeat_encodeBetterBlockAsm:
|
|
ADD R8, R1, R5
|
|
|
|
// genMemMoveLong
|
|
MOVD R9, R10
|
|
MOVD R1, R12
|
|
MOVD R8, R13
|
|
|
|
emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsmlarge_big_loop_back:
|
|
FMOVQ (R10), F0
|
|
FMOVQ 16(R10), F1
|
|
FMOVQ F0, (R12)
|
|
FMOVQ F1, 16(R12)
|
|
ADD $0x20, R10, R10
|
|
ADD $0x20, R12, R12
|
|
SUB $0x20, R13, R13
|
|
CMP $0x20, R13
|
|
BHS emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsmlarge_big_loop_back
|
|
ADD R8, R9, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R8, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R5, R1
|
|
|
|
emit_literal_done_match_emit_repeat_encodeBetterBlockAsm:
|
|
ADDW R11, R2, R2
|
|
ADDW $0x04, R11, R11
|
|
MOVW R2, 20(RSP)
|
|
|
|
// emitRepeat
|
|
emit_repeat_again_match_nolit_repeat_encodeBetterBlockAsm:
|
|
MOVWU R11, R5
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
CMPW $0x08, R5
|
|
BLS repeat_two_match_nolit_repeat_encodeBetterBlockAsm
|
|
CMPW $0x0c, R5
|
|
BHS cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm
|
|
CMPW $0x00000800, R7
|
|
BLO repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm
|
|
|
|
cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm:
|
|
CMPW $0x00000104, R11
|
|
BLO repeat_three_match_nolit_repeat_encodeBetterBlockAsm
|
|
CMPW $0x00010100, R11
|
|
BLO repeat_four_match_nolit_repeat_encodeBetterBlockAsm
|
|
CMPW $0x0100ffff, R11
|
|
BLO repeat_five_match_nolit_repeat_encodeBetterBlockAsm
|
|
SUB $16842747, R11, R11
|
|
MOVWU R11, R11
|
|
MOVD $0xfffb001d, R16
|
|
MOVW R16, (R1)
|
|
MOVD $0xff, R16
|
|
MOVB R16, 4(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP emit_repeat_again_match_nolit_repeat_encodeBetterBlockAsm
|
|
|
|
repeat_five_match_nolit_repeat_encodeBetterBlockAsm:
|
|
SUB $65536, R11, R11
|
|
MOVWU R11, R11
|
|
MOVWU R11, R7
|
|
MOVD $0x001d, R16
|
|
MOVH R16, (R1)
|
|
MOVH R11, 2(R1)
|
|
ASRW $0x10, R7, R7
|
|
MOVB R7, 4(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
repeat_four_match_nolit_repeat_encodeBetterBlockAsm:
|
|
SUB $256, R11, R11
|
|
MOVWU R11, R11
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R11, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
repeat_three_match_nolit_repeat_encodeBetterBlockAsm:
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R11, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
repeat_two_match_nolit_repeat_encodeBetterBlockAsm:
|
|
LSLW $0x02, R11, R11
|
|
ORRW $0x01, R11, R11
|
|
MOVH R11, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm
|
|
|
|
repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm:
|
|
MOVD $0, R5
|
|
ADD R11<<2, R5, R11
|
|
ADD $1, R11, R11
|
|
MOVWU R11, R11
|
|
MOVB R7, 1(R1)
|
|
ASRW $0x08, R7, R7
|
|
LSLW $0x05, R7, R7
|
|
ORRW R7, R11, R11
|
|
MOVB R11, (R1)
|
|
ADD $0x02, R1, R1
|
|
|
|
match_nolit_emitcopy_end_encodeBetterBlockAsm:
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R2
|
|
BHS emit_remainder_encodeBetterBlockAsm
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R1
|
|
BLO match_nolit_dst_ok_encodeBetterBlockAsm
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeBetterBlockAsm:
|
|
MOVD $0x00cf1bbcdcbfa563, R5
|
|
MOVD $0x9e3779b1, R7
|
|
ADD $1, R6, R6
|
|
SUB $2, R2, R8
|
|
MOVD (R3)(R6), R9
|
|
ADD R6, R3, R15
|
|
MOVD 1(R15), R10
|
|
MOVD (R3)(R8), R11
|
|
ADD R8, R3, R15
|
|
MOVD 1(R15), R12
|
|
LSL $0x08, R9, R9
|
|
MUL R5, R9, R9
|
|
LSR $0x2f, R9, R9
|
|
LSL $0x20, R10, R10
|
|
MUL R7, R10, R10
|
|
LSR $0x32, R10, R10
|
|
LSL $0x08, R11, R11
|
|
MUL R5, R11, R11
|
|
LSR $0x2f, R11, R11
|
|
LSL $0x20, R12, R12
|
|
MUL R7, R12, R12
|
|
LSR $0x32, R12, R12
|
|
ADD $1, R6, R7
|
|
ADD $1, R8, R13
|
|
MOVW R6, (R0)(R9<<2)
|
|
MOVW R8, (R0)(R11<<2)
|
|
ADD R10<<2, R0, R15
|
|
MOVW R7, 524288(R15)
|
|
ADD R12<<2, R0, R15
|
|
MOVW R13, 524288(R15)
|
|
ADD R6, R8, R7
|
|
ADD $1, R7, R7
|
|
LSR $0x01, R7, R7
|
|
ADD $0x01, R6, R6
|
|
SUB $0x01, R8, R8
|
|
|
|
index_loop_encodeBetterBlockAsm:
|
|
CMP R8, R7
|
|
BHS search_loop_encodeBetterBlockAsm
|
|
MOVD (R3)(R6), R9
|
|
MOVD (R3)(R7), R10
|
|
LSL $0x08, R9, R9
|
|
MUL R5, R9, R9
|
|
LSR $0x2f, R9, R9
|
|
LSL $0x08, R10, R10
|
|
MUL R5, R10, R10
|
|
LSR $0x2f, R10, R10
|
|
MOVW R6, (R0)(R9<<2)
|
|
MOVW R7, (R0)(R10<<2)
|
|
ADD $0x02, R6, R6
|
|
ADD $0x02, R7, R7
|
|
JMP index_loop_encodeBetterBlockAsm
|
|
|
|
emit_remainder_encodeBetterBlockAsm:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R0, R0
|
|
ADD R0, R1, R0
|
|
ADD $5, R0, R0
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R0
|
|
BLO emit_remainder_ok_encodeBetterBlockAsm
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeBetterBlockAsm:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R2
|
|
CMPW R0, R2
|
|
BEQ emit_literal_done_emit_remainder_encodeBetterBlockAsm
|
|
MOVWU R0, R5
|
|
MOVW R0, 20(RSP)
|
|
ADD R2, R3, R0
|
|
SUBW R2, R5, R5
|
|
SUB $1, R5, R2
|
|
MOVWU R2, R2
|
|
CMPW $0x3c, R2
|
|
BLO one_byte_emit_remainder_encodeBetterBlockAsm
|
|
CMPW $0x00000100, R2
|
|
BLO two_bytes_emit_remainder_encodeBetterBlockAsm
|
|
CMPW $0x00010000, R2
|
|
BLO three_bytes_emit_remainder_encodeBetterBlockAsm
|
|
CMPW $0x01000000, R2
|
|
BLO four_bytes_emit_remainder_encodeBetterBlockAsm
|
|
MOVD $0xfc, R16
|
|
MOVB R16, (R1)
|
|
MOVW R2, 1(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP memmove_long_emit_remainder_encodeBetterBlockAsm
|
|
|
|
four_bytes_emit_remainder_encodeBetterBlockAsm:
|
|
MOVWU R2, R3
|
|
LSRW $0x10, R3, R3
|
|
MOVD $0xf8, R16
|
|
MOVB R16, (R1)
|
|
MOVH R2, 1(R1)
|
|
MOVB R3, 3(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP memmove_long_emit_remainder_encodeBetterBlockAsm
|
|
|
|
three_bytes_emit_remainder_encodeBetterBlockAsm:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R2, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_emit_remainder_encodeBetterBlockAsm
|
|
|
|
two_bytes_emit_remainder_encodeBetterBlockAsm:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R2, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R2
|
|
BLO memmove_emit_remainder_encodeBetterBlockAsm
|
|
JMP memmove_long_emit_remainder_encodeBetterBlockAsm
|
|
|
|
one_byte_emit_remainder_encodeBetterBlockAsm:
|
|
LSLW $0x02, R2, R16
|
|
BFI $0, R16, $8, R2
|
|
MOVB R2, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_emit_remainder_encodeBetterBlockAsm:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveShort
|
|
CMP $0x03, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_1or2
|
|
BEQ emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_3
|
|
CMP $0x08, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_4through7
|
|
CMP $0x10, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_8through16
|
|
CMP $0x20, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_1or2:
|
|
MOVBU (R0), R16
|
|
BFI $0, R16, $8, R5
|
|
ADD R3, R0, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVB R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVB R0, -1(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_3:
|
|
MOVHU (R0), R16
|
|
BFI $0, R16, $16, R5
|
|
MOVBU 2(R0), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVH R5, (R1)
|
|
MOVB R0, 2(R1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_4through7:
|
|
MOVWU (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVWU -4(R15), R0
|
|
MOVW R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVW R0, -4(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_8through16:
|
|
MOVD (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVD -8(R15), R0
|
|
MOVD R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVD R0, -8(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_17through32:
|
|
FMOVQ (R0), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_33through64:
|
|
FMOVQ (R0), F0
|
|
FMOVQ 16(R0), F1
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_emit_remainder_encodeBetterBlockAsm:
|
|
MOVD R2, R1
|
|
JMP emit_literal_done_emit_remainder_encodeBetterBlockAsm
|
|
|
|
memmove_long_emit_remainder_encodeBetterBlockAsm:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveLong
|
|
MOVD R0, R5
|
|
MOVD R1, R6
|
|
MOVD R3, R7
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsmlarge_big_loop_back:
|
|
FMOVQ (R5), F0
|
|
FMOVQ 16(R5), F1
|
|
FMOVQ F0, (R6)
|
|
FMOVQ F1, 16(R6)
|
|
ADD $0x20, R5, R5
|
|
ADD $0x20, R6, R6
|
|
SUB $0x20, R7, R7
|
|
CMP $0x20, R7
|
|
BHS emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsmlarge_big_loop_back
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R3, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R2, R1
|
|
|
|
emit_literal_done_emit_remainder_encodeBetterBlockAsm:
|
|
MOVD dst_base+0(FP), R0
|
|
SUB R0, R1, R1
|
|
MOVD R1, ret+56(FP)
|
|
RET
|
|
|
|
// func encodeBetterBlockAsm4MB(dst []byte, src []byte, tmp *[589824]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeBetterBlockAsm4MB(SB), $24-64
|
|
MOVD tmp+48(FP), R0
|
|
MOVD dst_base+0(FP), R1
|
|
MOVD $0x00001200, R2
|
|
MOVD R0, R3
|
|
VEOR V0.B16, V0.B16, V0.B16
|
|
|
|
zero_loop_encodeBetterBlockAsm4MB:
|
|
FMOVQ F0, (R3)
|
|
FMOVQ F0, 16(R3)
|
|
FMOVQ F0, 32(R3)
|
|
FMOVQ F0, 48(R3)
|
|
FMOVQ F0, 64(R3)
|
|
FMOVQ F0, 80(R3)
|
|
FMOVQ F0, 96(R3)
|
|
FMOVQ F0, 112(R3)
|
|
ADD $0x80, R3, R3
|
|
SUBS $1, R2, R2
|
|
BNE zero_loop_encodeBetterBlockAsm4MB
|
|
MOVD $0x00000000, R16
|
|
MOVW R16, 20(RSP)
|
|
MOVD src_len+32(FP), R2
|
|
SUB $6, R2, R3
|
|
SUB $8, R2, R5
|
|
MOVW R5, 16(RSP)
|
|
LSR $0x05, R2, R2
|
|
SUBW R2, R3, R3
|
|
ADD R3, R1, R3
|
|
MOVD R3, 8(RSP)
|
|
MOVD $0x00000001, R2
|
|
MOVD $0x00000000, R16
|
|
MOVW R16, 24(RSP)
|
|
MOVD src_base+24(FP), R3
|
|
|
|
search_loop_encodeBetterBlockAsm4MB:
|
|
MOVWU R2, R5
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R5, R5
|
|
LSRW $0x07, R5, R5
|
|
CMPW $0x63, R5
|
|
BLS check_maxskip_ok_encodeBetterBlockAsm4MB
|
|
ADD $100, R2, R5
|
|
MOVWU R5, R5
|
|
JMP check_maxskip_cont_encodeBetterBlockAsm4MB
|
|
|
|
check_maxskip_ok_encodeBetterBlockAsm4MB:
|
|
ADD R5, R2, R5
|
|
ADD $1, R5, R5
|
|
MOVWU R5, R5
|
|
|
|
check_maxskip_cont_encodeBetterBlockAsm4MB:
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R5
|
|
BHS emit_remainder_encodeBetterBlockAsm4MB
|
|
MOVD (R3)(R2), R6
|
|
MOVW R5, 28(RSP)
|
|
MOVD $0x00cf1bbcdcbfa563, R8
|
|
MOVD $0x9e3779b1, R5
|
|
MOVD R6, R9
|
|
MOVD R6, R10
|
|
LSL $0x08, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x2f, R9, R9
|
|
LSL $0x20, R10, R10
|
|
MUL R5, R10, R10
|
|
LSR $0x32, R10, R10
|
|
MOVWU (R0)(R9<<2), R5
|
|
ADD R10<<2, R0, R15
|
|
MOVWU 524288(R15), R7
|
|
MOVW R2, (R0)(R9<<2)
|
|
ADD R10<<2, R0, R15
|
|
MOVW R2, 524288(R15)
|
|
MOVD (R3)(R5), R9
|
|
MOVD (R3)(R7), R10
|
|
CMP R6, R9
|
|
BEQ candidate_match_encodeBetterBlockAsm4MB
|
|
CMP R6, R10
|
|
BNE no_short_found_encodeBetterBlockAsm4MB
|
|
MOVWU R7, R5
|
|
JMP candidate_match_encodeBetterBlockAsm4MB
|
|
|
|
no_short_found_encodeBetterBlockAsm4MB:
|
|
CMPW R6, R9
|
|
BEQ candidate_match_encodeBetterBlockAsm4MB
|
|
CMPW R6, R10
|
|
BEQ candidateS_match_encodeBetterBlockAsm4MB
|
|
MOVWU 28(RSP), R2
|
|
JMP search_loop_encodeBetterBlockAsm4MB
|
|
|
|
candidateS_match_encodeBetterBlockAsm4MB:
|
|
LSR $0x08, R6, R6
|
|
MOVD R6, R9
|
|
LSL $0x08, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x2f, R9, R9
|
|
MOVWU (R0)(R9<<2), R5
|
|
ADDW $1, R2, R2
|
|
MOVW R2, (R0)(R9<<2)
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ candidate_match_encodeBetterBlockAsm4MB
|
|
SUBW $1, R2, R2
|
|
MOVWU R7, R5
|
|
|
|
candidate_match_encodeBetterBlockAsm4MB:
|
|
MOVWU 20(RSP), R6
|
|
TSTW R5, R5
|
|
BEQ match_extend_back_end_encodeBetterBlockAsm4MB
|
|
|
|
match_extend_back_loop_encodeBetterBlockAsm4MB:
|
|
CMPW R6, R2
|
|
BLS match_extend_back_end_encodeBetterBlockAsm4MB
|
|
ADD R5, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R7
|
|
ADD R2, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R8
|
|
AND $0xff, R8, R16
|
|
AND $0xff, R7, R15
|
|
CMP R16, R15
|
|
BNE match_extend_back_end_encodeBetterBlockAsm4MB
|
|
SUB $1, R2, R2
|
|
MOVWU R2, R2
|
|
SUBSW $1, R5, R5
|
|
BEQ match_extend_back_end_encodeBetterBlockAsm4MB
|
|
JMP match_extend_back_loop_encodeBetterBlockAsm4MB
|
|
|
|
match_extend_back_end_encodeBetterBlockAsm4MB:
|
|
MOVWU R2, R6
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R6, R6
|
|
ADD R6, R1, R6
|
|
ADD $4, R6, R6
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R6
|
|
BLO match_dst_size_check_encodeBetterBlockAsm4MB
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeBetterBlockAsm4MB:
|
|
MOVWU R2, R6
|
|
ADDW $0x04, R2, R2
|
|
ADDW $0x04, R5, R5
|
|
MOVD src_len+32(FP), R7
|
|
SUBW R2, R7, R7
|
|
ADD R2, R3, R8
|
|
ADD R5, R3, R9
|
|
|
|
// matchLen
|
|
MOVD $0, R11
|
|
|
|
matchlen_loopback_16_match_nolit_encodeBetterBlockAsm4MB:
|
|
CMPW $0x10, R7
|
|
BLO matchlen_match8_match_nolit_encodeBetterBlockAsm4MB
|
|
MOVD (R8)(R11), R10
|
|
ADD R11, R8, R15
|
|
MOVD 8(R15), R12
|
|
MOVD (R9)(R11), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_8_match_nolit_encodeBetterBlockAsm4MB
|
|
ADD R11, R9, R15
|
|
MOVD 8(R15), R16
|
|
EOR R16, R12, R12
|
|
TST R12, R12
|
|
BNE matchlen_bsf_16match_nolit_encodeBetterBlockAsm4MB
|
|
SUB $16, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $16, R11, R11
|
|
MOVWU R11, R11
|
|
JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm4MB
|
|
|
|
matchlen_bsf_16match_nolit_encodeBetterBlockAsm4MB:
|
|
RBIT R12, R12
|
|
CLZ R12, R12
|
|
ASR $0x03, R12, R12
|
|
ADD R12, R11, R11
|
|
ADD $8, R11, R11
|
|
MOVWU R11, R11
|
|
JMP match_nolit_end_encodeBetterBlockAsm4MB
|
|
|
|
matchlen_match8_match_nolit_encodeBetterBlockAsm4MB:
|
|
CMPW $0x08, R7
|
|
BLO matchlen_match4_match_nolit_encodeBetterBlockAsm4MB
|
|
MOVD (R8)(R11), R10
|
|
MOVD (R9)(R11), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_8_match_nolit_encodeBetterBlockAsm4MB
|
|
SUB $8, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $8, R11, R11
|
|
MOVWU R11, R11
|
|
JMP matchlen_match4_match_nolit_encodeBetterBlockAsm4MB
|
|
|
|
matchlen_bsf_8_match_nolit_encodeBetterBlockAsm4MB:
|
|
RBIT R10, R10
|
|
CLZ R10, R10
|
|
ASR $0x03, R10, R10
|
|
ADD R10, R11, R11
|
|
MOVWU R11, R11
|
|
JMP match_nolit_end_encodeBetterBlockAsm4MB
|
|
|
|
matchlen_match4_match_nolit_encodeBetterBlockAsm4MB:
|
|
CMPW $0x04, R7
|
|
BLO matchlen_match2_match_nolit_encodeBetterBlockAsm4MB
|
|
MOVWU (R8)(R11), R10
|
|
MOVWU (R9)(R11), R16
|
|
CMPW R10, R16
|
|
BNE matchlen_match2_match_nolit_encodeBetterBlockAsm4MB
|
|
SUB $4, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $4, R11, R11
|
|
MOVWU R11, R11
|
|
|
|
matchlen_match2_match_nolit_encodeBetterBlockAsm4MB:
|
|
CMPW $0x01, R7
|
|
BEQ matchlen_match1_match_nolit_encodeBetterBlockAsm4MB
|
|
BLO match_nolit_end_encodeBetterBlockAsm4MB
|
|
MOVHU (R8)(R11), R16
|
|
BFI $0, R16, $16, R10
|
|
ADD R11, R9, R15
|
|
MOVHU (R15), R15
|
|
AND $0xffff, R10, R16
|
|
CMP R16, R15
|
|
BNE matchlen_match1_match_nolit_encodeBetterBlockAsm4MB
|
|
ADD $2, R11, R11
|
|
MOVWU R11, R11
|
|
SUBSW $0x02, R7, R7
|
|
BEQ match_nolit_end_encodeBetterBlockAsm4MB
|
|
|
|
matchlen_match1_match_nolit_encodeBetterBlockAsm4MB:
|
|
MOVBU (R8)(R11), R16
|
|
BFI $0, R16, $8, R10
|
|
ADD R11, R9, R15
|
|
MOVBU (R15), R15
|
|
AND $0xff, R10, R16
|
|
CMP R16, R15
|
|
BNE match_nolit_end_encodeBetterBlockAsm4MB
|
|
ADD $1, R11, R11
|
|
MOVWU R11, R11
|
|
|
|
match_nolit_end_encodeBetterBlockAsm4MB:
|
|
MOVWU R2, R7
|
|
SUBW R5, R7, R7
|
|
|
|
// Check if repeat
|
|
MOVWU 24(RSP), R16
|
|
CMPW R7, R16
|
|
BEQ match_is_repeat_encodeBetterBlockAsm4MB
|
|
CMPW $0x01, R11
|
|
BHI match_length_ok_encodeBetterBlockAsm4MB
|
|
CMPW $0x0000ffff, R7
|
|
BLS match_length_ok_encodeBetterBlockAsm4MB
|
|
MOVWU 28(RSP), R2
|
|
ADDW $1, R2, R2
|
|
JMP search_loop_encodeBetterBlockAsm4MB
|
|
|
|
match_length_ok_encodeBetterBlockAsm4MB:
|
|
MOVW R7, 24(RSP)
|
|
MOVWU 20(RSP), R5
|
|
CMPW R6, R5
|
|
BEQ emit_literal_done_match_emit_encodeBetterBlockAsm4MB
|
|
MOVWU R6, R8
|
|
MOVW R6, 20(RSP)
|
|
ADD R5, R3, R9
|
|
SUBW R5, R8, R8
|
|
SUB $1, R8, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x3c, R5
|
|
BLO one_byte_match_emit_encodeBetterBlockAsm4MB
|
|
CMPW $0x00000100, R5
|
|
BLO two_bytes_match_emit_encodeBetterBlockAsm4MB
|
|
CMPW $0x00010000, R5
|
|
BLO three_bytes_match_emit_encodeBetterBlockAsm4MB
|
|
MOVWU R5, R10
|
|
LSRW $0x10, R10, R10
|
|
MOVD $0xf8, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
MOVB R10, 3(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP memmove_long_match_emit_encodeBetterBlockAsm4MB
|
|
|
|
three_bytes_match_emit_encodeBetterBlockAsm4MB:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_match_emit_encodeBetterBlockAsm4MB
|
|
|
|
two_bytes_match_emit_encodeBetterBlockAsm4MB:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R5, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R5
|
|
BLO memmove_match_emit_encodeBetterBlockAsm4MB
|
|
JMP memmove_long_match_emit_encodeBetterBlockAsm4MB
|
|
|
|
one_byte_match_emit_encodeBetterBlockAsm4MB:
|
|
LSLW $0x02, R5, R16
|
|
BFI $0, R16, $8, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_match_emit_encodeBetterBlockAsm4MB:
|
|
ADD R8, R1, R5
|
|
|
|
// genMemMoveShort
|
|
CMP $0x04, R8
|
|
BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_4
|
|
CMP $0x08, R8
|
|
BLO emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_4through7
|
|
CMP $0x10, R8
|
|
BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_8through16
|
|
CMP $0x20, R8
|
|
BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_4:
|
|
MOVWU (R9), R10
|
|
MOVW R10, (R1)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm4MB
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_4through7:
|
|
MOVWU (R9), R10
|
|
ADD R8, R9, R15
|
|
MOVWU -4(R15), R9
|
|
MOVW R10, (R1)
|
|
ADD R8, R1, R15
|
|
MOVW R9, -4(R15)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm4MB
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_8through16:
|
|
MOVD (R9), R10
|
|
ADD R8, R9, R15
|
|
MOVD -8(R15), R9
|
|
MOVD R10, (R1)
|
|
ADD R8, R1, R15
|
|
MOVD R9, -8(R15)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm4MB
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_17through32:
|
|
FMOVQ (R9), F0
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm4MB
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_33through64:
|
|
FMOVQ (R9), F0
|
|
FMOVQ 16(R9), F1
|
|
ADD R8, R9, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_match_emit_encodeBetterBlockAsm4MB:
|
|
MOVD R5, R1
|
|
JMP emit_literal_done_match_emit_encodeBetterBlockAsm4MB
|
|
|
|
memmove_long_match_emit_encodeBetterBlockAsm4MB:
|
|
ADD R8, R1, R5
|
|
|
|
// genMemMoveLong
|
|
MOVD R9, R10
|
|
MOVD R1, R12
|
|
MOVD R8, R13
|
|
|
|
emit_lit_memmove_long_match_emit_encodeBetterBlockAsm4MBlarge_big_loop_back:
|
|
FMOVQ (R10), F0
|
|
FMOVQ 16(R10), F1
|
|
FMOVQ F0, (R12)
|
|
FMOVQ F1, 16(R12)
|
|
ADD $0x20, R10, R10
|
|
ADD $0x20, R12, R12
|
|
SUB $0x20, R13, R13
|
|
CMP $0x20, R13
|
|
BHS emit_lit_memmove_long_match_emit_encodeBetterBlockAsm4MBlarge_big_loop_back
|
|
ADD R8, R9, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R8, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R5, R1
|
|
|
|
emit_literal_done_match_emit_encodeBetterBlockAsm4MB:
|
|
ADDW R11, R2, R2
|
|
ADDW $0x04, R11, R11
|
|
MOVW R2, 20(RSP)
|
|
|
|
// emitCopy
|
|
CMPW $0x00010000, R7
|
|
BLO two_byte_offset_match_nolit_encodeBetterBlockAsm4MB
|
|
CMPW $0x40, R11
|
|
BLS four_bytes_remain_match_nolit_encodeBetterBlockAsm4MB
|
|
MOVD $0xff, R16
|
|
MOVB R16, (R1)
|
|
MOVW R7, 1(R1)
|
|
SUB $64, R11, R11
|
|
MOVWU R11, R11
|
|
ADD $0x05, R1, R1
|
|
CMPW $0x04, R11
|
|
BLO four_bytes_remain_match_nolit_encodeBetterBlockAsm4MB
|
|
|
|
// emitRepeat
|
|
MOVWU R11, R5
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
CMPW $0x08, R5
|
|
BLS repeat_two_match_nolit_encodeBetterBlockAsm4MB_emit_copy
|
|
CMPW $0x0c, R5
|
|
BHS cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy
|
|
CMPW $0x00000800, R7
|
|
BLO repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy:
|
|
CMPW $0x00000104, R11
|
|
BLO repeat_three_match_nolit_encodeBetterBlockAsm4MB_emit_copy
|
|
CMPW $0x00010100, R11
|
|
BLO repeat_four_match_nolit_encodeBetterBlockAsm4MB_emit_copy
|
|
SUB $65536, R11, R11
|
|
MOVWU R11, R11
|
|
MOVWU R11, R7
|
|
MOVD $0x001d, R16
|
|
MOVH R16, (R1)
|
|
MOVH R11, 2(R1)
|
|
ASRW $0x10, R7, R7
|
|
MOVB R7, 4(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
repeat_four_match_nolit_encodeBetterBlockAsm4MB_emit_copy:
|
|
SUB $256, R11, R11
|
|
MOVWU R11, R11
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R11, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
repeat_three_match_nolit_encodeBetterBlockAsm4MB_emit_copy:
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R11, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
repeat_two_match_nolit_encodeBetterBlockAsm4MB_emit_copy:
|
|
LSLW $0x02, R11, R11
|
|
ORRW $0x01, R11, R11
|
|
MOVH R11, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy:
|
|
MOVD $0, R5
|
|
ADD R11<<2, R5, R11
|
|
ADD $1, R11, R11
|
|
MOVWU R11, R11
|
|
MOVB R7, 1(R1)
|
|
ASRW $0x08, R7, R7
|
|
LSLW $0x05, R7, R7
|
|
ORRW R7, R11, R11
|
|
MOVB R11, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
four_bytes_remain_match_nolit_encodeBetterBlockAsm4MB:
|
|
TSTW R11, R11
|
|
BEQ match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
MOVD $0, R5
|
|
ADD R11<<2, R5, R11
|
|
SUB $1, R11, R11
|
|
MOVWU R11, R11
|
|
MOVB R11, (R1)
|
|
MOVW R7, 1(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
two_byte_offset_match_nolit_encodeBetterBlockAsm4MB:
|
|
CMPW $0x40, R11
|
|
BLS two_byte_offset_short_match_nolit_encodeBetterBlockAsm4MB
|
|
CMPW $0x00000800, R7
|
|
BHS long_offset_short_match_nolit_encodeBetterBlockAsm4MB
|
|
MOVD $0x00000001, R5
|
|
ADD $16, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R7, 1(R1)
|
|
LSRW $0x08, R7, R7
|
|
LSLW $0x05, R7, R7
|
|
ORRW R7, R5, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
SUBW $0x08, R11, R11
|
|
|
|
// emitRepeat
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
JMP cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b
|
|
MOVWU R11, R5
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
CMPW $0x08, R5
|
|
BLS repeat_two_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b
|
|
CMPW $0x0c, R5
|
|
BHS cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b
|
|
CMPW $0x00000800, R7
|
|
BLO repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b:
|
|
CMPW $0x00000104, R11
|
|
BLO repeat_three_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b
|
|
CMPW $0x00010100, R11
|
|
BLO repeat_four_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b
|
|
SUB $65536, R11, R11
|
|
MOVWU R11, R11
|
|
MOVWU R11, R7
|
|
MOVD $0x001d, R16
|
|
MOVH R16, (R1)
|
|
MOVH R11, 2(R1)
|
|
ASRW $0x10, R7, R7
|
|
MOVB R7, 4(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
repeat_four_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b:
|
|
SUB $256, R11, R11
|
|
MOVWU R11, R11
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R11, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
repeat_three_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b:
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R11, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
repeat_two_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b:
|
|
LSLW $0x02, R11, R11
|
|
ORRW $0x01, R11, R11
|
|
MOVH R11, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b:
|
|
MOVD $0, R5
|
|
ADD R11<<2, R5, R11
|
|
ADD $1, R11, R11
|
|
MOVWU R11, R11
|
|
MOVB R7, 1(R1)
|
|
ASRW $0x08, R7, R7
|
|
LSLW $0x05, R7, R7
|
|
ORRW R7, R11, R11
|
|
MOVB R11, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
long_offset_short_match_nolit_encodeBetterBlockAsm4MB:
|
|
MOVD $0xee, R16
|
|
MOVB R16, (R1)
|
|
MOVH R7, 1(R1)
|
|
SUB $60, R11, R11
|
|
MOVWU R11, R11
|
|
ADD $0x03, R1, R1
|
|
|
|
// emitRepeat
|
|
MOVWU R11, R5
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
CMPW $0x08, R5
|
|
BLS repeat_two_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short
|
|
CMPW $0x0c, R5
|
|
BHS cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short
|
|
CMPW $0x00000800, R7
|
|
BLO repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short:
|
|
CMPW $0x00000104, R11
|
|
BLO repeat_three_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short
|
|
CMPW $0x00010100, R11
|
|
BLO repeat_four_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short
|
|
SUB $65536, R11, R11
|
|
MOVWU R11, R11
|
|
MOVWU R11, R7
|
|
MOVD $0x001d, R16
|
|
MOVH R16, (R1)
|
|
MOVH R11, 2(R1)
|
|
ASRW $0x10, R7, R7
|
|
MOVB R7, 4(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
repeat_four_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short:
|
|
SUB $256, R11, R11
|
|
MOVWU R11, R11
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R11, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
repeat_three_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short:
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R11, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
repeat_two_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short:
|
|
LSLW $0x02, R11, R11
|
|
ORRW $0x01, R11, R11
|
|
MOVH R11, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short:
|
|
MOVD $0, R5
|
|
ADD R11<<2, R5, R11
|
|
ADD $1, R11, R11
|
|
MOVWU R11, R11
|
|
MOVB R7, 1(R1)
|
|
ASRW $0x08, R7, R7
|
|
LSLW $0x05, R7, R7
|
|
ORRW R7, R11, R11
|
|
MOVB R11, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
two_byte_offset_short_match_nolit_encodeBetterBlockAsm4MB:
|
|
MOVWU R11, R5
|
|
LSLW $0x02, R5, R5
|
|
CMPW $0x0c, R11
|
|
BHS emit_copy_three_match_nolit_encodeBetterBlockAsm4MB
|
|
CMPW $0x00000800, R7
|
|
BHS emit_copy_three_match_nolit_encodeBetterBlockAsm4MB
|
|
SUB $15, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R7, 1(R1)
|
|
LSRW $0x08, R7, R7
|
|
LSLW $0x05, R7, R7
|
|
ORRW R7, R5, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
emit_copy_three_match_nolit_encodeBetterBlockAsm4MB:
|
|
SUB $2, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R5, (R1)
|
|
MOVH R7, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
match_is_repeat_encodeBetterBlockAsm4MB:
|
|
MOVWU 20(RSP), R5
|
|
CMPW R6, R5
|
|
BEQ emit_literal_done_match_emit_repeat_encodeBetterBlockAsm4MB
|
|
MOVWU R6, R8
|
|
MOVW R6, 20(RSP)
|
|
ADD R5, R3, R9
|
|
SUBW R5, R8, R8
|
|
SUB $1, R8, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x3c, R5
|
|
BLO one_byte_match_emit_repeat_encodeBetterBlockAsm4MB
|
|
CMPW $0x00000100, R5
|
|
BLO two_bytes_match_emit_repeat_encodeBetterBlockAsm4MB
|
|
CMPW $0x00010000, R5
|
|
BLO three_bytes_match_emit_repeat_encodeBetterBlockAsm4MB
|
|
MOVWU R5, R10
|
|
LSRW $0x10, R10, R10
|
|
MOVD $0xf8, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
MOVB R10, 3(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm4MB
|
|
|
|
three_bytes_match_emit_repeat_encodeBetterBlockAsm4MB:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm4MB
|
|
|
|
two_bytes_match_emit_repeat_encodeBetterBlockAsm4MB:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R5, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R5
|
|
BLO memmove_match_emit_repeat_encodeBetterBlockAsm4MB
|
|
JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm4MB
|
|
|
|
one_byte_match_emit_repeat_encodeBetterBlockAsm4MB:
|
|
LSLW $0x02, R5, R16
|
|
BFI $0, R16, $8, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_match_emit_repeat_encodeBetterBlockAsm4MB:
|
|
ADD R8, R1, R5
|
|
|
|
// genMemMoveShort
|
|
CMP $0x04, R8
|
|
BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_4
|
|
CMP $0x08, R8
|
|
BLO emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_4through7
|
|
CMP $0x10, R8
|
|
BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_8through16
|
|
CMP $0x20, R8
|
|
BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_4:
|
|
MOVWU (R9), R10
|
|
MOVW R10, (R1)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm4MB
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_4through7:
|
|
MOVWU (R9), R10
|
|
ADD R8, R9, R15
|
|
MOVWU -4(R15), R9
|
|
MOVW R10, (R1)
|
|
ADD R8, R1, R15
|
|
MOVW R9, -4(R15)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm4MB
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_8through16:
|
|
MOVD (R9), R10
|
|
ADD R8, R9, R15
|
|
MOVD -8(R15), R9
|
|
MOVD R10, (R1)
|
|
ADD R8, R1, R15
|
|
MOVD R9, -8(R15)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm4MB
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_17through32:
|
|
FMOVQ (R9), F0
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm4MB
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_33through64:
|
|
FMOVQ (R9), F0
|
|
FMOVQ 16(R9), F1
|
|
ADD R8, R9, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm4MB:
|
|
MOVD R5, R1
|
|
JMP emit_literal_done_match_emit_repeat_encodeBetterBlockAsm4MB
|
|
|
|
memmove_long_match_emit_repeat_encodeBetterBlockAsm4MB:
|
|
ADD R8, R1, R5
|
|
|
|
// genMemMoveLong
|
|
MOVD R9, R10
|
|
MOVD R1, R12
|
|
MOVD R8, R13
|
|
|
|
emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm4MBlarge_big_loop_back:
|
|
FMOVQ (R10), F0
|
|
FMOVQ 16(R10), F1
|
|
FMOVQ F0, (R12)
|
|
FMOVQ F1, 16(R12)
|
|
ADD $0x20, R10, R10
|
|
ADD $0x20, R12, R12
|
|
SUB $0x20, R13, R13
|
|
CMP $0x20, R13
|
|
BHS emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm4MBlarge_big_loop_back
|
|
ADD R8, R9, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R8, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R5, R1
|
|
|
|
emit_literal_done_match_emit_repeat_encodeBetterBlockAsm4MB:
|
|
ADDW R11, R2, R2
|
|
ADDW $0x04, R11, R11
|
|
MOVW R2, 20(RSP)
|
|
|
|
// emitRepeat
|
|
MOVWU R11, R5
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
CMPW $0x08, R5
|
|
BLS repeat_two_match_nolit_repeat_encodeBetterBlockAsm4MB
|
|
CMPW $0x0c, R5
|
|
BHS cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm4MB
|
|
CMPW $0x00000800, R7
|
|
BLO repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm4MB
|
|
|
|
cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm4MB:
|
|
CMPW $0x00000104, R11
|
|
BLO repeat_three_match_nolit_repeat_encodeBetterBlockAsm4MB
|
|
CMPW $0x00010100, R11
|
|
BLO repeat_four_match_nolit_repeat_encodeBetterBlockAsm4MB
|
|
SUB $65536, R11, R11
|
|
MOVWU R11, R11
|
|
MOVWU R11, R7
|
|
MOVD $0x001d, R16
|
|
MOVH R16, (R1)
|
|
MOVH R11, 2(R1)
|
|
ASRW $0x10, R7, R7
|
|
MOVB R7, 4(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
repeat_four_match_nolit_repeat_encodeBetterBlockAsm4MB:
|
|
SUB $256, R11, R11
|
|
MOVWU R11, R11
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R11, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
repeat_three_match_nolit_repeat_encodeBetterBlockAsm4MB:
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R11, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
repeat_two_match_nolit_repeat_encodeBetterBlockAsm4MB:
|
|
LSLW $0x02, R11, R11
|
|
ORRW $0x01, R11, R11
|
|
MOVH R11, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
|
|
|
|
repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm4MB:
|
|
MOVD $0, R5
|
|
ADD R11<<2, R5, R11
|
|
ADD $1, R11, R11
|
|
MOVWU R11, R11
|
|
MOVB R7, 1(R1)
|
|
ASRW $0x08, R7, R7
|
|
LSLW $0x05, R7, R7
|
|
ORRW R7, R11, R11
|
|
MOVB R11, (R1)
|
|
ADD $0x02, R1, R1
|
|
|
|
match_nolit_emitcopy_end_encodeBetterBlockAsm4MB:
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R2
|
|
BHS emit_remainder_encodeBetterBlockAsm4MB
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R1
|
|
BLO match_nolit_dst_ok_encodeBetterBlockAsm4MB
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeBetterBlockAsm4MB:
|
|
MOVD $0x00cf1bbcdcbfa563, R5
|
|
MOVD $0x9e3779b1, R7
|
|
ADD $1, R6, R6
|
|
SUB $2, R2, R8
|
|
MOVD (R3)(R6), R9
|
|
ADD R6, R3, R15
|
|
MOVD 1(R15), R10
|
|
MOVD (R3)(R8), R11
|
|
ADD R8, R3, R15
|
|
MOVD 1(R15), R12
|
|
LSL $0x08, R9, R9
|
|
MUL R5, R9, R9
|
|
LSR $0x2f, R9, R9
|
|
LSL $0x20, R10, R10
|
|
MUL R7, R10, R10
|
|
LSR $0x32, R10, R10
|
|
LSL $0x08, R11, R11
|
|
MUL R5, R11, R11
|
|
LSR $0x2f, R11, R11
|
|
LSL $0x20, R12, R12
|
|
MUL R7, R12, R12
|
|
LSR $0x32, R12, R12
|
|
ADD $1, R6, R7
|
|
ADD $1, R8, R13
|
|
MOVW R6, (R0)(R9<<2)
|
|
MOVW R8, (R0)(R11<<2)
|
|
ADD R10<<2, R0, R15
|
|
MOVW R7, 524288(R15)
|
|
ADD R12<<2, R0, R15
|
|
MOVW R13, 524288(R15)
|
|
ADD R6, R8, R7
|
|
ADD $1, R7, R7
|
|
LSR $0x01, R7, R7
|
|
ADD $0x01, R6, R6
|
|
SUB $0x01, R8, R8
|
|
|
|
index_loop_encodeBetterBlockAsm4MB:
|
|
CMP R8, R7
|
|
BHS search_loop_encodeBetterBlockAsm4MB
|
|
MOVD (R3)(R6), R9
|
|
MOVD (R3)(R7), R10
|
|
LSL $0x08, R9, R9
|
|
MUL R5, R9, R9
|
|
LSR $0x2f, R9, R9
|
|
LSL $0x08, R10, R10
|
|
MUL R5, R10, R10
|
|
LSR $0x2f, R10, R10
|
|
MOVW R6, (R0)(R9<<2)
|
|
MOVW R7, (R0)(R10<<2)
|
|
ADD $0x02, R6, R6
|
|
ADD $0x02, R7, R7
|
|
JMP index_loop_encodeBetterBlockAsm4MB
|
|
|
|
emit_remainder_encodeBetterBlockAsm4MB:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R0, R0
|
|
ADD R0, R1, R0
|
|
ADD $4, R0, R0
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R0
|
|
BLO emit_remainder_ok_encodeBetterBlockAsm4MB
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeBetterBlockAsm4MB:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R2
|
|
CMPW R0, R2
|
|
BEQ emit_literal_done_emit_remainder_encodeBetterBlockAsm4MB
|
|
MOVWU R0, R5
|
|
MOVW R0, 20(RSP)
|
|
ADD R2, R3, R0
|
|
SUBW R2, R5, R5
|
|
SUB $1, R5, R2
|
|
MOVWU R2, R2
|
|
CMPW $0x3c, R2
|
|
BLO one_byte_emit_remainder_encodeBetterBlockAsm4MB
|
|
CMPW $0x00000100, R2
|
|
BLO two_bytes_emit_remainder_encodeBetterBlockAsm4MB
|
|
CMPW $0x00010000, R2
|
|
BLO three_bytes_emit_remainder_encodeBetterBlockAsm4MB
|
|
MOVWU R2, R3
|
|
LSRW $0x10, R3, R3
|
|
MOVD $0xf8, R16
|
|
MOVB R16, (R1)
|
|
MOVH R2, 1(R1)
|
|
MOVB R3, 3(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP memmove_long_emit_remainder_encodeBetterBlockAsm4MB
|
|
|
|
three_bytes_emit_remainder_encodeBetterBlockAsm4MB:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R2, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_emit_remainder_encodeBetterBlockAsm4MB
|
|
|
|
two_bytes_emit_remainder_encodeBetterBlockAsm4MB:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R2, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R2
|
|
BLO memmove_emit_remainder_encodeBetterBlockAsm4MB
|
|
JMP memmove_long_emit_remainder_encodeBetterBlockAsm4MB
|
|
|
|
one_byte_emit_remainder_encodeBetterBlockAsm4MB:
|
|
LSLW $0x02, R2, R16
|
|
BFI $0, R16, $8, R2
|
|
MOVB R2, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_emit_remainder_encodeBetterBlockAsm4MB:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveShort
|
|
CMP $0x03, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_1or2
|
|
BEQ emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_3
|
|
CMP $0x08, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_4through7
|
|
CMP $0x10, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_8through16
|
|
CMP $0x20, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_1or2:
|
|
MOVBU (R0), R16
|
|
BFI $0, R16, $8, R5
|
|
ADD R3, R0, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVB R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVB R0, -1(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm4MB
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_3:
|
|
MOVHU (R0), R16
|
|
BFI $0, R16, $16, R5
|
|
MOVBU 2(R0), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVH R5, (R1)
|
|
MOVB R0, 2(R1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm4MB
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_4through7:
|
|
MOVWU (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVWU -4(R15), R0
|
|
MOVW R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVW R0, -4(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm4MB
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_8through16:
|
|
MOVD (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVD -8(R15), R0
|
|
MOVD R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVD R0, -8(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm4MB
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_17through32:
|
|
FMOVQ (R0), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm4MB
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_33through64:
|
|
FMOVQ (R0), F0
|
|
FMOVQ 16(R0), F1
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_emit_remainder_encodeBetterBlockAsm4MB:
|
|
MOVD R2, R1
|
|
JMP emit_literal_done_emit_remainder_encodeBetterBlockAsm4MB
|
|
|
|
memmove_long_emit_remainder_encodeBetterBlockAsm4MB:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveLong
|
|
MOVD R0, R5
|
|
MOVD R1, R6
|
|
MOVD R3, R7
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm4MBlarge_big_loop_back:
|
|
FMOVQ (R5), F0
|
|
FMOVQ 16(R5), F1
|
|
FMOVQ F0, (R6)
|
|
FMOVQ F1, 16(R6)
|
|
ADD $0x20, R5, R5
|
|
ADD $0x20, R6, R6
|
|
SUB $0x20, R7, R7
|
|
CMP $0x20, R7
|
|
BHS emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm4MBlarge_big_loop_back
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R3, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R2, R1
|
|
|
|
emit_literal_done_emit_remainder_encodeBetterBlockAsm4MB:
|
|
MOVD dst_base+0(FP), R0
|
|
SUB R0, R1, R1
|
|
MOVD R1, ret+56(FP)
|
|
RET
|
|
|
|
// func encodeBetterBlockAsm12B(dst []byte, src []byte, tmp *[81920]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeBetterBlockAsm12B(SB), $24-64
|
|
MOVD tmp+48(FP), R0
|
|
MOVD dst_base+0(FP), R1
|
|
MOVD $0x00000280, R2
|
|
MOVD R0, R3
|
|
VEOR V0.B16, V0.B16, V0.B16
|
|
|
|
zero_loop_encodeBetterBlockAsm12B:
|
|
FMOVQ F0, (R3)
|
|
FMOVQ F0, 16(R3)
|
|
FMOVQ F0, 32(R3)
|
|
FMOVQ F0, 48(R3)
|
|
FMOVQ F0, 64(R3)
|
|
FMOVQ F0, 80(R3)
|
|
FMOVQ F0, 96(R3)
|
|
FMOVQ F0, 112(R3)
|
|
ADD $0x80, R3, R3
|
|
SUBS $1, R2, R2
|
|
BNE zero_loop_encodeBetterBlockAsm12B
|
|
MOVD $0x00000000, R16
|
|
MOVW R16, 20(RSP)
|
|
MOVD src_len+32(FP), R2
|
|
SUB $6, R2, R3
|
|
SUB $8, R2, R5
|
|
MOVW R5, 16(RSP)
|
|
LSR $0x05, R2, R2
|
|
SUBW R2, R3, R3
|
|
ADD R3, R1, R3
|
|
MOVD R3, 8(RSP)
|
|
MOVD $0x00000001, R2
|
|
MOVD $0x00000000, R16
|
|
MOVW R16, 24(RSP)
|
|
MOVD src_base+24(FP), R3
|
|
|
|
search_loop_encodeBetterBlockAsm12B:
|
|
MOVWU R2, R5
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R5, R5
|
|
LSRW $0x06, R5, R5
|
|
ADD R5, R2, R5
|
|
ADD $1, R5, R5
|
|
MOVWU R5, R5
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R5
|
|
BHS emit_remainder_encodeBetterBlockAsm12B
|
|
MOVD (R3)(R2), R6
|
|
MOVW R5, 28(RSP)
|
|
MOVD $0x0000cf1bbcdcbf9b, R8
|
|
MOVD $0x9e3779b1, R5
|
|
MOVD R6, R9
|
|
MOVD R6, R10
|
|
LSL $0x10, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x32, R9, R9
|
|
LSL $0x20, R10, R10
|
|
MUL R5, R10, R10
|
|
LSR $0x34, R10, R10
|
|
MOVWU (R0)(R9<<2), R5
|
|
ADD R10<<2, R0, R15
|
|
MOVWU 65536(R15), R7
|
|
MOVW R2, (R0)(R9<<2)
|
|
ADD R10<<2, R0, R15
|
|
MOVW R2, 65536(R15)
|
|
MOVD (R3)(R5), R9
|
|
MOVD (R3)(R7), R10
|
|
CMP R6, R9
|
|
BEQ candidate_match_encodeBetterBlockAsm12B
|
|
CMP R6, R10
|
|
BNE no_short_found_encodeBetterBlockAsm12B
|
|
MOVWU R7, R5
|
|
JMP candidate_match_encodeBetterBlockAsm12B
|
|
|
|
no_short_found_encodeBetterBlockAsm12B:
|
|
CMPW R6, R9
|
|
BEQ candidate_match_encodeBetterBlockAsm12B
|
|
CMPW R6, R10
|
|
BEQ candidateS_match_encodeBetterBlockAsm12B
|
|
MOVWU 28(RSP), R2
|
|
JMP search_loop_encodeBetterBlockAsm12B
|
|
|
|
candidateS_match_encodeBetterBlockAsm12B:
|
|
LSR $0x08, R6, R6
|
|
MOVD R6, R9
|
|
LSL $0x10, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x32, R9, R9
|
|
MOVWU (R0)(R9<<2), R5
|
|
ADDW $1, R2, R2
|
|
MOVW R2, (R0)(R9<<2)
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ candidate_match_encodeBetterBlockAsm12B
|
|
SUBW $1, R2, R2
|
|
MOVWU R7, R5
|
|
|
|
candidate_match_encodeBetterBlockAsm12B:
|
|
MOVWU 20(RSP), R6
|
|
TSTW R5, R5
|
|
BEQ match_extend_back_end_encodeBetterBlockAsm12B
|
|
|
|
match_extend_back_loop_encodeBetterBlockAsm12B:
|
|
CMPW R6, R2
|
|
BLS match_extend_back_end_encodeBetterBlockAsm12B
|
|
ADD R5, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R7
|
|
ADD R2, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R8
|
|
AND $0xff, R8, R16
|
|
AND $0xff, R7, R15
|
|
CMP R16, R15
|
|
BNE match_extend_back_end_encodeBetterBlockAsm12B
|
|
SUB $1, R2, R2
|
|
MOVWU R2, R2
|
|
SUBSW $1, R5, R5
|
|
BEQ match_extend_back_end_encodeBetterBlockAsm12B
|
|
JMP match_extend_back_loop_encodeBetterBlockAsm12B
|
|
|
|
match_extend_back_end_encodeBetterBlockAsm12B:
|
|
MOVWU R2, R6
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R6, R6
|
|
ADD R6, R1, R6
|
|
ADD $3, R6, R6
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R6
|
|
BLO match_dst_size_check_encodeBetterBlockAsm12B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeBetterBlockAsm12B:
|
|
MOVWU R2, R6
|
|
ADDW $0x04, R2, R2
|
|
ADDW $0x04, R5, R5
|
|
MOVD src_len+32(FP), R7
|
|
SUBW R2, R7, R7
|
|
ADD R2, R3, R8
|
|
ADD R5, R3, R9
|
|
|
|
// matchLen
|
|
MOVD $0, R11
|
|
|
|
matchlen_loopback_16_match_nolit_encodeBetterBlockAsm12B:
|
|
CMPW $0x10, R7
|
|
BLO matchlen_match8_match_nolit_encodeBetterBlockAsm12B
|
|
MOVD (R8)(R11), R10
|
|
ADD R11, R8, R15
|
|
MOVD 8(R15), R12
|
|
MOVD (R9)(R11), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_8_match_nolit_encodeBetterBlockAsm12B
|
|
ADD R11, R9, R15
|
|
MOVD 8(R15), R16
|
|
EOR R16, R12, R12
|
|
TST R12, R12
|
|
BNE matchlen_bsf_16match_nolit_encodeBetterBlockAsm12B
|
|
SUB $16, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $16, R11, R11
|
|
MOVWU R11, R11
|
|
JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm12B
|
|
|
|
matchlen_bsf_16match_nolit_encodeBetterBlockAsm12B:
|
|
RBIT R12, R12
|
|
CLZ R12, R12
|
|
ASR $0x03, R12, R12
|
|
ADD R12, R11, R11
|
|
ADD $8, R11, R11
|
|
MOVWU R11, R11
|
|
JMP match_nolit_end_encodeBetterBlockAsm12B
|
|
|
|
matchlen_match8_match_nolit_encodeBetterBlockAsm12B:
|
|
CMPW $0x08, R7
|
|
BLO matchlen_match4_match_nolit_encodeBetterBlockAsm12B
|
|
MOVD (R8)(R11), R10
|
|
MOVD (R9)(R11), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_8_match_nolit_encodeBetterBlockAsm12B
|
|
SUB $8, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $8, R11, R11
|
|
MOVWU R11, R11
|
|
JMP matchlen_match4_match_nolit_encodeBetterBlockAsm12B
|
|
|
|
matchlen_bsf_8_match_nolit_encodeBetterBlockAsm12B:
|
|
RBIT R10, R10
|
|
CLZ R10, R10
|
|
ASR $0x03, R10, R10
|
|
ADD R10, R11, R11
|
|
MOVWU R11, R11
|
|
JMP match_nolit_end_encodeBetterBlockAsm12B
|
|
|
|
matchlen_match4_match_nolit_encodeBetterBlockAsm12B:
|
|
CMPW $0x04, R7
|
|
BLO matchlen_match2_match_nolit_encodeBetterBlockAsm12B
|
|
MOVWU (R8)(R11), R10
|
|
MOVWU (R9)(R11), R16
|
|
CMPW R10, R16
|
|
BNE matchlen_match2_match_nolit_encodeBetterBlockAsm12B
|
|
SUB $4, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $4, R11, R11
|
|
MOVWU R11, R11
|
|
|
|
matchlen_match2_match_nolit_encodeBetterBlockAsm12B:
|
|
CMPW $0x01, R7
|
|
BEQ matchlen_match1_match_nolit_encodeBetterBlockAsm12B
|
|
BLO match_nolit_end_encodeBetterBlockAsm12B
|
|
MOVHU (R8)(R11), R16
|
|
BFI $0, R16, $16, R10
|
|
ADD R11, R9, R15
|
|
MOVHU (R15), R15
|
|
AND $0xffff, R10, R16
|
|
CMP R16, R15
|
|
BNE matchlen_match1_match_nolit_encodeBetterBlockAsm12B
|
|
ADD $2, R11, R11
|
|
MOVWU R11, R11
|
|
SUBSW $0x02, R7, R7
|
|
BEQ match_nolit_end_encodeBetterBlockAsm12B
|
|
|
|
matchlen_match1_match_nolit_encodeBetterBlockAsm12B:
|
|
MOVBU (R8)(R11), R16
|
|
BFI $0, R16, $8, R10
|
|
ADD R11, R9, R15
|
|
MOVBU (R15), R15
|
|
AND $0xff, R10, R16
|
|
CMP R16, R15
|
|
BNE match_nolit_end_encodeBetterBlockAsm12B
|
|
ADD $1, R11, R11
|
|
MOVWU R11, R11
|
|
|
|
match_nolit_end_encodeBetterBlockAsm12B:
|
|
MOVWU R2, R7
|
|
SUBW R5, R7, R7
|
|
|
|
// Check if repeat
|
|
MOVWU 24(RSP), R16
|
|
CMPW R7, R16
|
|
BEQ match_is_repeat_encodeBetterBlockAsm12B
|
|
MOVW R7, 24(RSP)
|
|
MOVWU 20(RSP), R5
|
|
CMPW R6, R5
|
|
BEQ emit_literal_done_match_emit_encodeBetterBlockAsm12B
|
|
MOVWU R6, R8
|
|
MOVW R6, 20(RSP)
|
|
ADD R5, R3, R9
|
|
SUBW R5, R8, R8
|
|
SUB $1, R8, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x3c, R5
|
|
BLO one_byte_match_emit_encodeBetterBlockAsm12B
|
|
CMPW $0x00000100, R5
|
|
BLO two_bytes_match_emit_encodeBetterBlockAsm12B
|
|
BLO three_bytes_match_emit_encodeBetterBlockAsm12B
|
|
|
|
three_bytes_match_emit_encodeBetterBlockAsm12B:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_match_emit_encodeBetterBlockAsm12B
|
|
|
|
two_bytes_match_emit_encodeBetterBlockAsm12B:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R5, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R5
|
|
BLO memmove_match_emit_encodeBetterBlockAsm12B
|
|
JMP memmove_long_match_emit_encodeBetterBlockAsm12B
|
|
|
|
one_byte_match_emit_encodeBetterBlockAsm12B:
|
|
LSLW $0x02, R5, R16
|
|
BFI $0, R16, $8, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_match_emit_encodeBetterBlockAsm12B:
|
|
ADD R8, R1, R5
|
|
|
|
// genMemMoveShort
|
|
CMP $0x04, R8
|
|
BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_4
|
|
CMP $0x08, R8
|
|
BLO emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_4through7
|
|
CMP $0x10, R8
|
|
BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_8through16
|
|
CMP $0x20, R8
|
|
BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_4:
|
|
MOVWU (R9), R10
|
|
MOVW R10, (R1)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_4through7:
|
|
MOVWU (R9), R10
|
|
ADD R8, R9, R15
|
|
MOVWU -4(R15), R9
|
|
MOVW R10, (R1)
|
|
ADD R8, R1, R15
|
|
MOVW R9, -4(R15)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_8through16:
|
|
MOVD (R9), R10
|
|
ADD R8, R9, R15
|
|
MOVD -8(R15), R9
|
|
MOVD R10, (R1)
|
|
ADD R8, R1, R15
|
|
MOVD R9, -8(R15)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_17through32:
|
|
FMOVQ (R9), F0
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_33through64:
|
|
FMOVQ (R9), F0
|
|
FMOVQ 16(R9), F1
|
|
ADD R8, R9, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_match_emit_encodeBetterBlockAsm12B:
|
|
MOVD R5, R1
|
|
JMP emit_literal_done_match_emit_encodeBetterBlockAsm12B
|
|
|
|
memmove_long_match_emit_encodeBetterBlockAsm12B:
|
|
ADD R8, R1, R5
|
|
|
|
// genMemMoveLong
|
|
MOVD R9, R10
|
|
MOVD R1, R12
|
|
MOVD R8, R13
|
|
|
|
emit_lit_memmove_long_match_emit_encodeBetterBlockAsm12Blarge_big_loop_back:
|
|
FMOVQ (R10), F0
|
|
FMOVQ 16(R10), F1
|
|
FMOVQ F0, (R12)
|
|
FMOVQ F1, 16(R12)
|
|
ADD $0x20, R10, R10
|
|
ADD $0x20, R12, R12
|
|
SUB $0x20, R13, R13
|
|
CMP $0x20, R13
|
|
BHS emit_lit_memmove_long_match_emit_encodeBetterBlockAsm12Blarge_big_loop_back
|
|
ADD R8, R9, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R8, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R5, R1
|
|
|
|
emit_literal_done_match_emit_encodeBetterBlockAsm12B:
|
|
ADDW R11, R2, R2
|
|
ADDW $0x04, R11, R11
|
|
MOVW R2, 20(RSP)
|
|
|
|
// emitCopy
|
|
CMPW $0x40, R11
|
|
BLS two_byte_offset_short_match_nolit_encodeBetterBlockAsm12B
|
|
CMPW $0x00000800, R7
|
|
BHS long_offset_short_match_nolit_encodeBetterBlockAsm12B
|
|
MOVD $0x00000001, R5
|
|
ADD $16, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R7, 1(R1)
|
|
LSRW $0x08, R7, R7
|
|
LSLW $0x05, R7, R7
|
|
ORRW R7, R5, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
SUBW $0x08, R11, R11
|
|
|
|
// emitRepeat
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
JMP cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b
|
|
MOVWU R11, R5
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
CMPW $0x08, R5
|
|
BLS repeat_two_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b
|
|
CMPW $0x0c, R5
|
|
BHS cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b
|
|
CMPW $0x00000800, R7
|
|
BLO repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b:
|
|
CMPW $0x00000104, R11
|
|
BLO repeat_three_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b
|
|
SUB $256, R11, R11
|
|
MOVWU R11, R11
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R11, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B
|
|
|
|
repeat_three_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b:
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R11, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B
|
|
|
|
repeat_two_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b:
|
|
LSLW $0x02, R11, R11
|
|
ORRW $0x01, R11, R11
|
|
MOVH R11, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B
|
|
|
|
repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b:
|
|
MOVD $0, R5
|
|
ADD R11<<2, R5, R11
|
|
ADD $1, R11, R11
|
|
MOVWU R11, R11
|
|
MOVB R7, 1(R1)
|
|
ASRW $0x08, R7, R7
|
|
LSLW $0x05, R7, R7
|
|
ORRW R7, R11, R11
|
|
MOVB R11, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B
|
|
|
|
long_offset_short_match_nolit_encodeBetterBlockAsm12B:
|
|
MOVD $0xee, R16
|
|
MOVB R16, (R1)
|
|
MOVH R7, 1(R1)
|
|
SUB $60, R11, R11
|
|
MOVWU R11, R11
|
|
ADD $0x03, R1, R1
|
|
|
|
// emitRepeat
|
|
MOVWU R11, R5
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
CMPW $0x08, R5
|
|
BLS repeat_two_match_nolit_encodeBetterBlockAsm12B_emit_copy_short
|
|
CMPW $0x0c, R5
|
|
BHS cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short
|
|
CMPW $0x00000800, R7
|
|
BLO repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short:
|
|
CMPW $0x00000104, R11
|
|
BLO repeat_three_match_nolit_encodeBetterBlockAsm12B_emit_copy_short
|
|
SUB $256, R11, R11
|
|
MOVWU R11, R11
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R11, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B
|
|
|
|
repeat_three_match_nolit_encodeBetterBlockAsm12B_emit_copy_short:
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R11, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B
|
|
|
|
repeat_two_match_nolit_encodeBetterBlockAsm12B_emit_copy_short:
|
|
LSLW $0x02, R11, R11
|
|
ORRW $0x01, R11, R11
|
|
MOVH R11, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B
|
|
|
|
repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short:
|
|
MOVD $0, R5
|
|
ADD R11<<2, R5, R11
|
|
ADD $1, R11, R11
|
|
MOVWU R11, R11
|
|
MOVB R7, 1(R1)
|
|
ASRW $0x08, R7, R7
|
|
LSLW $0x05, R7, R7
|
|
ORRW R7, R11, R11
|
|
MOVB R11, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B
|
|
|
|
two_byte_offset_short_match_nolit_encodeBetterBlockAsm12B:
|
|
MOVWU R11, R5
|
|
LSLW $0x02, R5, R5
|
|
CMPW $0x0c, R11
|
|
BHS emit_copy_three_match_nolit_encodeBetterBlockAsm12B
|
|
CMPW $0x00000800, R7
|
|
BHS emit_copy_three_match_nolit_encodeBetterBlockAsm12B
|
|
SUB $15, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R7, 1(R1)
|
|
LSRW $0x08, R7, R7
|
|
LSLW $0x05, R7, R7
|
|
ORRW R7, R5, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B
|
|
|
|
emit_copy_three_match_nolit_encodeBetterBlockAsm12B:
|
|
SUB $2, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R5, (R1)
|
|
MOVH R7, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B
|
|
|
|
match_is_repeat_encodeBetterBlockAsm12B:
|
|
MOVWU 20(RSP), R5
|
|
CMPW R6, R5
|
|
BEQ emit_literal_done_match_emit_repeat_encodeBetterBlockAsm12B
|
|
MOVWU R6, R8
|
|
MOVW R6, 20(RSP)
|
|
ADD R5, R3, R9
|
|
SUBW R5, R8, R8
|
|
SUB $1, R8, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x3c, R5
|
|
BLO one_byte_match_emit_repeat_encodeBetterBlockAsm12B
|
|
CMPW $0x00000100, R5
|
|
BLO two_bytes_match_emit_repeat_encodeBetterBlockAsm12B
|
|
BLO three_bytes_match_emit_repeat_encodeBetterBlockAsm12B
|
|
|
|
three_bytes_match_emit_repeat_encodeBetterBlockAsm12B:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm12B
|
|
|
|
two_bytes_match_emit_repeat_encodeBetterBlockAsm12B:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R5, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R5
|
|
BLO memmove_match_emit_repeat_encodeBetterBlockAsm12B
|
|
JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm12B
|
|
|
|
one_byte_match_emit_repeat_encodeBetterBlockAsm12B:
|
|
LSLW $0x02, R5, R16
|
|
BFI $0, R16, $8, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_match_emit_repeat_encodeBetterBlockAsm12B:
|
|
ADD R8, R1, R5
|
|
|
|
// genMemMoveShort
|
|
CMP $0x04, R8
|
|
BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_4
|
|
CMP $0x08, R8
|
|
BLO emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_4through7
|
|
CMP $0x10, R8
|
|
BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_8through16
|
|
CMP $0x20, R8
|
|
BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_4:
|
|
MOVWU (R9), R10
|
|
MOVW R10, (R1)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_4through7:
|
|
MOVWU (R9), R10
|
|
ADD R8, R9, R15
|
|
MOVWU -4(R15), R9
|
|
MOVW R10, (R1)
|
|
ADD R8, R1, R15
|
|
MOVW R9, -4(R15)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_8through16:
|
|
MOVD (R9), R10
|
|
ADD R8, R9, R15
|
|
MOVD -8(R15), R9
|
|
MOVD R10, (R1)
|
|
ADD R8, R1, R15
|
|
MOVD R9, -8(R15)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_17through32:
|
|
FMOVQ (R9), F0
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_33through64:
|
|
FMOVQ (R9), F0
|
|
FMOVQ 16(R9), F1
|
|
ADD R8, R9, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm12B:
|
|
MOVD R5, R1
|
|
JMP emit_literal_done_match_emit_repeat_encodeBetterBlockAsm12B
|
|
|
|
memmove_long_match_emit_repeat_encodeBetterBlockAsm12B:
|
|
ADD R8, R1, R5
|
|
|
|
// genMemMoveLong
|
|
MOVD R9, R10
|
|
MOVD R1, R12
|
|
MOVD R8, R13
|
|
|
|
emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm12Blarge_big_loop_back:
|
|
FMOVQ (R10), F0
|
|
FMOVQ 16(R10), F1
|
|
FMOVQ F0, (R12)
|
|
FMOVQ F1, 16(R12)
|
|
ADD $0x20, R10, R10
|
|
ADD $0x20, R12, R12
|
|
SUB $0x20, R13, R13
|
|
CMP $0x20, R13
|
|
BHS emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm12Blarge_big_loop_back
|
|
ADD R8, R9, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R8, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R5, R1
|
|
|
|
emit_literal_done_match_emit_repeat_encodeBetterBlockAsm12B:
|
|
ADDW R11, R2, R2
|
|
ADDW $0x04, R11, R11
|
|
MOVW R2, 20(RSP)
|
|
|
|
// emitRepeat
|
|
MOVWU R11, R5
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
CMPW $0x08, R5
|
|
BLS repeat_two_match_nolit_repeat_encodeBetterBlockAsm12B
|
|
CMPW $0x0c, R5
|
|
BHS cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm12B
|
|
CMPW $0x00000800, R7
|
|
BLO repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm12B
|
|
|
|
cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm12B:
|
|
CMPW $0x00000104, R11
|
|
BLO repeat_three_match_nolit_repeat_encodeBetterBlockAsm12B
|
|
SUB $256, R11, R11
|
|
MOVWU R11, R11
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R11, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B
|
|
|
|
repeat_three_match_nolit_repeat_encodeBetterBlockAsm12B:
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R11, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B
|
|
|
|
repeat_two_match_nolit_repeat_encodeBetterBlockAsm12B:
|
|
LSLW $0x02, R11, R11
|
|
ORRW $0x01, R11, R11
|
|
MOVH R11, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B
|
|
|
|
repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm12B:
|
|
MOVD $0, R5
|
|
ADD R11<<2, R5, R11
|
|
ADD $1, R11, R11
|
|
MOVWU R11, R11
|
|
MOVB R7, 1(R1)
|
|
ASRW $0x08, R7, R7
|
|
LSLW $0x05, R7, R7
|
|
ORRW R7, R11, R11
|
|
MOVB R11, (R1)
|
|
ADD $0x02, R1, R1
|
|
|
|
match_nolit_emitcopy_end_encodeBetterBlockAsm12B:
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R2
|
|
BHS emit_remainder_encodeBetterBlockAsm12B
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R1
|
|
BLO match_nolit_dst_ok_encodeBetterBlockAsm12B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeBetterBlockAsm12B:
|
|
MOVD $0x0000cf1bbcdcbf9b, R5
|
|
MOVD $0x9e3779b1, R7
|
|
ADD $1, R6, R6
|
|
SUB $2, R2, R8
|
|
MOVD (R3)(R6), R9
|
|
ADD R6, R3, R15
|
|
MOVD 1(R15), R10
|
|
MOVD (R3)(R8), R11
|
|
ADD R8, R3, R15
|
|
MOVD 1(R15), R12
|
|
LSL $0x10, R9, R9
|
|
MUL R5, R9, R9
|
|
LSR $0x32, R9, R9
|
|
LSL $0x20, R10, R10
|
|
MUL R7, R10, R10
|
|
LSR $0x34, R10, R10
|
|
LSL $0x10, R11, R11
|
|
MUL R5, R11, R11
|
|
LSR $0x32, R11, R11
|
|
LSL $0x20, R12, R12
|
|
MUL R7, R12, R12
|
|
LSR $0x34, R12, R12
|
|
ADD $1, R6, R7
|
|
ADD $1, R8, R13
|
|
MOVW R6, (R0)(R9<<2)
|
|
MOVW R8, (R0)(R11<<2)
|
|
ADD R10<<2, R0, R15
|
|
MOVW R7, 65536(R15)
|
|
ADD R12<<2, R0, R15
|
|
MOVW R13, 65536(R15)
|
|
ADD R6, R8, R7
|
|
ADD $1, R7, R7
|
|
LSR $0x01, R7, R7
|
|
ADD $0x01, R6, R6
|
|
SUB $0x01, R8, R8
|
|
|
|
index_loop_encodeBetterBlockAsm12B:
|
|
CMP R8, R7
|
|
BHS search_loop_encodeBetterBlockAsm12B
|
|
MOVD (R3)(R6), R9
|
|
MOVD (R3)(R7), R10
|
|
LSL $0x10, R9, R9
|
|
MUL R5, R9, R9
|
|
LSR $0x32, R9, R9
|
|
LSL $0x10, R10, R10
|
|
MUL R5, R10, R10
|
|
LSR $0x32, R10, R10
|
|
MOVW R6, (R0)(R9<<2)
|
|
MOVW R7, (R0)(R10<<2)
|
|
ADD $0x02, R6, R6
|
|
ADD $0x02, R7, R7
|
|
JMP index_loop_encodeBetterBlockAsm12B
|
|
|
|
emit_remainder_encodeBetterBlockAsm12B:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R0, R0
|
|
ADD R0, R1, R0
|
|
ADD $3, R0, R0
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R0
|
|
BLO emit_remainder_ok_encodeBetterBlockAsm12B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeBetterBlockAsm12B:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R2
|
|
CMPW R0, R2
|
|
BEQ emit_literal_done_emit_remainder_encodeBetterBlockAsm12B
|
|
MOVWU R0, R5
|
|
MOVW R0, 20(RSP)
|
|
ADD R2, R3, R0
|
|
SUBW R2, R5, R5
|
|
SUB $1, R5, R2
|
|
MOVWU R2, R2
|
|
CMPW $0x3c, R2
|
|
BLO one_byte_emit_remainder_encodeBetterBlockAsm12B
|
|
CMPW $0x00000100, R2
|
|
BLO two_bytes_emit_remainder_encodeBetterBlockAsm12B
|
|
BLO three_bytes_emit_remainder_encodeBetterBlockAsm12B
|
|
|
|
three_bytes_emit_remainder_encodeBetterBlockAsm12B:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R2, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_emit_remainder_encodeBetterBlockAsm12B
|
|
|
|
two_bytes_emit_remainder_encodeBetterBlockAsm12B:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R2, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R2
|
|
BLO memmove_emit_remainder_encodeBetterBlockAsm12B
|
|
JMP memmove_long_emit_remainder_encodeBetterBlockAsm12B
|
|
|
|
one_byte_emit_remainder_encodeBetterBlockAsm12B:
|
|
LSLW $0x02, R2, R16
|
|
BFI $0, R16, $8, R2
|
|
MOVB R2, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_emit_remainder_encodeBetterBlockAsm12B:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveShort
|
|
CMP $0x03, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_1or2
|
|
BEQ emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_3
|
|
CMP $0x08, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_4through7
|
|
CMP $0x10, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_8through16
|
|
CMP $0x20, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_1or2:
|
|
MOVBU (R0), R16
|
|
BFI $0, R16, $8, R5
|
|
ADD R3, R0, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVB R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVB R0, -1(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_3:
|
|
MOVHU (R0), R16
|
|
BFI $0, R16, $16, R5
|
|
MOVBU 2(R0), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVH R5, (R1)
|
|
MOVB R0, 2(R1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_4through7:
|
|
MOVWU (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVWU -4(R15), R0
|
|
MOVW R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVW R0, -4(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_8through16:
|
|
MOVD (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVD -8(R15), R0
|
|
MOVD R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVD R0, -8(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_17through32:
|
|
FMOVQ (R0), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_33through64:
|
|
FMOVQ (R0), F0
|
|
FMOVQ 16(R0), F1
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_emit_remainder_encodeBetterBlockAsm12B:
|
|
MOVD R2, R1
|
|
JMP emit_literal_done_emit_remainder_encodeBetterBlockAsm12B
|
|
|
|
memmove_long_emit_remainder_encodeBetterBlockAsm12B:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveLong
|
|
MOVD R0, R5
|
|
MOVD R1, R6
|
|
MOVD R3, R7
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm12Blarge_big_loop_back:
|
|
FMOVQ (R5), F0
|
|
FMOVQ 16(R5), F1
|
|
FMOVQ F0, (R6)
|
|
FMOVQ F1, 16(R6)
|
|
ADD $0x20, R5, R5
|
|
ADD $0x20, R6, R6
|
|
SUB $0x20, R7, R7
|
|
CMP $0x20, R7
|
|
BHS emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm12Blarge_big_loop_back
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R3, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R2, R1
|
|
|
|
emit_literal_done_emit_remainder_encodeBetterBlockAsm12B:
|
|
MOVD dst_base+0(FP), R0
|
|
SUB R0, R1, R1
|
|
MOVD R1, ret+56(FP)
|
|
RET
|
|
|
|
// func encodeBetterBlockAsm10B(dst []byte, src []byte, tmp *[20480]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeBetterBlockAsm10B(SB), $24-64
|
|
MOVD tmp+48(FP), R0
|
|
MOVD dst_base+0(FP), R1
|
|
MOVD $0x000000a0, R2
|
|
MOVD R0, R3
|
|
VEOR V0.B16, V0.B16, V0.B16
|
|
|
|
zero_loop_encodeBetterBlockAsm10B:
|
|
FMOVQ F0, (R3)
|
|
FMOVQ F0, 16(R3)
|
|
FMOVQ F0, 32(R3)
|
|
FMOVQ F0, 48(R3)
|
|
FMOVQ F0, 64(R3)
|
|
FMOVQ F0, 80(R3)
|
|
FMOVQ F0, 96(R3)
|
|
FMOVQ F0, 112(R3)
|
|
ADD $0x80, R3, R3
|
|
SUBS $1, R2, R2
|
|
BNE zero_loop_encodeBetterBlockAsm10B
|
|
MOVD $0x00000000, R16
|
|
MOVW R16, 20(RSP)
|
|
MOVD src_len+32(FP), R2
|
|
SUB $6, R2, R3
|
|
SUB $8, R2, R5
|
|
MOVW R5, 16(RSP)
|
|
LSR $0x05, R2, R2
|
|
SUBW R2, R3, R3
|
|
ADD R3, R1, R3
|
|
MOVD R3, 8(RSP)
|
|
MOVD $0x00000001, R2
|
|
MOVD $0x00000000, R16
|
|
MOVW R16, 24(RSP)
|
|
MOVD src_base+24(FP), R3
|
|
|
|
search_loop_encodeBetterBlockAsm10B:
|
|
MOVWU R2, R5
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R5, R5
|
|
LSRW $0x05, R5, R5
|
|
ADD R5, R2, R5
|
|
ADD $1, R5, R5
|
|
MOVWU R5, R5
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R5
|
|
BHS emit_remainder_encodeBetterBlockAsm10B
|
|
MOVD (R3)(R2), R6
|
|
MOVW R5, 28(RSP)
|
|
MOVD $0x0000cf1bbcdcbf9b, R8
|
|
MOVD $0x9e3779b1, R5
|
|
MOVD R6, R9
|
|
MOVD R6, R10
|
|
LSL $0x10, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x34, R9, R9
|
|
LSL $0x20, R10, R10
|
|
MUL R5, R10, R10
|
|
LSR $0x36, R10, R10
|
|
MOVWU (R0)(R9<<2), R5
|
|
ADD R10<<2, R0, R15
|
|
MOVWU 16384(R15), R7
|
|
MOVW R2, (R0)(R9<<2)
|
|
ADD R10<<2, R0, R15
|
|
MOVW R2, 16384(R15)
|
|
MOVD (R3)(R5), R9
|
|
MOVD (R3)(R7), R10
|
|
CMP R6, R9
|
|
BEQ candidate_match_encodeBetterBlockAsm10B
|
|
CMP R6, R10
|
|
BNE no_short_found_encodeBetterBlockAsm10B
|
|
MOVWU R7, R5
|
|
JMP candidate_match_encodeBetterBlockAsm10B
|
|
|
|
no_short_found_encodeBetterBlockAsm10B:
|
|
CMPW R6, R9
|
|
BEQ candidate_match_encodeBetterBlockAsm10B
|
|
CMPW R6, R10
|
|
BEQ candidateS_match_encodeBetterBlockAsm10B
|
|
MOVWU 28(RSP), R2
|
|
JMP search_loop_encodeBetterBlockAsm10B
|
|
|
|
candidateS_match_encodeBetterBlockAsm10B:
|
|
LSR $0x08, R6, R6
|
|
MOVD R6, R9
|
|
LSL $0x10, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x34, R9, R9
|
|
MOVWU (R0)(R9<<2), R5
|
|
ADDW $1, R2, R2
|
|
MOVW R2, (R0)(R9<<2)
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ candidate_match_encodeBetterBlockAsm10B
|
|
SUBW $1, R2, R2
|
|
MOVWU R7, R5
|
|
|
|
candidate_match_encodeBetterBlockAsm10B:
|
|
MOVWU 20(RSP), R6
|
|
TSTW R5, R5
|
|
BEQ match_extend_back_end_encodeBetterBlockAsm10B
|
|
|
|
match_extend_back_loop_encodeBetterBlockAsm10B:
|
|
CMPW R6, R2
|
|
BLS match_extend_back_end_encodeBetterBlockAsm10B
|
|
ADD R5, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R7
|
|
ADD R2, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R8
|
|
AND $0xff, R8, R16
|
|
AND $0xff, R7, R15
|
|
CMP R16, R15
|
|
BNE match_extend_back_end_encodeBetterBlockAsm10B
|
|
SUB $1, R2, R2
|
|
MOVWU R2, R2
|
|
SUBSW $1, R5, R5
|
|
BEQ match_extend_back_end_encodeBetterBlockAsm10B
|
|
JMP match_extend_back_loop_encodeBetterBlockAsm10B
|
|
|
|
match_extend_back_end_encodeBetterBlockAsm10B:
|
|
MOVWU R2, R6
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R6, R6
|
|
ADD R6, R1, R6
|
|
ADD $3, R6, R6
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R6
|
|
BLO match_dst_size_check_encodeBetterBlockAsm10B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeBetterBlockAsm10B:
|
|
MOVWU R2, R6
|
|
ADDW $0x04, R2, R2
|
|
ADDW $0x04, R5, R5
|
|
MOVD src_len+32(FP), R7
|
|
SUBW R2, R7, R7
|
|
ADD R2, R3, R8
|
|
ADD R5, R3, R9
|
|
|
|
// matchLen
|
|
MOVD $0, R11
|
|
|
|
matchlen_loopback_16_match_nolit_encodeBetterBlockAsm10B:
|
|
CMPW $0x10, R7
|
|
BLO matchlen_match8_match_nolit_encodeBetterBlockAsm10B
|
|
MOVD (R8)(R11), R10
|
|
ADD R11, R8, R15
|
|
MOVD 8(R15), R12
|
|
MOVD (R9)(R11), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_8_match_nolit_encodeBetterBlockAsm10B
|
|
ADD R11, R9, R15
|
|
MOVD 8(R15), R16
|
|
EOR R16, R12, R12
|
|
TST R12, R12
|
|
BNE matchlen_bsf_16match_nolit_encodeBetterBlockAsm10B
|
|
SUB $16, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $16, R11, R11
|
|
MOVWU R11, R11
|
|
JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm10B
|
|
|
|
matchlen_bsf_16match_nolit_encodeBetterBlockAsm10B:
|
|
RBIT R12, R12
|
|
CLZ R12, R12
|
|
ASR $0x03, R12, R12
|
|
ADD R12, R11, R11
|
|
ADD $8, R11, R11
|
|
MOVWU R11, R11
|
|
JMP match_nolit_end_encodeBetterBlockAsm10B
|
|
|
|
matchlen_match8_match_nolit_encodeBetterBlockAsm10B:
|
|
CMPW $0x08, R7
|
|
BLO matchlen_match4_match_nolit_encodeBetterBlockAsm10B
|
|
MOVD (R8)(R11), R10
|
|
MOVD (R9)(R11), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_8_match_nolit_encodeBetterBlockAsm10B
|
|
SUB $8, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $8, R11, R11
|
|
MOVWU R11, R11
|
|
JMP matchlen_match4_match_nolit_encodeBetterBlockAsm10B
|
|
|
|
matchlen_bsf_8_match_nolit_encodeBetterBlockAsm10B:
|
|
RBIT R10, R10
|
|
CLZ R10, R10
|
|
ASR $0x03, R10, R10
|
|
ADD R10, R11, R11
|
|
MOVWU R11, R11
|
|
JMP match_nolit_end_encodeBetterBlockAsm10B
|
|
|
|
matchlen_match4_match_nolit_encodeBetterBlockAsm10B:
|
|
CMPW $0x04, R7
|
|
BLO matchlen_match2_match_nolit_encodeBetterBlockAsm10B
|
|
MOVWU (R8)(R11), R10
|
|
MOVWU (R9)(R11), R16
|
|
CMPW R10, R16
|
|
BNE matchlen_match2_match_nolit_encodeBetterBlockAsm10B
|
|
SUB $4, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $4, R11, R11
|
|
MOVWU R11, R11
|
|
|
|
matchlen_match2_match_nolit_encodeBetterBlockAsm10B:
|
|
CMPW $0x01, R7
|
|
BEQ matchlen_match1_match_nolit_encodeBetterBlockAsm10B
|
|
BLO match_nolit_end_encodeBetterBlockAsm10B
|
|
MOVHU (R8)(R11), R16
|
|
BFI $0, R16, $16, R10
|
|
ADD R11, R9, R15
|
|
MOVHU (R15), R15
|
|
AND $0xffff, R10, R16
|
|
CMP R16, R15
|
|
BNE matchlen_match1_match_nolit_encodeBetterBlockAsm10B
|
|
ADD $2, R11, R11
|
|
MOVWU R11, R11
|
|
SUBSW $0x02, R7, R7
|
|
BEQ match_nolit_end_encodeBetterBlockAsm10B
|
|
|
|
matchlen_match1_match_nolit_encodeBetterBlockAsm10B:
|
|
MOVBU (R8)(R11), R16
|
|
BFI $0, R16, $8, R10
|
|
ADD R11, R9, R15
|
|
MOVBU (R15), R15
|
|
AND $0xff, R10, R16
|
|
CMP R16, R15
|
|
BNE match_nolit_end_encodeBetterBlockAsm10B
|
|
ADD $1, R11, R11
|
|
MOVWU R11, R11
|
|
|
|
match_nolit_end_encodeBetterBlockAsm10B:
|
|
MOVWU R2, R7
|
|
SUBW R5, R7, R7
|
|
|
|
// Check if repeat
|
|
MOVWU 24(RSP), R16
|
|
CMPW R7, R16
|
|
BEQ match_is_repeat_encodeBetterBlockAsm10B
|
|
MOVW R7, 24(RSP)
|
|
MOVWU 20(RSP), R5
|
|
CMPW R6, R5
|
|
BEQ emit_literal_done_match_emit_encodeBetterBlockAsm10B
|
|
MOVWU R6, R8
|
|
MOVW R6, 20(RSP)
|
|
ADD R5, R3, R9
|
|
SUBW R5, R8, R8
|
|
SUB $1, R8, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x3c, R5
|
|
BLO one_byte_match_emit_encodeBetterBlockAsm10B
|
|
CMPW $0x00000100, R5
|
|
BLO two_bytes_match_emit_encodeBetterBlockAsm10B
|
|
BLO three_bytes_match_emit_encodeBetterBlockAsm10B
|
|
|
|
three_bytes_match_emit_encodeBetterBlockAsm10B:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_match_emit_encodeBetterBlockAsm10B
|
|
|
|
two_bytes_match_emit_encodeBetterBlockAsm10B:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R5, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R5
|
|
BLO memmove_match_emit_encodeBetterBlockAsm10B
|
|
JMP memmove_long_match_emit_encodeBetterBlockAsm10B
|
|
|
|
one_byte_match_emit_encodeBetterBlockAsm10B:
|
|
LSLW $0x02, R5, R16
|
|
BFI $0, R16, $8, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_match_emit_encodeBetterBlockAsm10B:
|
|
ADD R8, R1, R5
|
|
|
|
// genMemMoveShort
|
|
CMP $0x04, R8
|
|
BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_4
|
|
CMP $0x08, R8
|
|
BLO emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_4through7
|
|
CMP $0x10, R8
|
|
BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_8through16
|
|
CMP $0x20, R8
|
|
BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_4:
|
|
MOVWU (R9), R10
|
|
MOVW R10, (R1)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_4through7:
|
|
MOVWU (R9), R10
|
|
ADD R8, R9, R15
|
|
MOVWU -4(R15), R9
|
|
MOVW R10, (R1)
|
|
ADD R8, R1, R15
|
|
MOVW R9, -4(R15)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_8through16:
|
|
MOVD (R9), R10
|
|
ADD R8, R9, R15
|
|
MOVD -8(R15), R9
|
|
MOVD R10, (R1)
|
|
ADD R8, R1, R15
|
|
MOVD R9, -8(R15)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_17through32:
|
|
FMOVQ (R9), F0
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_33through64:
|
|
FMOVQ (R9), F0
|
|
FMOVQ 16(R9), F1
|
|
ADD R8, R9, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_match_emit_encodeBetterBlockAsm10B:
|
|
MOVD R5, R1
|
|
JMP emit_literal_done_match_emit_encodeBetterBlockAsm10B
|
|
|
|
memmove_long_match_emit_encodeBetterBlockAsm10B:
|
|
ADD R8, R1, R5
|
|
|
|
// genMemMoveLong
|
|
MOVD R9, R10
|
|
MOVD R1, R12
|
|
MOVD R8, R13
|
|
|
|
emit_lit_memmove_long_match_emit_encodeBetterBlockAsm10Blarge_big_loop_back:
|
|
FMOVQ (R10), F0
|
|
FMOVQ 16(R10), F1
|
|
FMOVQ F0, (R12)
|
|
FMOVQ F1, 16(R12)
|
|
ADD $0x20, R10, R10
|
|
ADD $0x20, R12, R12
|
|
SUB $0x20, R13, R13
|
|
CMP $0x20, R13
|
|
BHS emit_lit_memmove_long_match_emit_encodeBetterBlockAsm10Blarge_big_loop_back
|
|
ADD R8, R9, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R8, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R5, R1
|
|
|
|
emit_literal_done_match_emit_encodeBetterBlockAsm10B:
|
|
ADDW R11, R2, R2
|
|
ADDW $0x04, R11, R11
|
|
MOVW R2, 20(RSP)
|
|
|
|
// emitCopy
|
|
CMPW $0x40, R11
|
|
BLS two_byte_offset_short_match_nolit_encodeBetterBlockAsm10B
|
|
CMPW $0x00000800, R7
|
|
BHS long_offset_short_match_nolit_encodeBetterBlockAsm10B
|
|
MOVD $0x00000001, R5
|
|
ADD $16, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R7, 1(R1)
|
|
LSRW $0x08, R7, R7
|
|
LSLW $0x05, R7, R7
|
|
ORRW R7, R5, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
SUBW $0x08, R11, R11
|
|
|
|
// emitRepeat
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
JMP cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b
|
|
MOVWU R11, R5
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
CMPW $0x08, R5
|
|
BLS repeat_two_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b
|
|
CMPW $0x0c, R5
|
|
BHS cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b
|
|
CMPW $0x00000800, R7
|
|
BLO repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b:
|
|
CMPW $0x00000104, R11
|
|
BLO repeat_three_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b
|
|
SUB $256, R11, R11
|
|
MOVWU R11, R11
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R11, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B
|
|
|
|
repeat_three_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b:
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R11, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B
|
|
|
|
repeat_two_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b:
|
|
LSLW $0x02, R11, R11
|
|
ORRW $0x01, R11, R11
|
|
MOVH R11, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B
|
|
|
|
repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b:
|
|
MOVD $0, R5
|
|
ADD R11<<2, R5, R11
|
|
ADD $1, R11, R11
|
|
MOVWU R11, R11
|
|
MOVB R7, 1(R1)
|
|
ASRW $0x08, R7, R7
|
|
LSLW $0x05, R7, R7
|
|
ORRW R7, R11, R11
|
|
MOVB R11, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B
|
|
|
|
long_offset_short_match_nolit_encodeBetterBlockAsm10B:
|
|
MOVD $0xee, R16
|
|
MOVB R16, (R1)
|
|
MOVH R7, 1(R1)
|
|
SUB $60, R11, R11
|
|
MOVWU R11, R11
|
|
ADD $0x03, R1, R1
|
|
|
|
// emitRepeat
|
|
MOVWU R11, R5
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
CMPW $0x08, R5
|
|
BLS repeat_two_match_nolit_encodeBetterBlockAsm10B_emit_copy_short
|
|
CMPW $0x0c, R5
|
|
BHS cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short
|
|
CMPW $0x00000800, R7
|
|
BLO repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short:
|
|
CMPW $0x00000104, R11
|
|
BLO repeat_three_match_nolit_encodeBetterBlockAsm10B_emit_copy_short
|
|
SUB $256, R11, R11
|
|
MOVWU R11, R11
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R11, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B
|
|
|
|
repeat_three_match_nolit_encodeBetterBlockAsm10B_emit_copy_short:
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R11, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B
|
|
|
|
repeat_two_match_nolit_encodeBetterBlockAsm10B_emit_copy_short:
|
|
LSLW $0x02, R11, R11
|
|
ORRW $0x01, R11, R11
|
|
MOVH R11, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B
|
|
|
|
repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short:
|
|
MOVD $0, R5
|
|
ADD R11<<2, R5, R11
|
|
ADD $1, R11, R11
|
|
MOVWU R11, R11
|
|
MOVB R7, 1(R1)
|
|
ASRW $0x08, R7, R7
|
|
LSLW $0x05, R7, R7
|
|
ORRW R7, R11, R11
|
|
MOVB R11, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B
|
|
|
|
two_byte_offset_short_match_nolit_encodeBetterBlockAsm10B:
|
|
MOVWU R11, R5
|
|
LSLW $0x02, R5, R5
|
|
CMPW $0x0c, R11
|
|
BHS emit_copy_three_match_nolit_encodeBetterBlockAsm10B
|
|
CMPW $0x00000800, R7
|
|
BHS emit_copy_three_match_nolit_encodeBetterBlockAsm10B
|
|
SUB $15, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R7, 1(R1)
|
|
LSRW $0x08, R7, R7
|
|
LSLW $0x05, R7, R7
|
|
ORRW R7, R5, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B
|
|
|
|
emit_copy_three_match_nolit_encodeBetterBlockAsm10B:
|
|
SUB $2, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R5, (R1)
|
|
MOVH R7, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B
|
|
|
|
match_is_repeat_encodeBetterBlockAsm10B:
|
|
MOVWU 20(RSP), R5
|
|
CMPW R6, R5
|
|
BEQ emit_literal_done_match_emit_repeat_encodeBetterBlockAsm10B
|
|
MOVWU R6, R8
|
|
MOVW R6, 20(RSP)
|
|
ADD R5, R3, R9
|
|
SUBW R5, R8, R8
|
|
SUB $1, R8, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x3c, R5
|
|
BLO one_byte_match_emit_repeat_encodeBetterBlockAsm10B
|
|
CMPW $0x00000100, R5
|
|
BLO two_bytes_match_emit_repeat_encodeBetterBlockAsm10B
|
|
BLO three_bytes_match_emit_repeat_encodeBetterBlockAsm10B
|
|
|
|
three_bytes_match_emit_repeat_encodeBetterBlockAsm10B:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm10B
|
|
|
|
two_bytes_match_emit_repeat_encodeBetterBlockAsm10B:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R5, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R5
|
|
BLO memmove_match_emit_repeat_encodeBetterBlockAsm10B
|
|
JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm10B
|
|
|
|
one_byte_match_emit_repeat_encodeBetterBlockAsm10B:
|
|
LSLW $0x02, R5, R16
|
|
BFI $0, R16, $8, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_match_emit_repeat_encodeBetterBlockAsm10B:
|
|
ADD R8, R1, R5
|
|
|
|
// genMemMoveShort
|
|
CMP $0x04, R8
|
|
BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_4
|
|
CMP $0x08, R8
|
|
BLO emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_4through7
|
|
CMP $0x10, R8
|
|
BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_8through16
|
|
CMP $0x20, R8
|
|
BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_4:
|
|
MOVWU (R9), R10
|
|
MOVW R10, (R1)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_4through7:
|
|
MOVWU (R9), R10
|
|
ADD R8, R9, R15
|
|
MOVWU -4(R15), R9
|
|
MOVW R10, (R1)
|
|
ADD R8, R1, R15
|
|
MOVW R9, -4(R15)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_8through16:
|
|
MOVD (R9), R10
|
|
ADD R8, R9, R15
|
|
MOVD -8(R15), R9
|
|
MOVD R10, (R1)
|
|
ADD R8, R1, R15
|
|
MOVD R9, -8(R15)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_17through32:
|
|
FMOVQ (R9), F0
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_33through64:
|
|
FMOVQ (R9), F0
|
|
FMOVQ 16(R9), F1
|
|
ADD R8, R9, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm10B:
|
|
MOVD R5, R1
|
|
JMP emit_literal_done_match_emit_repeat_encodeBetterBlockAsm10B
|
|
|
|
memmove_long_match_emit_repeat_encodeBetterBlockAsm10B:
|
|
ADD R8, R1, R5
|
|
|
|
// genMemMoveLong
|
|
MOVD R9, R10
|
|
MOVD R1, R12
|
|
MOVD R8, R13
|
|
|
|
emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm10Blarge_big_loop_back:
|
|
FMOVQ (R10), F0
|
|
FMOVQ 16(R10), F1
|
|
FMOVQ F0, (R12)
|
|
FMOVQ F1, 16(R12)
|
|
ADD $0x20, R10, R10
|
|
ADD $0x20, R12, R12
|
|
SUB $0x20, R13, R13
|
|
CMP $0x20, R13
|
|
BHS emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm10Blarge_big_loop_back
|
|
ADD R8, R9, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R8, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R5, R1
|
|
|
|
emit_literal_done_match_emit_repeat_encodeBetterBlockAsm10B:
|
|
ADDW R11, R2, R2
|
|
ADDW $0x04, R11, R11
|
|
MOVW R2, 20(RSP)
|
|
|
|
// emitRepeat
|
|
MOVWU R11, R5
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
CMPW $0x08, R5
|
|
BLS repeat_two_match_nolit_repeat_encodeBetterBlockAsm10B
|
|
CMPW $0x0c, R5
|
|
BHS cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm10B
|
|
CMPW $0x00000800, R7
|
|
BLO repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm10B
|
|
|
|
cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm10B:
|
|
CMPW $0x00000104, R11
|
|
BLO repeat_three_match_nolit_repeat_encodeBetterBlockAsm10B
|
|
SUB $256, R11, R11
|
|
MOVWU R11, R11
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R11, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B
|
|
|
|
repeat_three_match_nolit_repeat_encodeBetterBlockAsm10B:
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R11, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B
|
|
|
|
repeat_two_match_nolit_repeat_encodeBetterBlockAsm10B:
|
|
LSLW $0x02, R11, R11
|
|
ORRW $0x01, R11, R11
|
|
MOVH R11, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B
|
|
|
|
repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm10B:
|
|
MOVD $0, R5
|
|
ADD R11<<2, R5, R11
|
|
ADD $1, R11, R11
|
|
MOVWU R11, R11
|
|
MOVB R7, 1(R1)
|
|
ASRW $0x08, R7, R7
|
|
LSLW $0x05, R7, R7
|
|
ORRW R7, R11, R11
|
|
MOVB R11, (R1)
|
|
ADD $0x02, R1, R1
|
|
|
|
match_nolit_emitcopy_end_encodeBetterBlockAsm10B:
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R2
|
|
BHS emit_remainder_encodeBetterBlockAsm10B
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R1
|
|
BLO match_nolit_dst_ok_encodeBetterBlockAsm10B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeBetterBlockAsm10B:
|
|
MOVD $0x0000cf1bbcdcbf9b, R5
|
|
MOVD $0x9e3779b1, R7
|
|
ADD $1, R6, R6
|
|
SUB $2, R2, R8
|
|
MOVD (R3)(R6), R9
|
|
ADD R6, R3, R15
|
|
MOVD 1(R15), R10
|
|
MOVD (R3)(R8), R11
|
|
ADD R8, R3, R15
|
|
MOVD 1(R15), R12
|
|
LSL $0x10, R9, R9
|
|
MUL R5, R9, R9
|
|
LSR $0x34, R9, R9
|
|
LSL $0x20, R10, R10
|
|
MUL R7, R10, R10
|
|
LSR $0x36, R10, R10
|
|
LSL $0x10, R11, R11
|
|
MUL R5, R11, R11
|
|
LSR $0x34, R11, R11
|
|
LSL $0x20, R12, R12
|
|
MUL R7, R12, R12
|
|
LSR $0x36, R12, R12
|
|
ADD $1, R6, R7
|
|
ADD $1, R8, R13
|
|
MOVW R6, (R0)(R9<<2)
|
|
MOVW R8, (R0)(R11<<2)
|
|
ADD R10<<2, R0, R15
|
|
MOVW R7, 16384(R15)
|
|
ADD R12<<2, R0, R15
|
|
MOVW R13, 16384(R15)
|
|
ADD R6, R8, R7
|
|
ADD $1, R7, R7
|
|
LSR $0x01, R7, R7
|
|
ADD $0x01, R6, R6
|
|
SUB $0x01, R8, R8
|
|
|
|
index_loop_encodeBetterBlockAsm10B:
|
|
CMP R8, R7
|
|
BHS search_loop_encodeBetterBlockAsm10B
|
|
MOVD (R3)(R6), R9
|
|
MOVD (R3)(R7), R10
|
|
LSL $0x10, R9, R9
|
|
MUL R5, R9, R9
|
|
LSR $0x34, R9, R9
|
|
LSL $0x10, R10, R10
|
|
MUL R5, R10, R10
|
|
LSR $0x34, R10, R10
|
|
MOVW R6, (R0)(R9<<2)
|
|
MOVW R7, (R0)(R10<<2)
|
|
ADD $0x02, R6, R6
|
|
ADD $0x02, R7, R7
|
|
JMP index_loop_encodeBetterBlockAsm10B
|
|
|
|
emit_remainder_encodeBetterBlockAsm10B:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R0, R0
|
|
ADD R0, R1, R0
|
|
ADD $3, R0, R0
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R0
|
|
BLO emit_remainder_ok_encodeBetterBlockAsm10B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeBetterBlockAsm10B:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R2
|
|
CMPW R0, R2
|
|
BEQ emit_literal_done_emit_remainder_encodeBetterBlockAsm10B
|
|
MOVWU R0, R5
|
|
MOVW R0, 20(RSP)
|
|
ADD R2, R3, R0
|
|
SUBW R2, R5, R5
|
|
SUB $1, R5, R2
|
|
MOVWU R2, R2
|
|
CMPW $0x3c, R2
|
|
BLO one_byte_emit_remainder_encodeBetterBlockAsm10B
|
|
CMPW $0x00000100, R2
|
|
BLO two_bytes_emit_remainder_encodeBetterBlockAsm10B
|
|
BLO three_bytes_emit_remainder_encodeBetterBlockAsm10B
|
|
|
|
three_bytes_emit_remainder_encodeBetterBlockAsm10B:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R2, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_emit_remainder_encodeBetterBlockAsm10B
|
|
|
|
two_bytes_emit_remainder_encodeBetterBlockAsm10B:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R2, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R2
|
|
BLO memmove_emit_remainder_encodeBetterBlockAsm10B
|
|
JMP memmove_long_emit_remainder_encodeBetterBlockAsm10B
|
|
|
|
one_byte_emit_remainder_encodeBetterBlockAsm10B:
|
|
LSLW $0x02, R2, R16
|
|
BFI $0, R16, $8, R2
|
|
MOVB R2, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_emit_remainder_encodeBetterBlockAsm10B:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveShort
|
|
CMP $0x03, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_1or2
|
|
BEQ emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_3
|
|
CMP $0x08, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_4through7
|
|
CMP $0x10, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_8through16
|
|
CMP $0x20, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_1or2:
|
|
MOVBU (R0), R16
|
|
BFI $0, R16, $8, R5
|
|
ADD R3, R0, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVB R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVB R0, -1(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_3:
|
|
MOVHU (R0), R16
|
|
BFI $0, R16, $16, R5
|
|
MOVBU 2(R0), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVH R5, (R1)
|
|
MOVB R0, 2(R1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_4through7:
|
|
MOVWU (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVWU -4(R15), R0
|
|
MOVW R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVW R0, -4(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_8through16:
|
|
MOVD (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVD -8(R15), R0
|
|
MOVD R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVD R0, -8(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_17through32:
|
|
FMOVQ (R0), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_33through64:
|
|
FMOVQ (R0), F0
|
|
FMOVQ 16(R0), F1
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_emit_remainder_encodeBetterBlockAsm10B:
|
|
MOVD R2, R1
|
|
JMP emit_literal_done_emit_remainder_encodeBetterBlockAsm10B
|
|
|
|
memmove_long_emit_remainder_encodeBetterBlockAsm10B:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveLong
|
|
MOVD R0, R5
|
|
MOVD R1, R6
|
|
MOVD R3, R7
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm10Blarge_big_loop_back:
|
|
FMOVQ (R5), F0
|
|
FMOVQ 16(R5), F1
|
|
FMOVQ F0, (R6)
|
|
FMOVQ F1, 16(R6)
|
|
ADD $0x20, R5, R5
|
|
ADD $0x20, R6, R6
|
|
SUB $0x20, R7, R7
|
|
CMP $0x20, R7
|
|
BHS emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm10Blarge_big_loop_back
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R3, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R2, R1
|
|
|
|
emit_literal_done_emit_remainder_encodeBetterBlockAsm10B:
|
|
MOVD dst_base+0(FP), R0
|
|
SUB R0, R1, R1
|
|
MOVD R1, ret+56(FP)
|
|
RET
|
|
|
|
// func encodeBetterBlockAsm8B(dst []byte, src []byte, tmp *[5120]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeBetterBlockAsm8B(SB), $24-64
|
|
MOVD tmp+48(FP), R0
|
|
MOVD dst_base+0(FP), R1
|
|
MOVD $0x00000028, R2
|
|
MOVD R0, R3
|
|
VEOR V0.B16, V0.B16, V0.B16
|
|
|
|
zero_loop_encodeBetterBlockAsm8B:
|
|
FMOVQ F0, (R3)
|
|
FMOVQ F0, 16(R3)
|
|
FMOVQ F0, 32(R3)
|
|
FMOVQ F0, 48(R3)
|
|
FMOVQ F0, 64(R3)
|
|
FMOVQ F0, 80(R3)
|
|
FMOVQ F0, 96(R3)
|
|
FMOVQ F0, 112(R3)
|
|
ADD $0x80, R3, R3
|
|
SUBS $1, R2, R2
|
|
BNE zero_loop_encodeBetterBlockAsm8B
|
|
MOVD $0x00000000, R16
|
|
MOVW R16, 20(RSP)
|
|
MOVD src_len+32(FP), R2
|
|
SUB $6, R2, R3
|
|
SUB $8, R2, R5
|
|
MOVW R5, 16(RSP)
|
|
LSR $0x05, R2, R2
|
|
SUBW R2, R3, R3
|
|
ADD R3, R1, R3
|
|
MOVD R3, 8(RSP)
|
|
MOVD $0x00000001, R2
|
|
MOVD $0x00000000, R16
|
|
MOVW R16, 24(RSP)
|
|
MOVD src_base+24(FP), R3
|
|
|
|
search_loop_encodeBetterBlockAsm8B:
|
|
MOVWU R2, R5
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R5, R5
|
|
LSRW $0x04, R5, R5
|
|
ADD R5, R2, R5
|
|
ADD $1, R5, R5
|
|
MOVWU R5, R5
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R5
|
|
BHS emit_remainder_encodeBetterBlockAsm8B
|
|
MOVD (R3)(R2), R6
|
|
MOVW R5, 28(RSP)
|
|
MOVD $0x0000cf1bbcdcbf9b, R8
|
|
MOVD $0x9e3779b1, R5
|
|
MOVD R6, R9
|
|
MOVD R6, R10
|
|
LSL $0x10, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x36, R9, R9
|
|
LSL $0x20, R10, R10
|
|
MUL R5, R10, R10
|
|
LSR $0x38, R10, R10
|
|
MOVWU (R0)(R9<<2), R5
|
|
ADD R10<<2, R0, R15
|
|
MOVWU 4096(R15), R7
|
|
MOVW R2, (R0)(R9<<2)
|
|
ADD R10<<2, R0, R15
|
|
MOVW R2, 4096(R15)
|
|
MOVD (R3)(R5), R9
|
|
MOVD (R3)(R7), R10
|
|
CMP R6, R9
|
|
BEQ candidate_match_encodeBetterBlockAsm8B
|
|
CMP R6, R10
|
|
BNE no_short_found_encodeBetterBlockAsm8B
|
|
MOVWU R7, R5
|
|
JMP candidate_match_encodeBetterBlockAsm8B
|
|
|
|
no_short_found_encodeBetterBlockAsm8B:
|
|
CMPW R6, R9
|
|
BEQ candidate_match_encodeBetterBlockAsm8B
|
|
CMPW R6, R10
|
|
BEQ candidateS_match_encodeBetterBlockAsm8B
|
|
MOVWU 28(RSP), R2
|
|
JMP search_loop_encodeBetterBlockAsm8B
|
|
|
|
candidateS_match_encodeBetterBlockAsm8B:
|
|
LSR $0x08, R6, R6
|
|
MOVD R6, R9
|
|
LSL $0x10, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x36, R9, R9
|
|
MOVWU (R0)(R9<<2), R5
|
|
ADDW $1, R2, R2
|
|
MOVW R2, (R0)(R9<<2)
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ candidate_match_encodeBetterBlockAsm8B
|
|
SUBW $1, R2, R2
|
|
MOVWU R7, R5
|
|
|
|
candidate_match_encodeBetterBlockAsm8B:
|
|
MOVWU 20(RSP), R6
|
|
TSTW R5, R5
|
|
BEQ match_extend_back_end_encodeBetterBlockAsm8B
|
|
|
|
match_extend_back_loop_encodeBetterBlockAsm8B:
|
|
CMPW R6, R2
|
|
BLS match_extend_back_end_encodeBetterBlockAsm8B
|
|
ADD R5, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R7
|
|
ADD R2, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R8
|
|
AND $0xff, R8, R16
|
|
AND $0xff, R7, R15
|
|
CMP R16, R15
|
|
BNE match_extend_back_end_encodeBetterBlockAsm8B
|
|
SUB $1, R2, R2
|
|
MOVWU R2, R2
|
|
SUBSW $1, R5, R5
|
|
BEQ match_extend_back_end_encodeBetterBlockAsm8B
|
|
JMP match_extend_back_loop_encodeBetterBlockAsm8B
|
|
|
|
match_extend_back_end_encodeBetterBlockAsm8B:
|
|
MOVWU R2, R6
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R6, R6
|
|
ADD R6, R1, R6
|
|
ADD $3, R6, R6
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R6
|
|
BLO match_dst_size_check_encodeBetterBlockAsm8B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeBetterBlockAsm8B:
|
|
MOVWU R2, R6
|
|
ADDW $0x04, R2, R2
|
|
ADDW $0x04, R5, R5
|
|
MOVD src_len+32(FP), R7
|
|
SUBW R2, R7, R7
|
|
ADD R2, R3, R8
|
|
ADD R5, R3, R9
|
|
|
|
// matchLen
|
|
MOVD $0, R11
|
|
|
|
matchlen_loopback_16_match_nolit_encodeBetterBlockAsm8B:
|
|
CMPW $0x10, R7
|
|
BLO matchlen_match8_match_nolit_encodeBetterBlockAsm8B
|
|
MOVD (R8)(R11), R10
|
|
ADD R11, R8, R15
|
|
MOVD 8(R15), R12
|
|
MOVD (R9)(R11), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_8_match_nolit_encodeBetterBlockAsm8B
|
|
ADD R11, R9, R15
|
|
MOVD 8(R15), R16
|
|
EOR R16, R12, R12
|
|
TST R12, R12
|
|
BNE matchlen_bsf_16match_nolit_encodeBetterBlockAsm8B
|
|
SUB $16, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $16, R11, R11
|
|
MOVWU R11, R11
|
|
JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm8B
|
|
|
|
matchlen_bsf_16match_nolit_encodeBetterBlockAsm8B:
|
|
RBIT R12, R12
|
|
CLZ R12, R12
|
|
ASR $0x03, R12, R12
|
|
ADD R12, R11, R11
|
|
ADD $8, R11, R11
|
|
MOVWU R11, R11
|
|
JMP match_nolit_end_encodeBetterBlockAsm8B
|
|
|
|
matchlen_match8_match_nolit_encodeBetterBlockAsm8B:
|
|
CMPW $0x08, R7
|
|
BLO matchlen_match4_match_nolit_encodeBetterBlockAsm8B
|
|
MOVD (R8)(R11), R10
|
|
MOVD (R9)(R11), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_8_match_nolit_encodeBetterBlockAsm8B
|
|
SUB $8, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $8, R11, R11
|
|
MOVWU R11, R11
|
|
JMP matchlen_match4_match_nolit_encodeBetterBlockAsm8B
|
|
|
|
matchlen_bsf_8_match_nolit_encodeBetterBlockAsm8B:
|
|
RBIT R10, R10
|
|
CLZ R10, R10
|
|
ASR $0x03, R10, R10
|
|
ADD R10, R11, R11
|
|
MOVWU R11, R11
|
|
JMP match_nolit_end_encodeBetterBlockAsm8B
|
|
|
|
matchlen_match4_match_nolit_encodeBetterBlockAsm8B:
|
|
CMPW $0x04, R7
|
|
BLO matchlen_match2_match_nolit_encodeBetterBlockAsm8B
|
|
MOVWU (R8)(R11), R10
|
|
MOVWU (R9)(R11), R16
|
|
CMPW R10, R16
|
|
BNE matchlen_match2_match_nolit_encodeBetterBlockAsm8B
|
|
SUB $4, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $4, R11, R11
|
|
MOVWU R11, R11
|
|
|
|
matchlen_match2_match_nolit_encodeBetterBlockAsm8B:
|
|
CMPW $0x01, R7
|
|
BEQ matchlen_match1_match_nolit_encodeBetterBlockAsm8B
|
|
BLO match_nolit_end_encodeBetterBlockAsm8B
|
|
MOVHU (R8)(R11), R16
|
|
BFI $0, R16, $16, R10
|
|
ADD R11, R9, R15
|
|
MOVHU (R15), R15
|
|
AND $0xffff, R10, R16
|
|
CMP R16, R15
|
|
BNE matchlen_match1_match_nolit_encodeBetterBlockAsm8B
|
|
ADD $2, R11, R11
|
|
MOVWU R11, R11
|
|
SUBSW $0x02, R7, R7
|
|
BEQ match_nolit_end_encodeBetterBlockAsm8B
|
|
|
|
matchlen_match1_match_nolit_encodeBetterBlockAsm8B:
|
|
MOVBU (R8)(R11), R16
|
|
BFI $0, R16, $8, R10
|
|
ADD R11, R9, R15
|
|
MOVBU (R15), R15
|
|
AND $0xff, R10, R16
|
|
CMP R16, R15
|
|
BNE match_nolit_end_encodeBetterBlockAsm8B
|
|
ADD $1, R11, R11
|
|
MOVWU R11, R11
|
|
|
|
match_nolit_end_encodeBetterBlockAsm8B:
|
|
MOVWU R2, R7
|
|
SUBW R5, R7, R7
|
|
|
|
// Check if repeat
|
|
MOVWU 24(RSP), R16
|
|
CMPW R7, R16
|
|
BEQ match_is_repeat_encodeBetterBlockAsm8B
|
|
MOVW R7, 24(RSP)
|
|
MOVWU 20(RSP), R5
|
|
CMPW R6, R5
|
|
BEQ emit_literal_done_match_emit_encodeBetterBlockAsm8B
|
|
MOVWU R6, R8
|
|
MOVW R6, 20(RSP)
|
|
ADD R5, R3, R9
|
|
SUBW R5, R8, R8
|
|
SUB $1, R8, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x3c, R5
|
|
BLO one_byte_match_emit_encodeBetterBlockAsm8B
|
|
CMPW $0x00000100, R5
|
|
BLO two_bytes_match_emit_encodeBetterBlockAsm8B
|
|
BLO three_bytes_match_emit_encodeBetterBlockAsm8B
|
|
|
|
three_bytes_match_emit_encodeBetterBlockAsm8B:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_match_emit_encodeBetterBlockAsm8B
|
|
|
|
two_bytes_match_emit_encodeBetterBlockAsm8B:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R5, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R5
|
|
BLO memmove_match_emit_encodeBetterBlockAsm8B
|
|
JMP memmove_long_match_emit_encodeBetterBlockAsm8B
|
|
|
|
one_byte_match_emit_encodeBetterBlockAsm8B:
|
|
LSLW $0x02, R5, R16
|
|
BFI $0, R16, $8, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_match_emit_encodeBetterBlockAsm8B:
|
|
ADD R8, R1, R5
|
|
|
|
// genMemMoveShort
|
|
CMP $0x04, R8
|
|
BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_4
|
|
CMP $0x08, R8
|
|
BLO emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_4through7
|
|
CMP $0x10, R8
|
|
BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_8through16
|
|
CMP $0x20, R8
|
|
BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_4:
|
|
MOVWU (R9), R10
|
|
MOVW R10, (R1)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_4through7:
|
|
MOVWU (R9), R10
|
|
ADD R8, R9, R15
|
|
MOVWU -4(R15), R9
|
|
MOVW R10, (R1)
|
|
ADD R8, R1, R15
|
|
MOVW R9, -4(R15)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_8through16:
|
|
MOVD (R9), R10
|
|
ADD R8, R9, R15
|
|
MOVD -8(R15), R9
|
|
MOVD R10, (R1)
|
|
ADD R8, R1, R15
|
|
MOVD R9, -8(R15)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_17through32:
|
|
FMOVQ (R9), F0
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_match_emit_encodeBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_33through64:
|
|
FMOVQ (R9), F0
|
|
FMOVQ 16(R9), F1
|
|
ADD R8, R9, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_match_emit_encodeBetterBlockAsm8B:
|
|
MOVD R5, R1
|
|
JMP emit_literal_done_match_emit_encodeBetterBlockAsm8B
|
|
|
|
memmove_long_match_emit_encodeBetterBlockAsm8B:
|
|
ADD R8, R1, R5
|
|
|
|
// genMemMoveLong
|
|
MOVD R9, R10
|
|
MOVD R1, R12
|
|
MOVD R8, R13
|
|
|
|
emit_lit_memmove_long_match_emit_encodeBetterBlockAsm8Blarge_big_loop_back:
|
|
FMOVQ (R10), F0
|
|
FMOVQ 16(R10), F1
|
|
FMOVQ F0, (R12)
|
|
FMOVQ F1, 16(R12)
|
|
ADD $0x20, R10, R10
|
|
ADD $0x20, R12, R12
|
|
SUB $0x20, R13, R13
|
|
CMP $0x20, R13
|
|
BHS emit_lit_memmove_long_match_emit_encodeBetterBlockAsm8Blarge_big_loop_back
|
|
ADD R8, R9, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R8, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R5, R1
|
|
|
|
emit_literal_done_match_emit_encodeBetterBlockAsm8B:
|
|
ADDW R11, R2, R2
|
|
ADDW $0x04, R11, R11
|
|
MOVW R2, 20(RSP)
|
|
|
|
// emitCopy
|
|
CMPW $0x40, R11
|
|
BLS two_byte_offset_short_match_nolit_encodeBetterBlockAsm8B
|
|
CMPW $0x00000800, R7
|
|
BHS long_offset_short_match_nolit_encodeBetterBlockAsm8B
|
|
MOVD $0x00000001, R5
|
|
ADD $16, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R7, 1(R1)
|
|
LSRW $0x08, R7, R7
|
|
LSLW $0x05, R7, R7
|
|
ORRW R7, R5, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
SUBW $0x08, R11, R11
|
|
|
|
// emitRepeat
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
JMP cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b
|
|
MOVWU R11, R5
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
CMPW $0x08, R5
|
|
BLS repeat_two_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b
|
|
CMPW $0x0c, R5
|
|
BHS cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b:
|
|
CMPW $0x00000104, R11
|
|
BLO repeat_three_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b
|
|
SUB $256, R11, R11
|
|
MOVWU R11, R11
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R11, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B
|
|
|
|
repeat_three_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b:
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R11, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B
|
|
|
|
repeat_two_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b:
|
|
LSLW $0x02, R11, R11
|
|
ORRW $0x01, R11, R11
|
|
MOVH R11, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B
|
|
MOVD $0, R5
|
|
ADD R11<<2, R5, R11
|
|
ADD $1, R11, R11
|
|
MOVWU R11, R11
|
|
MOVB R7, 1(R1)
|
|
ASRW $0x08, R7, R7
|
|
LSLW $0x05, R7, R7
|
|
ORRW R7, R11, R11
|
|
MOVB R11, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B
|
|
|
|
long_offset_short_match_nolit_encodeBetterBlockAsm8B:
|
|
MOVD $0xee, R16
|
|
MOVB R16, (R1)
|
|
MOVH R7, 1(R1)
|
|
SUB $60, R11, R11
|
|
MOVWU R11, R11
|
|
ADD $0x03, R1, R1
|
|
|
|
// emitRepeat
|
|
MOVWU R11, R5
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
CMPW $0x08, R5
|
|
BLS repeat_two_match_nolit_encodeBetterBlockAsm8B_emit_copy_short
|
|
CMPW $0x0c, R5
|
|
BHS cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm8B_emit_copy_short
|
|
|
|
cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm8B_emit_copy_short:
|
|
CMPW $0x00000104, R11
|
|
BLO repeat_three_match_nolit_encodeBetterBlockAsm8B_emit_copy_short
|
|
SUB $256, R11, R11
|
|
MOVWU R11, R11
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R11, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B
|
|
|
|
repeat_three_match_nolit_encodeBetterBlockAsm8B_emit_copy_short:
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R11, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B
|
|
|
|
repeat_two_match_nolit_encodeBetterBlockAsm8B_emit_copy_short:
|
|
LSLW $0x02, R11, R11
|
|
ORRW $0x01, R11, R11
|
|
MOVH R11, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B
|
|
MOVD $0, R5
|
|
ADD R11<<2, R5, R11
|
|
ADD $1, R11, R11
|
|
MOVWU R11, R11
|
|
MOVB R7, 1(R1)
|
|
ASRW $0x08, R7, R7
|
|
LSLW $0x05, R7, R7
|
|
ORRW R7, R11, R11
|
|
MOVB R11, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B
|
|
|
|
two_byte_offset_short_match_nolit_encodeBetterBlockAsm8B:
|
|
MOVWU R11, R5
|
|
LSLW $0x02, R5, R5
|
|
CMPW $0x0c, R11
|
|
BHS emit_copy_three_match_nolit_encodeBetterBlockAsm8B
|
|
SUB $15, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R7, 1(R1)
|
|
LSRW $0x08, R7, R7
|
|
LSLW $0x05, R7, R7
|
|
ORRW R7, R5, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B
|
|
|
|
emit_copy_three_match_nolit_encodeBetterBlockAsm8B:
|
|
SUB $2, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R5, (R1)
|
|
MOVH R7, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B
|
|
|
|
match_is_repeat_encodeBetterBlockAsm8B:
|
|
MOVWU 20(RSP), R5
|
|
CMPW R6, R5
|
|
BEQ emit_literal_done_match_emit_repeat_encodeBetterBlockAsm8B
|
|
MOVWU R6, R7
|
|
MOVW R6, 20(RSP)
|
|
ADD R5, R3, R8
|
|
SUBW R5, R7, R7
|
|
SUB $1, R7, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x3c, R5
|
|
BLO one_byte_match_emit_repeat_encodeBetterBlockAsm8B
|
|
CMPW $0x00000100, R5
|
|
BLO two_bytes_match_emit_repeat_encodeBetterBlockAsm8B
|
|
BLO three_bytes_match_emit_repeat_encodeBetterBlockAsm8B
|
|
|
|
three_bytes_match_emit_repeat_encodeBetterBlockAsm8B:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm8B
|
|
|
|
two_bytes_match_emit_repeat_encodeBetterBlockAsm8B:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R5, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R5
|
|
BLO memmove_match_emit_repeat_encodeBetterBlockAsm8B
|
|
JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm8B
|
|
|
|
one_byte_match_emit_repeat_encodeBetterBlockAsm8B:
|
|
LSLW $0x02, R5, R16
|
|
BFI $0, R16, $8, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_match_emit_repeat_encodeBetterBlockAsm8B:
|
|
ADD R7, R1, R5
|
|
|
|
// genMemMoveShort
|
|
CMP $0x04, R7
|
|
BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_4
|
|
CMP $0x08, R7
|
|
BLO emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_4through7
|
|
CMP $0x10, R7
|
|
BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_8through16
|
|
CMP $0x20, R7
|
|
BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_4:
|
|
MOVWU (R8), R9
|
|
MOVW R9, (R1)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_4through7:
|
|
MOVWU (R8), R9
|
|
ADD R7, R8, R15
|
|
MOVWU -4(R15), R8
|
|
MOVW R9, (R1)
|
|
ADD R7, R1, R15
|
|
MOVW R8, -4(R15)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_8through16:
|
|
MOVD (R8), R9
|
|
ADD R7, R8, R15
|
|
MOVD -8(R15), R8
|
|
MOVD R9, (R1)
|
|
ADD R7, R1, R15
|
|
MOVD R8, -8(R15)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_17through32:
|
|
FMOVQ (R8), F0
|
|
ADD R7, R8, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R7, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_33through64:
|
|
FMOVQ (R8), F0
|
|
FMOVQ 16(R8), F1
|
|
ADD R7, R8, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R7, R8, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R7, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R7, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm8B:
|
|
MOVD R5, R1
|
|
JMP emit_literal_done_match_emit_repeat_encodeBetterBlockAsm8B
|
|
|
|
memmove_long_match_emit_repeat_encodeBetterBlockAsm8B:
|
|
ADD R7, R1, R5
|
|
|
|
// genMemMoveLong
|
|
MOVD R8, R9
|
|
MOVD R1, R10
|
|
MOVD R7, R12
|
|
|
|
emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm8Blarge_big_loop_back:
|
|
FMOVQ (R9), F0
|
|
FMOVQ 16(R9), F1
|
|
FMOVQ F0, (R10)
|
|
FMOVQ F1, 16(R10)
|
|
ADD $0x20, R9, R9
|
|
ADD $0x20, R10, R10
|
|
SUB $0x20, R12, R12
|
|
CMP $0x20, R12
|
|
BHS emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm8Blarge_big_loop_back
|
|
ADD R7, R8, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R7, R8, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R7, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R7, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R5, R1
|
|
|
|
emit_literal_done_match_emit_repeat_encodeBetterBlockAsm8B:
|
|
ADDW R11, R2, R2
|
|
ADDW $0x04, R11, R11
|
|
MOVW R2, 20(RSP)
|
|
|
|
// emitRepeat
|
|
MOVWU R11, R5
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
CMPW $0x08, R5
|
|
BLS repeat_two_match_nolit_repeat_encodeBetterBlockAsm8B
|
|
CMPW $0x0c, R5
|
|
BHS cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm8B
|
|
|
|
cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm8B:
|
|
CMPW $0x00000104, R11
|
|
BLO repeat_three_match_nolit_repeat_encodeBetterBlockAsm8B
|
|
SUB $256, R11, R11
|
|
MOVWU R11, R11
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R1)
|
|
MOVH R11, 2(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B
|
|
|
|
repeat_three_match_nolit_repeat_encodeBetterBlockAsm8B:
|
|
SUB $4, R11, R11
|
|
MOVWU R11, R11
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R1)
|
|
MOVB R11, 2(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B
|
|
|
|
repeat_two_match_nolit_repeat_encodeBetterBlockAsm8B:
|
|
LSLW $0x02, R11, R11
|
|
ORRW $0x01, R11, R11
|
|
MOVH R11, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B
|
|
MOVD $0, R5
|
|
ADD R11<<2, R5, R11
|
|
ADD $1, R11, R11
|
|
MOVWU R11, R11
|
|
MOVB R7, 1(R1)
|
|
ASRW $0x08, R7, R7
|
|
LSLW $0x05, R7, R7
|
|
ORRW R7, R11, R11
|
|
MOVB R11, (R1)
|
|
ADD $0x02, R1, R1
|
|
|
|
match_nolit_emitcopy_end_encodeBetterBlockAsm8B:
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R2
|
|
BHS emit_remainder_encodeBetterBlockAsm8B
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R1
|
|
BLO match_nolit_dst_ok_encodeBetterBlockAsm8B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeBetterBlockAsm8B:
|
|
MOVD $0x0000cf1bbcdcbf9b, R5
|
|
MOVD $0x9e3779b1, R7
|
|
ADD $1, R6, R6
|
|
SUB $2, R2, R8
|
|
MOVD (R3)(R6), R9
|
|
ADD R6, R3, R15
|
|
MOVD 1(R15), R10
|
|
MOVD (R3)(R8), R11
|
|
ADD R8, R3, R15
|
|
MOVD 1(R15), R12
|
|
LSL $0x10, R9, R9
|
|
MUL R5, R9, R9
|
|
LSR $0x36, R9, R9
|
|
LSL $0x20, R10, R10
|
|
MUL R7, R10, R10
|
|
LSR $0x38, R10, R10
|
|
LSL $0x10, R11, R11
|
|
MUL R5, R11, R11
|
|
LSR $0x36, R11, R11
|
|
LSL $0x20, R12, R12
|
|
MUL R7, R12, R12
|
|
LSR $0x38, R12, R12
|
|
ADD $1, R6, R7
|
|
ADD $1, R8, R13
|
|
MOVW R6, (R0)(R9<<2)
|
|
MOVW R8, (R0)(R11<<2)
|
|
ADD R10<<2, R0, R15
|
|
MOVW R7, 4096(R15)
|
|
ADD R12<<2, R0, R15
|
|
MOVW R13, 4096(R15)
|
|
ADD R6, R8, R7
|
|
ADD $1, R7, R7
|
|
LSR $0x01, R7, R7
|
|
ADD $0x01, R6, R6
|
|
SUB $0x01, R8, R8
|
|
|
|
index_loop_encodeBetterBlockAsm8B:
|
|
CMP R8, R7
|
|
BHS search_loop_encodeBetterBlockAsm8B
|
|
MOVD (R3)(R6), R9
|
|
MOVD (R3)(R7), R10
|
|
LSL $0x10, R9, R9
|
|
MUL R5, R9, R9
|
|
LSR $0x36, R9, R9
|
|
LSL $0x10, R10, R10
|
|
MUL R5, R10, R10
|
|
LSR $0x36, R10, R10
|
|
MOVW R6, (R0)(R9<<2)
|
|
MOVW R7, (R0)(R10<<2)
|
|
ADD $0x02, R6, R6
|
|
ADD $0x02, R7, R7
|
|
JMP index_loop_encodeBetterBlockAsm8B
|
|
|
|
emit_remainder_encodeBetterBlockAsm8B:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R0, R0
|
|
ADD R0, R1, R0
|
|
ADD $3, R0, R0
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R0
|
|
BLO emit_remainder_ok_encodeBetterBlockAsm8B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeBetterBlockAsm8B:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R2
|
|
CMPW R0, R2
|
|
BEQ emit_literal_done_emit_remainder_encodeBetterBlockAsm8B
|
|
MOVWU R0, R5
|
|
MOVW R0, 20(RSP)
|
|
ADD R2, R3, R0
|
|
SUBW R2, R5, R5
|
|
SUB $1, R5, R2
|
|
MOVWU R2, R2
|
|
CMPW $0x3c, R2
|
|
BLO one_byte_emit_remainder_encodeBetterBlockAsm8B
|
|
CMPW $0x00000100, R2
|
|
BLO two_bytes_emit_remainder_encodeBetterBlockAsm8B
|
|
BLO three_bytes_emit_remainder_encodeBetterBlockAsm8B
|
|
|
|
three_bytes_emit_remainder_encodeBetterBlockAsm8B:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R2, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_emit_remainder_encodeBetterBlockAsm8B
|
|
|
|
two_bytes_emit_remainder_encodeBetterBlockAsm8B:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R2, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R2
|
|
BLO memmove_emit_remainder_encodeBetterBlockAsm8B
|
|
JMP memmove_long_emit_remainder_encodeBetterBlockAsm8B
|
|
|
|
one_byte_emit_remainder_encodeBetterBlockAsm8B:
|
|
LSLW $0x02, R2, R16
|
|
BFI $0, R16, $8, R2
|
|
MOVB R2, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_emit_remainder_encodeBetterBlockAsm8B:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveShort
|
|
CMP $0x03, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_1or2
|
|
BEQ emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_3
|
|
CMP $0x08, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_4through7
|
|
CMP $0x10, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_8through16
|
|
CMP $0x20, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_1or2:
|
|
MOVBU (R0), R16
|
|
BFI $0, R16, $8, R5
|
|
ADD R3, R0, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVB R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVB R0, -1(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_3:
|
|
MOVHU (R0), R16
|
|
BFI $0, R16, $16, R5
|
|
MOVBU 2(R0), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVH R5, (R1)
|
|
MOVB R0, 2(R1)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_4through7:
|
|
MOVWU (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVWU -4(R15), R0
|
|
MOVW R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVW R0, -4(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_8through16:
|
|
MOVD (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVD -8(R15), R0
|
|
MOVD R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVD R0, -8(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_17through32:
|
|
FMOVQ (R0), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_33through64:
|
|
FMOVQ (R0), F0
|
|
FMOVQ 16(R0), F1
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_emit_remainder_encodeBetterBlockAsm8B:
|
|
MOVD R2, R1
|
|
JMP emit_literal_done_emit_remainder_encodeBetterBlockAsm8B
|
|
|
|
memmove_long_emit_remainder_encodeBetterBlockAsm8B:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveLong
|
|
MOVD R0, R5
|
|
MOVD R1, R6
|
|
MOVD R3, R7
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm8Blarge_big_loop_back:
|
|
FMOVQ (R5), F0
|
|
FMOVQ 16(R5), F1
|
|
FMOVQ F0, (R6)
|
|
FMOVQ F1, 16(R6)
|
|
ADD $0x20, R5, R5
|
|
ADD $0x20, R6, R6
|
|
SUB $0x20, R7, R7
|
|
CMP $0x20, R7
|
|
BHS emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm8Blarge_big_loop_back
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R3, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R2, R1
|
|
|
|
emit_literal_done_emit_remainder_encodeBetterBlockAsm8B:
|
|
MOVD dst_base+0(FP), R0
|
|
SUB R0, R1, R1
|
|
MOVD R1, ret+56(FP)
|
|
RET
|
|
|
|
// func encodeSnappyBlockAsm(dst []byte, src []byte, tmp *[65536]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeSnappyBlockAsm(SB), $24-64
|
|
MOVD tmp+48(FP), R0
|
|
MOVD dst_base+0(FP), R1
|
|
MOVD $0x00000200, R2
|
|
MOVD R0, R3
|
|
VEOR V0.B16, V0.B16, V0.B16
|
|
|
|
zero_loop_encodeSnappyBlockAsm:
|
|
FMOVQ F0, (R3)
|
|
FMOVQ F0, 16(R3)
|
|
FMOVQ F0, 32(R3)
|
|
FMOVQ F0, 48(R3)
|
|
FMOVQ F0, 64(R3)
|
|
FMOVQ F0, 80(R3)
|
|
FMOVQ F0, 96(R3)
|
|
FMOVQ F0, 112(R3)
|
|
ADD $0x80, R3, R3
|
|
SUBS $1, R2, R2
|
|
BNE zero_loop_encodeSnappyBlockAsm
|
|
MOVD $0x00000000, R16
|
|
MOVW R16, 20(RSP)
|
|
MOVD src_len+32(FP), R2
|
|
SUB $9, R2, R3
|
|
SUB $8, R2, R5
|
|
MOVW R5, 16(RSP)
|
|
LSR $0x05, R2, R2
|
|
SUBW R2, R3, R3
|
|
ADD R3, R1, R3
|
|
MOVD R3, 8(RSP)
|
|
MOVD $0x00000001, R2
|
|
MOVW R2, 24(RSP)
|
|
MOVD src_base+24(FP), R3
|
|
|
|
search_loop_encodeSnappyBlockAsm:
|
|
MOVWU R2, R5
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R5, R5
|
|
LSRW $0x06, R5, R5
|
|
ADD R5, R2, R5
|
|
ADD $4, R5, R5
|
|
MOVWU R5, R5
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R5
|
|
BHS emit_remainder_encodeSnappyBlockAsm
|
|
MOVD (R3)(R2), R6
|
|
MOVW R5, 28(RSP)
|
|
MOVD $0x0000cf1bbcdcbf9b, R8
|
|
MOVD R6, R9
|
|
MOVD R6, R10
|
|
LSR $0x08, R10, R10
|
|
LSL $0x10, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x32, R9, R9
|
|
LSL $0x10, R10, R10
|
|
MUL R8, R10, R10
|
|
LSR $0x32, R10, R10
|
|
MOVWU (R0)(R9<<2), R5
|
|
MOVWU (R0)(R10<<2), R7
|
|
MOVW R2, (R0)(R9<<2)
|
|
ADD $1, R2, R9
|
|
MOVWU R9, R9
|
|
MOVW R9, (R0)(R10<<2)
|
|
MOVD R6, R9
|
|
LSR $0x10, R9, R9
|
|
LSL $0x10, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x32, R9, R9
|
|
MOVWU R2, R8
|
|
MOVWU 24(RSP), R16
|
|
SUBW R16, R8, R8
|
|
ADD R8, R3, R15
|
|
MOVWU 1(R15), R10
|
|
MOVD R6, R8
|
|
LSR $0x08, R8, R8
|
|
CMPW R10, R8
|
|
BNE no_repeat_found_encodeSnappyBlockAsm
|
|
ADD $1, R2, R6
|
|
MOVWU R6, R6
|
|
MOVWU 20(RSP), R5
|
|
MOVWU R6, R7
|
|
MOVWU 24(RSP), R16
|
|
SUBSW R16, R7, R7
|
|
BEQ repeat_extend_back_end_encodeSnappyBlockAsm
|
|
|
|
repeat_extend_back_loop_encodeSnappyBlockAsm:
|
|
CMPW R5, R6
|
|
BLS repeat_extend_back_end_encodeSnappyBlockAsm
|
|
ADD R7, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R8
|
|
ADD R6, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R9
|
|
AND $0xff, R9, R16
|
|
AND $0xff, R8, R15
|
|
CMP R16, R15
|
|
BNE repeat_extend_back_end_encodeSnappyBlockAsm
|
|
SUB $1, R6, R6
|
|
MOVWU R6, R6
|
|
SUBSW $1, R7, R7
|
|
BNE repeat_extend_back_loop_encodeSnappyBlockAsm
|
|
|
|
repeat_extend_back_end_encodeSnappyBlockAsm:
|
|
MOVWU R6, R5
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R5, R5
|
|
ADD R5, R1, R5
|
|
ADD $5, R5, R5
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R5
|
|
BLO repeat_dst_size_check_encodeSnappyBlockAsm
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
repeat_dst_size_check_encodeSnappyBlockAsm:
|
|
MOVWU 20(RSP), R5
|
|
CMPW R6, R5
|
|
BEQ emit_literal_done_repeat_emit_encodeSnappyBlockAsm
|
|
MOVWU R6, R7
|
|
MOVW R6, 20(RSP)
|
|
ADD R5, R3, R8
|
|
SUBW R5, R7, R7
|
|
SUB $1, R7, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x3c, R5
|
|
BLO one_byte_repeat_emit_encodeSnappyBlockAsm
|
|
CMPW $0x00000100, R5
|
|
BLO two_bytes_repeat_emit_encodeSnappyBlockAsm
|
|
CMPW $0x00010000, R5
|
|
BLO three_bytes_repeat_emit_encodeSnappyBlockAsm
|
|
CMPW $0x01000000, R5
|
|
BLO four_bytes_repeat_emit_encodeSnappyBlockAsm
|
|
MOVD $0xfc, R16
|
|
MOVB R16, (R1)
|
|
MOVW R5, 1(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP memmove_long_repeat_emit_encodeSnappyBlockAsm
|
|
|
|
four_bytes_repeat_emit_encodeSnappyBlockAsm:
|
|
MOVWU R5, R9
|
|
LSRW $0x10, R9, R9
|
|
MOVD $0xf8, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
MOVB R9, 3(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP memmove_long_repeat_emit_encodeSnappyBlockAsm
|
|
|
|
three_bytes_repeat_emit_encodeSnappyBlockAsm:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_repeat_emit_encodeSnappyBlockAsm
|
|
|
|
two_bytes_repeat_emit_encodeSnappyBlockAsm:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R5, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R5
|
|
BLO memmove_repeat_emit_encodeSnappyBlockAsm
|
|
JMP memmove_long_repeat_emit_encodeSnappyBlockAsm
|
|
|
|
one_byte_repeat_emit_encodeSnappyBlockAsm:
|
|
LSLW $0x02, R5, R16
|
|
BFI $0, R16, $8, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_repeat_emit_encodeSnappyBlockAsm:
|
|
ADD R7, R1, R5
|
|
|
|
// genMemMoveShort
|
|
CMP $0x08, R7
|
|
BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_8
|
|
CMP $0x10, R7
|
|
BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_8through16
|
|
CMP $0x20, R7
|
|
BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_17through32
|
|
JMP emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_33through64
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_8:
|
|
MOVD (R8), R9
|
|
MOVD R9, (R1)
|
|
JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_8through16:
|
|
MOVD (R8), R9
|
|
ADD R7, R8, R15
|
|
MOVD -8(R15), R8
|
|
MOVD R9, (R1)
|
|
ADD R7, R1, R15
|
|
MOVD R8, -8(R15)
|
|
JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_17through32:
|
|
FMOVQ (R8), F0
|
|
ADD R7, R8, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R7, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_33through64:
|
|
FMOVQ (R8), F0
|
|
FMOVQ 16(R8), F1
|
|
ADD R7, R8, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R7, R8, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R7, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R7, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_repeat_emit_encodeSnappyBlockAsm:
|
|
MOVD R5, R1
|
|
JMP emit_literal_done_repeat_emit_encodeSnappyBlockAsm
|
|
|
|
memmove_long_repeat_emit_encodeSnappyBlockAsm:
|
|
ADD R7, R1, R5
|
|
|
|
// genMemMoveLong
|
|
MOVD R8, R9
|
|
MOVD R1, R10
|
|
MOVD R7, R11
|
|
|
|
emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsmlarge_big_loop_back:
|
|
FMOVQ (R9), F0
|
|
FMOVQ 16(R9), F1
|
|
FMOVQ F0, (R10)
|
|
FMOVQ F1, 16(R10)
|
|
ADD $0x20, R9, R9
|
|
ADD $0x20, R10, R10
|
|
SUB $0x20, R11, R11
|
|
CMP $0x20, R11
|
|
BHS emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsmlarge_big_loop_back
|
|
ADD R7, R8, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R7, R8, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R7, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R7, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R5, R1
|
|
|
|
emit_literal_done_repeat_emit_encodeSnappyBlockAsm:
|
|
ADDW $0x05, R2, R2
|
|
MOVWU R2, R5
|
|
MOVWU 24(RSP), R16
|
|
SUBW R16, R5, R5
|
|
MOVD src_len+32(FP), R7
|
|
SUBW R2, R7, R7
|
|
ADD R2, R3, R8
|
|
ADD R5, R3, R5
|
|
|
|
// matchLen
|
|
MOVD $0, R10
|
|
|
|
matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm:
|
|
CMPW $0x10, R7
|
|
BLO matchlen_match8_repeat_extend_encodeSnappyBlockAsm
|
|
MOVD (R8)(R10), R9
|
|
ADD R10, R8, R15
|
|
MOVD 8(R15), R11
|
|
MOVD (R5)(R10), R16
|
|
EOR R16, R9, R9
|
|
TST R9, R9
|
|
BNE matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm
|
|
ADD R10, R5, R15
|
|
MOVD 8(R15), R16
|
|
EOR R16, R11, R11
|
|
TST R11, R11
|
|
BNE matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm
|
|
SUB $16, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $16, R10, R10
|
|
MOVWU R10, R10
|
|
JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm
|
|
|
|
matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm:
|
|
RBIT R11, R11
|
|
CLZ R11, R11
|
|
ASR $0x03, R11, R11
|
|
ADD R11, R10, R10
|
|
ADD $8, R10, R10
|
|
MOVWU R10, R10
|
|
JMP repeat_extend_forward_end_encodeSnappyBlockAsm
|
|
|
|
matchlen_match8_repeat_extend_encodeSnappyBlockAsm:
|
|
CMPW $0x08, R7
|
|
BLO matchlen_match4_repeat_extend_encodeSnappyBlockAsm
|
|
MOVD (R8)(R10), R9
|
|
MOVD (R5)(R10), R16
|
|
EOR R16, R9, R9
|
|
TST R9, R9
|
|
BNE matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm
|
|
SUB $8, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $8, R10, R10
|
|
MOVWU R10, R10
|
|
JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm
|
|
|
|
matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm:
|
|
RBIT R9, R9
|
|
CLZ R9, R9
|
|
ASR $0x03, R9, R9
|
|
ADD R9, R10, R10
|
|
MOVWU R10, R10
|
|
JMP repeat_extend_forward_end_encodeSnappyBlockAsm
|
|
|
|
matchlen_match4_repeat_extend_encodeSnappyBlockAsm:
|
|
CMPW $0x04, R7
|
|
BLO matchlen_match2_repeat_extend_encodeSnappyBlockAsm
|
|
MOVWU (R8)(R10), R9
|
|
MOVWU (R5)(R10), R16
|
|
CMPW R9, R16
|
|
BNE matchlen_match2_repeat_extend_encodeSnappyBlockAsm
|
|
SUB $4, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $4, R10, R10
|
|
MOVWU R10, R10
|
|
|
|
matchlen_match2_repeat_extend_encodeSnappyBlockAsm:
|
|
CMPW $0x01, R7
|
|
BEQ matchlen_match1_repeat_extend_encodeSnappyBlockAsm
|
|
BLO repeat_extend_forward_end_encodeSnappyBlockAsm
|
|
MOVHU (R8)(R10), R16
|
|
BFI $0, R16, $16, R9
|
|
ADD R10, R5, R15
|
|
MOVHU (R15), R15
|
|
AND $0xffff, R9, R16
|
|
CMP R16, R15
|
|
BNE matchlen_match1_repeat_extend_encodeSnappyBlockAsm
|
|
ADD $2, R10, R10
|
|
MOVWU R10, R10
|
|
SUBSW $0x02, R7, R7
|
|
BEQ repeat_extend_forward_end_encodeSnappyBlockAsm
|
|
|
|
matchlen_match1_repeat_extend_encodeSnappyBlockAsm:
|
|
MOVBU (R8)(R10), R16
|
|
BFI $0, R16, $8, R9
|
|
ADD R10, R5, R15
|
|
MOVBU (R15), R15
|
|
AND $0xff, R9, R16
|
|
CMP R16, R15
|
|
BNE repeat_extend_forward_end_encodeSnappyBlockAsm
|
|
ADD $1, R10, R10
|
|
MOVWU R10, R10
|
|
|
|
repeat_extend_forward_end_encodeSnappyBlockAsm:
|
|
ADDW R10, R2, R2
|
|
MOVWU R2, R5
|
|
SUBW R6, R5, R5
|
|
MOVWU 24(RSP), R6
|
|
|
|
// emitCopy
|
|
CMPW $0x00010000, R6
|
|
BLO two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm
|
|
|
|
four_bytes_loop_back_repeat_as_copy_encodeSnappyBlockAsm:
|
|
CMPW $0x40, R5
|
|
BLS four_bytes_remain_repeat_as_copy_encodeSnappyBlockAsm
|
|
MOVD $0xff, R16
|
|
MOVB R16, (R1)
|
|
MOVW R6, 1(R1)
|
|
SUB $64, R5, R5
|
|
MOVWU R5, R5
|
|
ADD $0x05, R1, R1
|
|
CMPW $0x04, R5
|
|
BLO four_bytes_remain_repeat_as_copy_encodeSnappyBlockAsm
|
|
JMP four_bytes_loop_back_repeat_as_copy_encodeSnappyBlockAsm
|
|
|
|
four_bytes_remain_repeat_as_copy_encodeSnappyBlockAsm:
|
|
TSTW R5, R5
|
|
BEQ repeat_end_emit_encodeSnappyBlockAsm
|
|
MOVD $0, R7
|
|
ADD R5<<2, R7, R5
|
|
SUB $1, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R5, (R1)
|
|
MOVW R6, 1(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP repeat_end_emit_encodeSnappyBlockAsm
|
|
|
|
two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm:
|
|
CMPW $0x40, R5
|
|
BLS two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm
|
|
MOVD $0xee, R16
|
|
MOVB R16, (R1)
|
|
MOVH R6, 1(R1)
|
|
SUB $60, R5, R5
|
|
MOVWU R5, R5
|
|
ADD $0x03, R1, R1
|
|
JMP two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm
|
|
|
|
two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm:
|
|
MOVWU R5, R7
|
|
LSLW $0x02, R7, R7
|
|
CMPW $0x0c, R5
|
|
BHS emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm
|
|
CMPW $0x00000800, R6
|
|
BHS emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm
|
|
SUB $15, R7, R7
|
|
MOVWU R7, R7
|
|
MOVB R6, 1(R1)
|
|
LSRW $0x08, R6, R6
|
|
LSLW $0x05, R6, R6
|
|
ORRW R6, R7, R7
|
|
MOVB R7, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeSnappyBlockAsm
|
|
|
|
emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm:
|
|
SUB $2, R7, R7
|
|
MOVWU R7, R7
|
|
MOVB R7, (R1)
|
|
MOVH R6, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
|
|
repeat_end_emit_encodeSnappyBlockAsm:
|
|
MOVW R2, 20(RSP)
|
|
JMP search_loop_encodeSnappyBlockAsm
|
|
|
|
no_repeat_found_encodeSnappyBlockAsm:
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ candidate_match_encodeSnappyBlockAsm
|
|
LSR $0x08, R6, R6
|
|
MOVWU (R0)(R9<<2), R5
|
|
ADD $2, R2, R8
|
|
MOVWU R8, R8
|
|
MOVWU (R3)(R7), R16
|
|
CMPW R6, R16
|
|
BEQ candidate2_match_encodeSnappyBlockAsm
|
|
MOVW R8, (R0)(R9<<2)
|
|
LSR $0x08, R6, R6
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ candidate3_match_encodeSnappyBlockAsm
|
|
MOVWU 28(RSP), R2
|
|
JMP search_loop_encodeSnappyBlockAsm
|
|
|
|
candidate3_match_encodeSnappyBlockAsm:
|
|
ADDW $0x02, R2, R2
|
|
JMP candidate_match_encodeSnappyBlockAsm
|
|
|
|
candidate2_match_encodeSnappyBlockAsm:
|
|
MOVW R8, (R0)(R9<<2)
|
|
ADDW $1, R2, R2
|
|
MOVWU R7, R5
|
|
|
|
candidate_match_encodeSnappyBlockAsm:
|
|
MOVWU 20(RSP), R6
|
|
TSTW R5, R5
|
|
BEQ match_extend_back_end_encodeSnappyBlockAsm
|
|
|
|
match_extend_back_loop_encodeSnappyBlockAsm:
|
|
CMPW R6, R2
|
|
BLS match_extend_back_end_encodeSnappyBlockAsm
|
|
ADD R5, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R7
|
|
ADD R2, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R8
|
|
AND $0xff, R8, R16
|
|
AND $0xff, R7, R15
|
|
CMP R16, R15
|
|
BNE match_extend_back_end_encodeSnappyBlockAsm
|
|
SUB $1, R2, R2
|
|
MOVWU R2, R2
|
|
SUBSW $1, R5, R5
|
|
BEQ match_extend_back_end_encodeSnappyBlockAsm
|
|
JMP match_extend_back_loop_encodeSnappyBlockAsm
|
|
|
|
match_extend_back_end_encodeSnappyBlockAsm:
|
|
MOVWU R2, R6
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R6, R6
|
|
ADD R6, R1, R6
|
|
ADD $5, R6, R6
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R6
|
|
BLO match_dst_size_check_encodeSnappyBlockAsm
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeSnappyBlockAsm:
|
|
MOVWU R2, R6
|
|
MOVWU 20(RSP), R7
|
|
CMPW R6, R7
|
|
BEQ emit_literal_done_match_emit_encodeSnappyBlockAsm
|
|
MOVWU R6, R8
|
|
MOVW R6, 20(RSP)
|
|
ADD R7, R3, R6
|
|
SUBW R7, R8, R8
|
|
SUB $1, R8, R7
|
|
MOVWU R7, R7
|
|
CMPW $0x3c, R7
|
|
BLO one_byte_match_emit_encodeSnappyBlockAsm
|
|
CMPW $0x00000100, R7
|
|
BLO two_bytes_match_emit_encodeSnappyBlockAsm
|
|
CMPW $0x00010000, R7
|
|
BLO three_bytes_match_emit_encodeSnappyBlockAsm
|
|
CMPW $0x01000000, R7
|
|
BLO four_bytes_match_emit_encodeSnappyBlockAsm
|
|
MOVD $0xfc, R16
|
|
MOVB R16, (R1)
|
|
MOVW R7, 1(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP memmove_long_match_emit_encodeSnappyBlockAsm
|
|
|
|
four_bytes_match_emit_encodeSnappyBlockAsm:
|
|
MOVWU R7, R9
|
|
LSRW $0x10, R9, R9
|
|
MOVD $0xf8, R16
|
|
MOVB R16, (R1)
|
|
MOVH R7, 1(R1)
|
|
MOVB R9, 3(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP memmove_long_match_emit_encodeSnappyBlockAsm
|
|
|
|
three_bytes_match_emit_encodeSnappyBlockAsm:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R7, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_match_emit_encodeSnappyBlockAsm
|
|
|
|
two_bytes_match_emit_encodeSnappyBlockAsm:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R7, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R7
|
|
BLO memmove_match_emit_encodeSnappyBlockAsm
|
|
JMP memmove_long_match_emit_encodeSnappyBlockAsm
|
|
|
|
one_byte_match_emit_encodeSnappyBlockAsm:
|
|
LSLW $0x02, R7, R16
|
|
BFI $0, R16, $8, R7
|
|
MOVB R7, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_match_emit_encodeSnappyBlockAsm:
|
|
ADD R8, R1, R7
|
|
|
|
// genMemMoveShort
|
|
CMP $0x08, R8
|
|
BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_8
|
|
CMP $0x10, R8
|
|
BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_8through16
|
|
CMP $0x20, R8
|
|
BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_8:
|
|
MOVD (R6), R9
|
|
MOVD R9, (R1)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_8through16:
|
|
MOVD (R6), R9
|
|
ADD R8, R6, R15
|
|
MOVD -8(R15), R6
|
|
MOVD R9, (R1)
|
|
ADD R8, R1, R15
|
|
MOVD R6, -8(R15)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_17through32:
|
|
FMOVQ (R6), F0
|
|
ADD R8, R6, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_33through64:
|
|
FMOVQ (R6), F0
|
|
FMOVQ 16(R6), F1
|
|
ADD R8, R6, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R8, R6, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_match_emit_encodeSnappyBlockAsm:
|
|
MOVD R7, R1
|
|
JMP emit_literal_done_match_emit_encodeSnappyBlockAsm
|
|
|
|
memmove_long_match_emit_encodeSnappyBlockAsm:
|
|
ADD R8, R1, R7
|
|
|
|
// genMemMoveLong
|
|
MOVD R6, R9
|
|
MOVD R1, R10
|
|
MOVD R8, R11
|
|
|
|
emit_lit_memmove_long_match_emit_encodeSnappyBlockAsmlarge_big_loop_back:
|
|
FMOVQ (R9), F0
|
|
FMOVQ 16(R9), F1
|
|
FMOVQ F0, (R10)
|
|
FMOVQ F1, 16(R10)
|
|
ADD $0x20, R9, R9
|
|
ADD $0x20, R10, R10
|
|
SUB $0x20, R11, R11
|
|
CMP $0x20, R11
|
|
BHS emit_lit_memmove_long_match_emit_encodeSnappyBlockAsmlarge_big_loop_back
|
|
ADD R8, R6, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R8, R6, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R8, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R7, R1
|
|
|
|
emit_literal_done_match_emit_encodeSnappyBlockAsm:
|
|
match_nolit_loop_encodeSnappyBlockAsm:
|
|
MOVWU R2, R6
|
|
SUBW R5, R6, R6
|
|
MOVW R6, 24(RSP)
|
|
ADDW $0x04, R2, R2
|
|
ADDW $0x04, R5, R5
|
|
MOVD src_len+32(FP), R6
|
|
SUBW R2, R6, R6
|
|
ADD R2, R3, R7
|
|
ADD R5, R3, R5
|
|
|
|
// matchLen
|
|
MOVD $0, R9
|
|
|
|
matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm:
|
|
CMPW $0x10, R6
|
|
BLO matchlen_match8_match_nolit_encodeSnappyBlockAsm
|
|
MOVD (R7)(R9), R8
|
|
ADD R9, R7, R15
|
|
MOVD 8(R15), R10
|
|
MOVD (R5)(R9), R16
|
|
EOR R16, R8, R8
|
|
TST R8, R8
|
|
BNE matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm
|
|
ADD R9, R5, R15
|
|
MOVD 8(R15), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_16match_nolit_encodeSnappyBlockAsm
|
|
SUB $16, R6, R6
|
|
MOVWU R6, R6
|
|
ADD $16, R9, R9
|
|
MOVWU R9, R9
|
|
JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm
|
|
|
|
matchlen_bsf_16match_nolit_encodeSnappyBlockAsm:
|
|
RBIT R10, R10
|
|
CLZ R10, R10
|
|
ASR $0x03, R10, R10
|
|
ADD R10, R9, R9
|
|
ADD $8, R9, R9
|
|
MOVWU R9, R9
|
|
JMP match_nolit_end_encodeSnappyBlockAsm
|
|
|
|
matchlen_match8_match_nolit_encodeSnappyBlockAsm:
|
|
CMPW $0x08, R6
|
|
BLO matchlen_match4_match_nolit_encodeSnappyBlockAsm
|
|
MOVD (R7)(R9), R8
|
|
MOVD (R5)(R9), R16
|
|
EOR R16, R8, R8
|
|
TST R8, R8
|
|
BNE matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm
|
|
SUB $8, R6, R6
|
|
MOVWU R6, R6
|
|
ADD $8, R9, R9
|
|
MOVWU R9, R9
|
|
JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm
|
|
|
|
matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm:
|
|
RBIT R8, R8
|
|
CLZ R8, R8
|
|
ASR $0x03, R8, R8
|
|
ADD R8, R9, R9
|
|
MOVWU R9, R9
|
|
JMP match_nolit_end_encodeSnappyBlockAsm
|
|
|
|
matchlen_match4_match_nolit_encodeSnappyBlockAsm:
|
|
CMPW $0x04, R6
|
|
BLO matchlen_match2_match_nolit_encodeSnappyBlockAsm
|
|
MOVWU (R7)(R9), R8
|
|
MOVWU (R5)(R9), R16
|
|
CMPW R8, R16
|
|
BNE matchlen_match2_match_nolit_encodeSnappyBlockAsm
|
|
SUB $4, R6, R6
|
|
MOVWU R6, R6
|
|
ADD $4, R9, R9
|
|
MOVWU R9, R9
|
|
|
|
matchlen_match2_match_nolit_encodeSnappyBlockAsm:
|
|
CMPW $0x01, R6
|
|
BEQ matchlen_match1_match_nolit_encodeSnappyBlockAsm
|
|
BLO match_nolit_end_encodeSnappyBlockAsm
|
|
MOVHU (R7)(R9), R16
|
|
BFI $0, R16, $16, R8
|
|
ADD R9, R5, R15
|
|
MOVHU (R15), R15
|
|
AND $0xffff, R8, R16
|
|
CMP R16, R15
|
|
BNE matchlen_match1_match_nolit_encodeSnappyBlockAsm
|
|
ADD $2, R9, R9
|
|
MOVWU R9, R9
|
|
SUBSW $0x02, R6, R6
|
|
BEQ match_nolit_end_encodeSnappyBlockAsm
|
|
|
|
matchlen_match1_match_nolit_encodeSnappyBlockAsm:
|
|
MOVBU (R7)(R9), R16
|
|
BFI $0, R16, $8, R8
|
|
ADD R9, R5, R15
|
|
MOVBU (R15), R15
|
|
AND $0xff, R8, R16
|
|
CMP R16, R15
|
|
BNE match_nolit_end_encodeSnappyBlockAsm
|
|
ADD $1, R9, R9
|
|
MOVWU R9, R9
|
|
|
|
match_nolit_end_encodeSnappyBlockAsm:
|
|
ADDW R9, R2, R2
|
|
MOVWU 24(RSP), R5
|
|
ADDW $0x04, R9, R9
|
|
MOVW R2, 20(RSP)
|
|
|
|
// emitCopy
|
|
CMPW $0x00010000, R5
|
|
BLO two_byte_offset_match_nolit_encodeSnappyBlockAsm
|
|
|
|
four_bytes_loop_back_match_nolit_encodeSnappyBlockAsm:
|
|
CMPW $0x40, R9
|
|
BLS four_bytes_remain_match_nolit_encodeSnappyBlockAsm
|
|
MOVD $0xff, R16
|
|
MOVB R16, (R1)
|
|
MOVW R5, 1(R1)
|
|
SUB $64, R9, R9
|
|
MOVWU R9, R9
|
|
ADD $0x05, R1, R1
|
|
CMPW $0x04, R9
|
|
BLO four_bytes_remain_match_nolit_encodeSnappyBlockAsm
|
|
JMP four_bytes_loop_back_match_nolit_encodeSnappyBlockAsm
|
|
|
|
four_bytes_remain_match_nolit_encodeSnappyBlockAsm:
|
|
TSTW R9, R9
|
|
BEQ match_nolit_emitcopy_end_encodeSnappyBlockAsm
|
|
MOVD $0, R6
|
|
ADD R9<<2, R6, R9
|
|
SUB $1, R9, R9
|
|
MOVWU R9, R9
|
|
MOVB R9, (R1)
|
|
MOVW R5, 1(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeSnappyBlockAsm
|
|
|
|
two_byte_offset_match_nolit_encodeSnappyBlockAsm:
|
|
CMPW $0x40, R9
|
|
BLS two_byte_offset_short_match_nolit_encodeSnappyBlockAsm
|
|
MOVD $0xee, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
SUB $60, R9, R9
|
|
MOVWU R9, R9
|
|
ADD $0x03, R1, R1
|
|
JMP two_byte_offset_match_nolit_encodeSnappyBlockAsm
|
|
|
|
two_byte_offset_short_match_nolit_encodeSnappyBlockAsm:
|
|
MOVWU R9, R6
|
|
LSLW $0x02, R6, R6
|
|
CMPW $0x0c, R9
|
|
BHS emit_copy_three_match_nolit_encodeSnappyBlockAsm
|
|
CMPW $0x00000800, R5
|
|
BHS emit_copy_three_match_nolit_encodeSnappyBlockAsm
|
|
SUB $15, R6, R6
|
|
MOVWU R6, R6
|
|
MOVB R5, 1(R1)
|
|
LSRW $0x08, R5, R5
|
|
LSLW $0x05, R5, R5
|
|
ORRW R5, R6, R6
|
|
MOVB R6, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeSnappyBlockAsm
|
|
|
|
emit_copy_three_match_nolit_encodeSnappyBlockAsm:
|
|
SUB $2, R6, R6
|
|
MOVWU R6, R6
|
|
MOVB R6, (R1)
|
|
MOVH R5, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
|
|
match_nolit_emitcopy_end_encodeSnappyBlockAsm:
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R2
|
|
BHS emit_remainder_encodeSnappyBlockAsm
|
|
ADD R2, R3, R15
|
|
MOVD -2(R15), R6
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R1
|
|
BLO match_nolit_dst_ok_encodeSnappyBlockAsm
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeSnappyBlockAsm:
|
|
MOVD $0x0000cf1bbcdcbf9b, R8
|
|
MOVD R6, R7
|
|
LSR $0x10, R6, R6
|
|
MOVD R6, R5
|
|
LSL $0x10, R7, R7
|
|
MUL R8, R7, R7
|
|
LSR $0x32, R7, R7
|
|
LSL $0x10, R5, R5
|
|
MUL R8, R5, R5
|
|
LSR $0x32, R5, R5
|
|
SUB $2, R2, R8
|
|
MOVWU R8, R8
|
|
ADD R5<<2, R0, R9
|
|
MOVWU (R9), R5
|
|
MOVW R8, (R0)(R7<<2)
|
|
MOVW R2, (R9)
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ match_nolit_loop_encodeSnappyBlockAsm
|
|
ADDW $1, R2, R2
|
|
JMP search_loop_encodeSnappyBlockAsm
|
|
|
|
emit_remainder_encodeSnappyBlockAsm:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R0, R0
|
|
ADD R0, R1, R0
|
|
ADD $5, R0, R0
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R0
|
|
BLO emit_remainder_ok_encodeSnappyBlockAsm
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeSnappyBlockAsm:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R2
|
|
CMPW R0, R2
|
|
BEQ emit_literal_done_emit_remainder_encodeSnappyBlockAsm
|
|
MOVWU R0, R5
|
|
MOVW R0, 20(RSP)
|
|
ADD R2, R3, R0
|
|
SUBW R2, R5, R5
|
|
SUB $1, R5, R2
|
|
MOVWU R2, R2
|
|
CMPW $0x3c, R2
|
|
BLO one_byte_emit_remainder_encodeSnappyBlockAsm
|
|
CMPW $0x00000100, R2
|
|
BLO two_bytes_emit_remainder_encodeSnappyBlockAsm
|
|
CMPW $0x00010000, R2
|
|
BLO three_bytes_emit_remainder_encodeSnappyBlockAsm
|
|
CMPW $0x01000000, R2
|
|
BLO four_bytes_emit_remainder_encodeSnappyBlockAsm
|
|
MOVD $0xfc, R16
|
|
MOVB R16, (R1)
|
|
MOVW R2, 1(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP memmove_long_emit_remainder_encodeSnappyBlockAsm
|
|
|
|
four_bytes_emit_remainder_encodeSnappyBlockAsm:
|
|
MOVWU R2, R3
|
|
LSRW $0x10, R3, R3
|
|
MOVD $0xf8, R16
|
|
MOVB R16, (R1)
|
|
MOVH R2, 1(R1)
|
|
MOVB R3, 3(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP memmove_long_emit_remainder_encodeSnappyBlockAsm
|
|
|
|
three_bytes_emit_remainder_encodeSnappyBlockAsm:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R2, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_emit_remainder_encodeSnappyBlockAsm
|
|
|
|
two_bytes_emit_remainder_encodeSnappyBlockAsm:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R2, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R2
|
|
BLO memmove_emit_remainder_encodeSnappyBlockAsm
|
|
JMP memmove_long_emit_remainder_encodeSnappyBlockAsm
|
|
|
|
one_byte_emit_remainder_encodeSnappyBlockAsm:
|
|
LSLW $0x02, R2, R16
|
|
BFI $0, R16, $8, R2
|
|
MOVB R2, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_emit_remainder_encodeSnappyBlockAsm:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveShort
|
|
CMP $0x03, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_1or2
|
|
BEQ emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_3
|
|
CMP $0x08, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_4through7
|
|
CMP $0x10, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_8through16
|
|
CMP $0x20, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_1or2:
|
|
MOVBU (R0), R16
|
|
BFI $0, R16, $8, R5
|
|
ADD R3, R0, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVB R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVB R0, -1(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_3:
|
|
MOVHU (R0), R16
|
|
BFI $0, R16, $16, R5
|
|
MOVBU 2(R0), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVH R5, (R1)
|
|
MOVB R0, 2(R1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_4through7:
|
|
MOVWU (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVWU -4(R15), R0
|
|
MOVW R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVW R0, -4(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_8through16:
|
|
MOVD (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVD -8(R15), R0
|
|
MOVD R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVD R0, -8(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_17through32:
|
|
FMOVQ (R0), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_33through64:
|
|
FMOVQ (R0), F0
|
|
FMOVQ 16(R0), F1
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_emit_remainder_encodeSnappyBlockAsm:
|
|
MOVD R2, R1
|
|
JMP emit_literal_done_emit_remainder_encodeSnappyBlockAsm
|
|
|
|
memmove_long_emit_remainder_encodeSnappyBlockAsm:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveLong
|
|
MOVD R0, R5
|
|
MOVD R1, R6
|
|
MOVD R3, R7
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsmlarge_big_loop_back:
|
|
FMOVQ (R5), F0
|
|
FMOVQ 16(R5), F1
|
|
FMOVQ F0, (R6)
|
|
FMOVQ F1, 16(R6)
|
|
ADD $0x20, R5, R5
|
|
ADD $0x20, R6, R6
|
|
SUB $0x20, R7, R7
|
|
CMP $0x20, R7
|
|
BHS emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsmlarge_big_loop_back
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R3, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R2, R1
|
|
|
|
emit_literal_done_emit_remainder_encodeSnappyBlockAsm:
|
|
MOVD dst_base+0(FP), R0
|
|
SUB R0, R1, R1
|
|
MOVD R1, ret+56(FP)
|
|
RET
|
|
|
|
// func encodeSnappyBlockAsm64K(dst []byte, src []byte, tmp *[65536]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeSnappyBlockAsm64K(SB), $24-64
|
|
MOVD tmp+48(FP), R0
|
|
MOVD dst_base+0(FP), R1
|
|
MOVD $0x00000200, R2
|
|
MOVD R0, R3
|
|
VEOR V0.B16, V0.B16, V0.B16
|
|
|
|
zero_loop_encodeSnappyBlockAsm64K:
|
|
FMOVQ F0, (R3)
|
|
FMOVQ F0, 16(R3)
|
|
FMOVQ F0, 32(R3)
|
|
FMOVQ F0, 48(R3)
|
|
FMOVQ F0, 64(R3)
|
|
FMOVQ F0, 80(R3)
|
|
FMOVQ F0, 96(R3)
|
|
FMOVQ F0, 112(R3)
|
|
ADD $0x80, R3, R3
|
|
SUBS $1, R2, R2
|
|
BNE zero_loop_encodeSnappyBlockAsm64K
|
|
MOVD $0x00000000, R16
|
|
MOVW R16, 20(RSP)
|
|
MOVD src_len+32(FP), R2
|
|
SUB $9, R2, R3
|
|
SUB $8, R2, R5
|
|
MOVW R5, 16(RSP)
|
|
LSR $0x05, R2, R2
|
|
SUBW R2, R3, R3
|
|
ADD R3, R1, R3
|
|
MOVD R3, 8(RSP)
|
|
MOVD $0x00000001, R2
|
|
MOVW R2, 24(RSP)
|
|
MOVD src_base+24(FP), R3
|
|
|
|
search_loop_encodeSnappyBlockAsm64K:
|
|
MOVWU R2, R5
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R5, R5
|
|
LSRW $0x06, R5, R5
|
|
ADD R5, R2, R5
|
|
ADD $4, R5, R5
|
|
MOVWU R5, R5
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R5
|
|
BHS emit_remainder_encodeSnappyBlockAsm64K
|
|
MOVD (R3)(R2), R6
|
|
MOVW R5, 28(RSP)
|
|
MOVD $0x0000cf1bbcdcbf9b, R8
|
|
MOVD R6, R9
|
|
MOVD R6, R10
|
|
LSR $0x08, R10, R10
|
|
LSL $0x10, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x32, R9, R9
|
|
LSL $0x10, R10, R10
|
|
MUL R8, R10, R10
|
|
LSR $0x32, R10, R10
|
|
MOVWU (R0)(R9<<2), R5
|
|
MOVWU (R0)(R10<<2), R7
|
|
MOVW R2, (R0)(R9<<2)
|
|
ADD $1, R2, R9
|
|
MOVWU R9, R9
|
|
MOVW R9, (R0)(R10<<2)
|
|
MOVD R6, R9
|
|
LSR $0x10, R9, R9
|
|
LSL $0x10, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x32, R9, R9
|
|
MOVWU R2, R8
|
|
MOVWU 24(RSP), R16
|
|
SUBW R16, R8, R8
|
|
ADD R8, R3, R15
|
|
MOVWU 1(R15), R10
|
|
MOVD R6, R8
|
|
LSR $0x08, R8, R8
|
|
CMPW R10, R8
|
|
BNE no_repeat_found_encodeSnappyBlockAsm64K
|
|
ADD $1, R2, R6
|
|
MOVWU R6, R6
|
|
MOVWU 20(RSP), R5
|
|
MOVWU R6, R7
|
|
MOVWU 24(RSP), R16
|
|
SUBSW R16, R7, R7
|
|
BEQ repeat_extend_back_end_encodeSnappyBlockAsm64K
|
|
|
|
repeat_extend_back_loop_encodeSnappyBlockAsm64K:
|
|
CMPW R5, R6
|
|
BLS repeat_extend_back_end_encodeSnappyBlockAsm64K
|
|
ADD R7, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R8
|
|
ADD R6, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R9
|
|
AND $0xff, R9, R16
|
|
AND $0xff, R8, R15
|
|
CMP R16, R15
|
|
BNE repeat_extend_back_end_encodeSnappyBlockAsm64K
|
|
SUB $1, R6, R6
|
|
MOVWU R6, R6
|
|
SUBSW $1, R7, R7
|
|
BNE repeat_extend_back_loop_encodeSnappyBlockAsm64K
|
|
|
|
repeat_extend_back_end_encodeSnappyBlockAsm64K:
|
|
MOVWU R6, R5
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R5, R5
|
|
ADD R5, R1, R5
|
|
ADD $3, R5, R5
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R5
|
|
BLO repeat_dst_size_check_encodeSnappyBlockAsm64K
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
repeat_dst_size_check_encodeSnappyBlockAsm64K:
|
|
MOVWU 20(RSP), R5
|
|
CMPW R6, R5
|
|
BEQ emit_literal_done_repeat_emit_encodeSnappyBlockAsm64K
|
|
MOVWU R6, R7
|
|
MOVW R6, 20(RSP)
|
|
ADD R5, R3, R8
|
|
SUBW R5, R7, R7
|
|
SUB $1, R7, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x3c, R5
|
|
BLO one_byte_repeat_emit_encodeSnappyBlockAsm64K
|
|
CMPW $0x00000100, R5
|
|
BLO two_bytes_repeat_emit_encodeSnappyBlockAsm64K
|
|
BLO three_bytes_repeat_emit_encodeSnappyBlockAsm64K
|
|
|
|
three_bytes_repeat_emit_encodeSnappyBlockAsm64K:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_repeat_emit_encodeSnappyBlockAsm64K
|
|
|
|
two_bytes_repeat_emit_encodeSnappyBlockAsm64K:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R5, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R5
|
|
BLO memmove_repeat_emit_encodeSnappyBlockAsm64K
|
|
JMP memmove_long_repeat_emit_encodeSnappyBlockAsm64K
|
|
|
|
one_byte_repeat_emit_encodeSnappyBlockAsm64K:
|
|
LSLW $0x02, R5, R16
|
|
BFI $0, R16, $8, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_repeat_emit_encodeSnappyBlockAsm64K:
|
|
ADD R7, R1, R5
|
|
|
|
// genMemMoveShort
|
|
CMP $0x08, R7
|
|
BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_8
|
|
CMP $0x10, R7
|
|
BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_8through16
|
|
CMP $0x20, R7
|
|
BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_17through32
|
|
JMP emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_33through64
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_8:
|
|
MOVD (R8), R9
|
|
MOVD R9, (R1)
|
|
JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm64K
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_8through16:
|
|
MOVD (R8), R9
|
|
ADD R7, R8, R15
|
|
MOVD -8(R15), R8
|
|
MOVD R9, (R1)
|
|
ADD R7, R1, R15
|
|
MOVD R8, -8(R15)
|
|
JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm64K
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_17through32:
|
|
FMOVQ (R8), F0
|
|
ADD R7, R8, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R7, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm64K
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_33through64:
|
|
FMOVQ (R8), F0
|
|
FMOVQ 16(R8), F1
|
|
ADD R7, R8, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R7, R8, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R7, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R7, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_repeat_emit_encodeSnappyBlockAsm64K:
|
|
MOVD R5, R1
|
|
JMP emit_literal_done_repeat_emit_encodeSnappyBlockAsm64K
|
|
|
|
memmove_long_repeat_emit_encodeSnappyBlockAsm64K:
|
|
ADD R7, R1, R5
|
|
|
|
// genMemMoveLong
|
|
MOVD R8, R9
|
|
MOVD R1, R10
|
|
MOVD R7, R11
|
|
|
|
emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm64Klarge_big_loop_back:
|
|
FMOVQ (R9), F0
|
|
FMOVQ 16(R9), F1
|
|
FMOVQ F0, (R10)
|
|
FMOVQ F1, 16(R10)
|
|
ADD $0x20, R9, R9
|
|
ADD $0x20, R10, R10
|
|
SUB $0x20, R11, R11
|
|
CMP $0x20, R11
|
|
BHS emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm64Klarge_big_loop_back
|
|
ADD R7, R8, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R7, R8, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R7, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R7, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R5, R1
|
|
|
|
emit_literal_done_repeat_emit_encodeSnappyBlockAsm64K:
|
|
ADDW $0x05, R2, R2
|
|
MOVWU R2, R5
|
|
MOVWU 24(RSP), R16
|
|
SUBW R16, R5, R5
|
|
MOVD src_len+32(FP), R7
|
|
SUBW R2, R7, R7
|
|
ADD R2, R3, R8
|
|
ADD R5, R3, R5
|
|
|
|
// matchLen
|
|
MOVD $0, R10
|
|
|
|
matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm64K:
|
|
CMPW $0x10, R7
|
|
BLO matchlen_match8_repeat_extend_encodeSnappyBlockAsm64K
|
|
MOVD (R8)(R10), R9
|
|
ADD R10, R8, R15
|
|
MOVD 8(R15), R11
|
|
MOVD (R5)(R10), R16
|
|
EOR R16, R9, R9
|
|
TST R9, R9
|
|
BNE matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm64K
|
|
ADD R10, R5, R15
|
|
MOVD 8(R15), R16
|
|
EOR R16, R11, R11
|
|
TST R11, R11
|
|
BNE matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm64K
|
|
SUB $16, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $16, R10, R10
|
|
MOVWU R10, R10
|
|
JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm64K
|
|
|
|
matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm64K:
|
|
RBIT R11, R11
|
|
CLZ R11, R11
|
|
ASR $0x03, R11, R11
|
|
ADD R11, R10, R10
|
|
ADD $8, R10, R10
|
|
MOVWU R10, R10
|
|
JMP repeat_extend_forward_end_encodeSnappyBlockAsm64K
|
|
|
|
matchlen_match8_repeat_extend_encodeSnappyBlockAsm64K:
|
|
CMPW $0x08, R7
|
|
BLO matchlen_match4_repeat_extend_encodeSnappyBlockAsm64K
|
|
MOVD (R8)(R10), R9
|
|
MOVD (R5)(R10), R16
|
|
EOR R16, R9, R9
|
|
TST R9, R9
|
|
BNE matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm64K
|
|
SUB $8, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $8, R10, R10
|
|
MOVWU R10, R10
|
|
JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm64K
|
|
|
|
matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm64K:
|
|
RBIT R9, R9
|
|
CLZ R9, R9
|
|
ASR $0x03, R9, R9
|
|
ADD R9, R10, R10
|
|
MOVWU R10, R10
|
|
JMP repeat_extend_forward_end_encodeSnappyBlockAsm64K
|
|
|
|
matchlen_match4_repeat_extend_encodeSnappyBlockAsm64K:
|
|
CMPW $0x04, R7
|
|
BLO matchlen_match2_repeat_extend_encodeSnappyBlockAsm64K
|
|
MOVWU (R8)(R10), R9
|
|
MOVWU (R5)(R10), R16
|
|
CMPW R9, R16
|
|
BNE matchlen_match2_repeat_extend_encodeSnappyBlockAsm64K
|
|
SUB $4, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $4, R10, R10
|
|
MOVWU R10, R10
|
|
|
|
matchlen_match2_repeat_extend_encodeSnappyBlockAsm64K:
|
|
CMPW $0x01, R7
|
|
BEQ matchlen_match1_repeat_extend_encodeSnappyBlockAsm64K
|
|
BLO repeat_extend_forward_end_encodeSnappyBlockAsm64K
|
|
MOVHU (R8)(R10), R16
|
|
BFI $0, R16, $16, R9
|
|
ADD R10, R5, R15
|
|
MOVHU (R15), R15
|
|
AND $0xffff, R9, R16
|
|
CMP R16, R15
|
|
BNE matchlen_match1_repeat_extend_encodeSnappyBlockAsm64K
|
|
ADD $2, R10, R10
|
|
MOVWU R10, R10
|
|
SUBSW $0x02, R7, R7
|
|
BEQ repeat_extend_forward_end_encodeSnappyBlockAsm64K
|
|
|
|
matchlen_match1_repeat_extend_encodeSnappyBlockAsm64K:
|
|
MOVBU (R8)(R10), R16
|
|
BFI $0, R16, $8, R9
|
|
ADD R10, R5, R15
|
|
MOVBU (R15), R15
|
|
AND $0xff, R9, R16
|
|
CMP R16, R15
|
|
BNE repeat_extend_forward_end_encodeSnappyBlockAsm64K
|
|
ADD $1, R10, R10
|
|
MOVWU R10, R10
|
|
|
|
repeat_extend_forward_end_encodeSnappyBlockAsm64K:
|
|
ADDW R10, R2, R2
|
|
MOVWU R2, R5
|
|
SUBW R6, R5, R5
|
|
MOVWU 24(RSP), R6
|
|
|
|
// emitCopy
|
|
two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm64K:
|
|
CMPW $0x40, R5
|
|
BLS two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm64K
|
|
MOVD $0xee, R16
|
|
MOVB R16, (R1)
|
|
MOVH R6, 1(R1)
|
|
SUB $60, R5, R5
|
|
MOVWU R5, R5
|
|
ADD $0x03, R1, R1
|
|
JMP two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm64K
|
|
|
|
two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm64K:
|
|
MOVWU R5, R7
|
|
LSLW $0x02, R7, R7
|
|
CMPW $0x0c, R5
|
|
BHS emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm64K
|
|
CMPW $0x00000800, R6
|
|
BHS emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm64K
|
|
SUB $15, R7, R7
|
|
MOVWU R7, R7
|
|
MOVB R6, 1(R1)
|
|
LSRW $0x08, R6, R6
|
|
LSLW $0x05, R6, R6
|
|
ORRW R6, R7, R7
|
|
MOVB R7, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeSnappyBlockAsm64K
|
|
|
|
emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm64K:
|
|
SUB $2, R7, R7
|
|
MOVWU R7, R7
|
|
MOVB R7, (R1)
|
|
MOVH R6, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
|
|
repeat_end_emit_encodeSnappyBlockAsm64K:
|
|
MOVW R2, 20(RSP)
|
|
JMP search_loop_encodeSnappyBlockAsm64K
|
|
|
|
no_repeat_found_encodeSnappyBlockAsm64K:
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ candidate_match_encodeSnappyBlockAsm64K
|
|
LSR $0x08, R6, R6
|
|
MOVWU (R0)(R9<<2), R5
|
|
ADD $2, R2, R8
|
|
MOVWU R8, R8
|
|
MOVWU (R3)(R7), R16
|
|
CMPW R6, R16
|
|
BEQ candidate2_match_encodeSnappyBlockAsm64K
|
|
MOVW R8, (R0)(R9<<2)
|
|
LSR $0x08, R6, R6
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ candidate3_match_encodeSnappyBlockAsm64K
|
|
MOVWU 28(RSP), R2
|
|
JMP search_loop_encodeSnappyBlockAsm64K
|
|
|
|
candidate3_match_encodeSnappyBlockAsm64K:
|
|
ADDW $0x02, R2, R2
|
|
JMP candidate_match_encodeSnappyBlockAsm64K
|
|
|
|
candidate2_match_encodeSnappyBlockAsm64K:
|
|
MOVW R8, (R0)(R9<<2)
|
|
ADDW $1, R2, R2
|
|
MOVWU R7, R5
|
|
|
|
candidate_match_encodeSnappyBlockAsm64K:
|
|
MOVWU 20(RSP), R6
|
|
TSTW R5, R5
|
|
BEQ match_extend_back_end_encodeSnappyBlockAsm64K
|
|
|
|
match_extend_back_loop_encodeSnappyBlockAsm64K:
|
|
CMPW R6, R2
|
|
BLS match_extend_back_end_encodeSnappyBlockAsm64K
|
|
ADD R5, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R7
|
|
ADD R2, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R8
|
|
AND $0xff, R8, R16
|
|
AND $0xff, R7, R15
|
|
CMP R16, R15
|
|
BNE match_extend_back_end_encodeSnappyBlockAsm64K
|
|
SUB $1, R2, R2
|
|
MOVWU R2, R2
|
|
SUBSW $1, R5, R5
|
|
BEQ match_extend_back_end_encodeSnappyBlockAsm64K
|
|
JMP match_extend_back_loop_encodeSnappyBlockAsm64K
|
|
|
|
match_extend_back_end_encodeSnappyBlockAsm64K:
|
|
MOVWU R2, R6
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R6, R6
|
|
ADD R6, R1, R6
|
|
ADD $3, R6, R6
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R6
|
|
BLO match_dst_size_check_encodeSnappyBlockAsm64K
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeSnappyBlockAsm64K:
|
|
MOVWU R2, R6
|
|
MOVWU 20(RSP), R7
|
|
CMPW R6, R7
|
|
BEQ emit_literal_done_match_emit_encodeSnappyBlockAsm64K
|
|
MOVWU R6, R8
|
|
MOVW R6, 20(RSP)
|
|
ADD R7, R3, R6
|
|
SUBW R7, R8, R8
|
|
SUB $1, R8, R7
|
|
MOVWU R7, R7
|
|
CMPW $0x3c, R7
|
|
BLO one_byte_match_emit_encodeSnappyBlockAsm64K
|
|
CMPW $0x00000100, R7
|
|
BLO two_bytes_match_emit_encodeSnappyBlockAsm64K
|
|
BLO three_bytes_match_emit_encodeSnappyBlockAsm64K
|
|
|
|
three_bytes_match_emit_encodeSnappyBlockAsm64K:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R7, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_match_emit_encodeSnappyBlockAsm64K
|
|
|
|
two_bytes_match_emit_encodeSnappyBlockAsm64K:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R7, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R7
|
|
BLO memmove_match_emit_encodeSnappyBlockAsm64K
|
|
JMP memmove_long_match_emit_encodeSnappyBlockAsm64K
|
|
|
|
one_byte_match_emit_encodeSnappyBlockAsm64K:
|
|
LSLW $0x02, R7, R16
|
|
BFI $0, R16, $8, R7
|
|
MOVB R7, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_match_emit_encodeSnappyBlockAsm64K:
|
|
ADD R8, R1, R7
|
|
|
|
// genMemMoveShort
|
|
CMP $0x08, R8
|
|
BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_8
|
|
CMP $0x10, R8
|
|
BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_8through16
|
|
CMP $0x20, R8
|
|
BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_8:
|
|
MOVD (R6), R9
|
|
MOVD R9, (R1)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm64K
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_8through16:
|
|
MOVD (R6), R9
|
|
ADD R8, R6, R15
|
|
MOVD -8(R15), R6
|
|
MOVD R9, (R1)
|
|
ADD R8, R1, R15
|
|
MOVD R6, -8(R15)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm64K
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_17through32:
|
|
FMOVQ (R6), F0
|
|
ADD R8, R6, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm64K
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_33through64:
|
|
FMOVQ (R6), F0
|
|
FMOVQ 16(R6), F1
|
|
ADD R8, R6, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R8, R6, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_match_emit_encodeSnappyBlockAsm64K:
|
|
MOVD R7, R1
|
|
JMP emit_literal_done_match_emit_encodeSnappyBlockAsm64K
|
|
|
|
memmove_long_match_emit_encodeSnappyBlockAsm64K:
|
|
ADD R8, R1, R7
|
|
|
|
// genMemMoveLong
|
|
MOVD R6, R9
|
|
MOVD R1, R10
|
|
MOVD R8, R11
|
|
|
|
emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm64Klarge_big_loop_back:
|
|
FMOVQ (R9), F0
|
|
FMOVQ 16(R9), F1
|
|
FMOVQ F0, (R10)
|
|
FMOVQ F1, 16(R10)
|
|
ADD $0x20, R9, R9
|
|
ADD $0x20, R10, R10
|
|
SUB $0x20, R11, R11
|
|
CMP $0x20, R11
|
|
BHS emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm64Klarge_big_loop_back
|
|
ADD R8, R6, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R8, R6, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R8, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R7, R1
|
|
|
|
emit_literal_done_match_emit_encodeSnappyBlockAsm64K:
|
|
match_nolit_loop_encodeSnappyBlockAsm64K:
|
|
MOVWU R2, R6
|
|
SUBW R5, R6, R6
|
|
MOVW R6, 24(RSP)
|
|
ADDW $0x04, R2, R2
|
|
ADDW $0x04, R5, R5
|
|
MOVD src_len+32(FP), R6
|
|
SUBW R2, R6, R6
|
|
ADD R2, R3, R7
|
|
ADD R5, R3, R5
|
|
|
|
// matchLen
|
|
MOVD $0, R9
|
|
|
|
matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm64K:
|
|
CMPW $0x10, R6
|
|
BLO matchlen_match8_match_nolit_encodeSnappyBlockAsm64K
|
|
MOVD (R7)(R9), R8
|
|
ADD R9, R7, R15
|
|
MOVD 8(R15), R10
|
|
MOVD (R5)(R9), R16
|
|
EOR R16, R8, R8
|
|
TST R8, R8
|
|
BNE matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm64K
|
|
ADD R9, R5, R15
|
|
MOVD 8(R15), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_16match_nolit_encodeSnappyBlockAsm64K
|
|
SUB $16, R6, R6
|
|
MOVWU R6, R6
|
|
ADD $16, R9, R9
|
|
MOVWU R9, R9
|
|
JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm64K
|
|
|
|
matchlen_bsf_16match_nolit_encodeSnappyBlockAsm64K:
|
|
RBIT R10, R10
|
|
CLZ R10, R10
|
|
ASR $0x03, R10, R10
|
|
ADD R10, R9, R9
|
|
ADD $8, R9, R9
|
|
MOVWU R9, R9
|
|
JMP match_nolit_end_encodeSnappyBlockAsm64K
|
|
|
|
matchlen_match8_match_nolit_encodeSnappyBlockAsm64K:
|
|
CMPW $0x08, R6
|
|
BLO matchlen_match4_match_nolit_encodeSnappyBlockAsm64K
|
|
MOVD (R7)(R9), R8
|
|
MOVD (R5)(R9), R16
|
|
EOR R16, R8, R8
|
|
TST R8, R8
|
|
BNE matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm64K
|
|
SUB $8, R6, R6
|
|
MOVWU R6, R6
|
|
ADD $8, R9, R9
|
|
MOVWU R9, R9
|
|
JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm64K
|
|
|
|
matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm64K:
|
|
RBIT R8, R8
|
|
CLZ R8, R8
|
|
ASR $0x03, R8, R8
|
|
ADD R8, R9, R9
|
|
MOVWU R9, R9
|
|
JMP match_nolit_end_encodeSnappyBlockAsm64K
|
|
|
|
matchlen_match4_match_nolit_encodeSnappyBlockAsm64K:
|
|
CMPW $0x04, R6
|
|
BLO matchlen_match2_match_nolit_encodeSnappyBlockAsm64K
|
|
MOVWU (R7)(R9), R8
|
|
MOVWU (R5)(R9), R16
|
|
CMPW R8, R16
|
|
BNE matchlen_match2_match_nolit_encodeSnappyBlockAsm64K
|
|
SUB $4, R6, R6
|
|
MOVWU R6, R6
|
|
ADD $4, R9, R9
|
|
MOVWU R9, R9
|
|
|
|
matchlen_match2_match_nolit_encodeSnappyBlockAsm64K:
|
|
CMPW $0x01, R6
|
|
BEQ matchlen_match1_match_nolit_encodeSnappyBlockAsm64K
|
|
BLO match_nolit_end_encodeSnappyBlockAsm64K
|
|
MOVHU (R7)(R9), R16
|
|
BFI $0, R16, $16, R8
|
|
ADD R9, R5, R15
|
|
MOVHU (R15), R15
|
|
AND $0xffff, R8, R16
|
|
CMP R16, R15
|
|
BNE matchlen_match1_match_nolit_encodeSnappyBlockAsm64K
|
|
ADD $2, R9, R9
|
|
MOVWU R9, R9
|
|
SUBSW $0x02, R6, R6
|
|
BEQ match_nolit_end_encodeSnappyBlockAsm64K
|
|
|
|
matchlen_match1_match_nolit_encodeSnappyBlockAsm64K:
|
|
MOVBU (R7)(R9), R16
|
|
BFI $0, R16, $8, R8
|
|
ADD R9, R5, R15
|
|
MOVBU (R15), R15
|
|
AND $0xff, R8, R16
|
|
CMP R16, R15
|
|
BNE match_nolit_end_encodeSnappyBlockAsm64K
|
|
ADD $1, R9, R9
|
|
MOVWU R9, R9
|
|
|
|
match_nolit_end_encodeSnappyBlockAsm64K:
|
|
ADDW R9, R2, R2
|
|
MOVWU 24(RSP), R5
|
|
ADDW $0x04, R9, R9
|
|
MOVW R2, 20(RSP)
|
|
|
|
// emitCopy
|
|
two_byte_offset_match_nolit_encodeSnappyBlockAsm64K:
|
|
CMPW $0x40, R9
|
|
BLS two_byte_offset_short_match_nolit_encodeSnappyBlockAsm64K
|
|
MOVD $0xee, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
SUB $60, R9, R9
|
|
MOVWU R9, R9
|
|
ADD $0x03, R1, R1
|
|
JMP two_byte_offset_match_nolit_encodeSnappyBlockAsm64K
|
|
|
|
two_byte_offset_short_match_nolit_encodeSnappyBlockAsm64K:
|
|
MOVWU R9, R6
|
|
LSLW $0x02, R6, R6
|
|
CMPW $0x0c, R9
|
|
BHS emit_copy_three_match_nolit_encodeSnappyBlockAsm64K
|
|
CMPW $0x00000800, R5
|
|
BHS emit_copy_three_match_nolit_encodeSnappyBlockAsm64K
|
|
SUB $15, R6, R6
|
|
MOVWU R6, R6
|
|
MOVB R5, 1(R1)
|
|
LSRW $0x08, R5, R5
|
|
LSLW $0x05, R5, R5
|
|
ORRW R5, R6, R6
|
|
MOVB R6, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeSnappyBlockAsm64K
|
|
|
|
emit_copy_three_match_nolit_encodeSnappyBlockAsm64K:
|
|
SUB $2, R6, R6
|
|
MOVWU R6, R6
|
|
MOVB R6, (R1)
|
|
MOVH R5, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
|
|
match_nolit_emitcopy_end_encodeSnappyBlockAsm64K:
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R2
|
|
BHS emit_remainder_encodeSnappyBlockAsm64K
|
|
ADD R2, R3, R15
|
|
MOVD -2(R15), R6
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R1
|
|
BLO match_nolit_dst_ok_encodeSnappyBlockAsm64K
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeSnappyBlockAsm64K:
|
|
MOVD $0x0000cf1bbcdcbf9b, R8
|
|
MOVD R6, R7
|
|
LSR $0x10, R6, R6
|
|
MOVD R6, R5
|
|
LSL $0x10, R7, R7
|
|
MUL R8, R7, R7
|
|
LSR $0x32, R7, R7
|
|
LSL $0x10, R5, R5
|
|
MUL R8, R5, R5
|
|
LSR $0x32, R5, R5
|
|
SUB $2, R2, R8
|
|
MOVWU R8, R8
|
|
ADD R5<<2, R0, R9
|
|
MOVWU (R9), R5
|
|
MOVW R8, (R0)(R7<<2)
|
|
MOVW R2, (R9)
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ match_nolit_loop_encodeSnappyBlockAsm64K
|
|
ADDW $1, R2, R2
|
|
JMP search_loop_encodeSnappyBlockAsm64K
|
|
|
|
emit_remainder_encodeSnappyBlockAsm64K:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R0, R0
|
|
ADD R0, R1, R0
|
|
ADD $3, R0, R0
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R0
|
|
BLO emit_remainder_ok_encodeSnappyBlockAsm64K
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeSnappyBlockAsm64K:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R2
|
|
CMPW R0, R2
|
|
BEQ emit_literal_done_emit_remainder_encodeSnappyBlockAsm64K
|
|
MOVWU R0, R5
|
|
MOVW R0, 20(RSP)
|
|
ADD R2, R3, R0
|
|
SUBW R2, R5, R5
|
|
SUB $1, R5, R2
|
|
MOVWU R2, R2
|
|
CMPW $0x3c, R2
|
|
BLO one_byte_emit_remainder_encodeSnappyBlockAsm64K
|
|
CMPW $0x00000100, R2
|
|
BLO two_bytes_emit_remainder_encodeSnappyBlockAsm64K
|
|
BLO three_bytes_emit_remainder_encodeSnappyBlockAsm64K
|
|
|
|
three_bytes_emit_remainder_encodeSnappyBlockAsm64K:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R2, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_emit_remainder_encodeSnappyBlockAsm64K
|
|
|
|
two_bytes_emit_remainder_encodeSnappyBlockAsm64K:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R2, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R2
|
|
BLO memmove_emit_remainder_encodeSnappyBlockAsm64K
|
|
JMP memmove_long_emit_remainder_encodeSnappyBlockAsm64K
|
|
|
|
one_byte_emit_remainder_encodeSnappyBlockAsm64K:
|
|
LSLW $0x02, R2, R16
|
|
BFI $0, R16, $8, R2
|
|
MOVB R2, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_emit_remainder_encodeSnappyBlockAsm64K:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveShort
|
|
CMP $0x03, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_1or2
|
|
BEQ emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_3
|
|
CMP $0x08, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_4through7
|
|
CMP $0x10, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_8through16
|
|
CMP $0x20, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_1or2:
|
|
MOVBU (R0), R16
|
|
BFI $0, R16, $8, R5
|
|
ADD R3, R0, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVB R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVB R0, -1(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm64K
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_3:
|
|
MOVHU (R0), R16
|
|
BFI $0, R16, $16, R5
|
|
MOVBU 2(R0), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVH R5, (R1)
|
|
MOVB R0, 2(R1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm64K
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_4through7:
|
|
MOVWU (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVWU -4(R15), R0
|
|
MOVW R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVW R0, -4(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm64K
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_8through16:
|
|
MOVD (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVD -8(R15), R0
|
|
MOVD R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVD R0, -8(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm64K
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_17through32:
|
|
FMOVQ (R0), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm64K
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_33through64:
|
|
FMOVQ (R0), F0
|
|
FMOVQ 16(R0), F1
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_emit_remainder_encodeSnappyBlockAsm64K:
|
|
MOVD R2, R1
|
|
JMP emit_literal_done_emit_remainder_encodeSnappyBlockAsm64K
|
|
|
|
memmove_long_emit_remainder_encodeSnappyBlockAsm64K:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveLong
|
|
MOVD R0, R5
|
|
MOVD R1, R6
|
|
MOVD R3, R7
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm64Klarge_big_loop_back:
|
|
FMOVQ (R5), F0
|
|
FMOVQ 16(R5), F1
|
|
FMOVQ F0, (R6)
|
|
FMOVQ F1, 16(R6)
|
|
ADD $0x20, R5, R5
|
|
ADD $0x20, R6, R6
|
|
SUB $0x20, R7, R7
|
|
CMP $0x20, R7
|
|
BHS emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm64Klarge_big_loop_back
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R3, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R2, R1
|
|
|
|
emit_literal_done_emit_remainder_encodeSnappyBlockAsm64K:
|
|
MOVD dst_base+0(FP), R0
|
|
SUB R0, R1, R1
|
|
MOVD R1, ret+56(FP)
|
|
RET
|
|
|
|
// func encodeSnappyBlockAsm12B(dst []byte, src []byte, tmp *[16384]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeSnappyBlockAsm12B(SB), $24-64
|
|
MOVD tmp+48(FP), R0
|
|
MOVD dst_base+0(FP), R1
|
|
MOVD $0x00000080, R2
|
|
MOVD R0, R3
|
|
VEOR V0.B16, V0.B16, V0.B16
|
|
|
|
zero_loop_encodeSnappyBlockAsm12B:
|
|
FMOVQ F0, (R3)
|
|
FMOVQ F0, 16(R3)
|
|
FMOVQ F0, 32(R3)
|
|
FMOVQ F0, 48(R3)
|
|
FMOVQ F0, 64(R3)
|
|
FMOVQ F0, 80(R3)
|
|
FMOVQ F0, 96(R3)
|
|
FMOVQ F0, 112(R3)
|
|
ADD $0x80, R3, R3
|
|
SUBS $1, R2, R2
|
|
BNE zero_loop_encodeSnappyBlockAsm12B
|
|
MOVD $0x00000000, R16
|
|
MOVW R16, 20(RSP)
|
|
MOVD src_len+32(FP), R2
|
|
SUB $9, R2, R3
|
|
SUB $8, R2, R5
|
|
MOVW R5, 16(RSP)
|
|
LSR $0x05, R2, R2
|
|
SUBW R2, R3, R3
|
|
ADD R3, R1, R3
|
|
MOVD R3, 8(RSP)
|
|
MOVD $0x00000001, R2
|
|
MOVW R2, 24(RSP)
|
|
MOVD src_base+24(FP), R3
|
|
|
|
search_loop_encodeSnappyBlockAsm12B:
|
|
MOVWU R2, R5
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R5, R5
|
|
LSRW $0x05, R5, R5
|
|
ADD R5, R2, R5
|
|
ADD $4, R5, R5
|
|
MOVWU R5, R5
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R5
|
|
BHS emit_remainder_encodeSnappyBlockAsm12B
|
|
MOVD (R3)(R2), R6
|
|
MOVW R5, 28(RSP)
|
|
MOVD $0x000000cf1bbcdcbb, R8
|
|
MOVD R6, R9
|
|
MOVD R6, R10
|
|
LSR $0x08, R10, R10
|
|
LSL $0x18, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x34, R9, R9
|
|
LSL $0x18, R10, R10
|
|
MUL R8, R10, R10
|
|
LSR $0x34, R10, R10
|
|
MOVWU (R0)(R9<<2), R5
|
|
MOVWU (R0)(R10<<2), R7
|
|
MOVW R2, (R0)(R9<<2)
|
|
ADD $1, R2, R9
|
|
MOVWU R9, R9
|
|
MOVW R9, (R0)(R10<<2)
|
|
MOVD R6, R9
|
|
LSR $0x10, R9, R9
|
|
LSL $0x18, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x34, R9, R9
|
|
MOVWU R2, R8
|
|
MOVWU 24(RSP), R16
|
|
SUBW R16, R8, R8
|
|
ADD R8, R3, R15
|
|
MOVWU 1(R15), R10
|
|
MOVD R6, R8
|
|
LSR $0x08, R8, R8
|
|
CMPW R10, R8
|
|
BNE no_repeat_found_encodeSnappyBlockAsm12B
|
|
ADD $1, R2, R6
|
|
MOVWU R6, R6
|
|
MOVWU 20(RSP), R5
|
|
MOVWU R6, R7
|
|
MOVWU 24(RSP), R16
|
|
SUBSW R16, R7, R7
|
|
BEQ repeat_extend_back_end_encodeSnappyBlockAsm12B
|
|
|
|
repeat_extend_back_loop_encodeSnappyBlockAsm12B:
|
|
CMPW R5, R6
|
|
BLS repeat_extend_back_end_encodeSnappyBlockAsm12B
|
|
ADD R7, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R8
|
|
ADD R6, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R9
|
|
AND $0xff, R9, R16
|
|
AND $0xff, R8, R15
|
|
CMP R16, R15
|
|
BNE repeat_extend_back_end_encodeSnappyBlockAsm12B
|
|
SUB $1, R6, R6
|
|
MOVWU R6, R6
|
|
SUBSW $1, R7, R7
|
|
BNE repeat_extend_back_loop_encodeSnappyBlockAsm12B
|
|
|
|
repeat_extend_back_end_encodeSnappyBlockAsm12B:
|
|
MOVWU R6, R5
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R5, R5
|
|
ADD R5, R1, R5
|
|
ADD $3, R5, R5
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R5
|
|
BLO repeat_dst_size_check_encodeSnappyBlockAsm12B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
repeat_dst_size_check_encodeSnappyBlockAsm12B:
|
|
MOVWU 20(RSP), R5
|
|
CMPW R6, R5
|
|
BEQ emit_literal_done_repeat_emit_encodeSnappyBlockAsm12B
|
|
MOVWU R6, R7
|
|
MOVW R6, 20(RSP)
|
|
ADD R5, R3, R8
|
|
SUBW R5, R7, R7
|
|
SUB $1, R7, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x3c, R5
|
|
BLO one_byte_repeat_emit_encodeSnappyBlockAsm12B
|
|
CMPW $0x00000100, R5
|
|
BLO two_bytes_repeat_emit_encodeSnappyBlockAsm12B
|
|
BLO three_bytes_repeat_emit_encodeSnappyBlockAsm12B
|
|
|
|
three_bytes_repeat_emit_encodeSnappyBlockAsm12B:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_repeat_emit_encodeSnappyBlockAsm12B
|
|
|
|
two_bytes_repeat_emit_encodeSnappyBlockAsm12B:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R5, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R5
|
|
BLO memmove_repeat_emit_encodeSnappyBlockAsm12B
|
|
JMP memmove_long_repeat_emit_encodeSnappyBlockAsm12B
|
|
|
|
one_byte_repeat_emit_encodeSnappyBlockAsm12B:
|
|
LSLW $0x02, R5, R16
|
|
BFI $0, R16, $8, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_repeat_emit_encodeSnappyBlockAsm12B:
|
|
ADD R7, R1, R5
|
|
|
|
// genMemMoveShort
|
|
CMP $0x08, R7
|
|
BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_8
|
|
CMP $0x10, R7
|
|
BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_8through16
|
|
CMP $0x20, R7
|
|
BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_17through32
|
|
JMP emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_8:
|
|
MOVD (R8), R9
|
|
MOVD R9, (R1)
|
|
JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm12B
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_8through16:
|
|
MOVD (R8), R9
|
|
ADD R7, R8, R15
|
|
MOVD -8(R15), R8
|
|
MOVD R9, (R1)
|
|
ADD R7, R1, R15
|
|
MOVD R8, -8(R15)
|
|
JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm12B
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_17through32:
|
|
FMOVQ (R8), F0
|
|
ADD R7, R8, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R7, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm12B
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_33through64:
|
|
FMOVQ (R8), F0
|
|
FMOVQ 16(R8), F1
|
|
ADD R7, R8, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R7, R8, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R7, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R7, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_repeat_emit_encodeSnappyBlockAsm12B:
|
|
MOVD R5, R1
|
|
JMP emit_literal_done_repeat_emit_encodeSnappyBlockAsm12B
|
|
|
|
memmove_long_repeat_emit_encodeSnappyBlockAsm12B:
|
|
ADD R7, R1, R5
|
|
|
|
// genMemMoveLong
|
|
MOVD R8, R9
|
|
MOVD R1, R10
|
|
MOVD R7, R11
|
|
|
|
emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm12Blarge_big_loop_back:
|
|
FMOVQ (R9), F0
|
|
FMOVQ 16(R9), F1
|
|
FMOVQ F0, (R10)
|
|
FMOVQ F1, 16(R10)
|
|
ADD $0x20, R9, R9
|
|
ADD $0x20, R10, R10
|
|
SUB $0x20, R11, R11
|
|
CMP $0x20, R11
|
|
BHS emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm12Blarge_big_loop_back
|
|
ADD R7, R8, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R7, R8, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R7, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R7, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R5, R1
|
|
|
|
emit_literal_done_repeat_emit_encodeSnappyBlockAsm12B:
|
|
ADDW $0x05, R2, R2
|
|
MOVWU R2, R5
|
|
MOVWU 24(RSP), R16
|
|
SUBW R16, R5, R5
|
|
MOVD src_len+32(FP), R7
|
|
SUBW R2, R7, R7
|
|
ADD R2, R3, R8
|
|
ADD R5, R3, R5
|
|
|
|
// matchLen
|
|
MOVD $0, R10
|
|
|
|
matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm12B:
|
|
CMPW $0x10, R7
|
|
BLO matchlen_match8_repeat_extend_encodeSnappyBlockAsm12B
|
|
MOVD (R8)(R10), R9
|
|
ADD R10, R8, R15
|
|
MOVD 8(R15), R11
|
|
MOVD (R5)(R10), R16
|
|
EOR R16, R9, R9
|
|
TST R9, R9
|
|
BNE matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm12B
|
|
ADD R10, R5, R15
|
|
MOVD 8(R15), R16
|
|
EOR R16, R11, R11
|
|
TST R11, R11
|
|
BNE matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm12B
|
|
SUB $16, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $16, R10, R10
|
|
MOVWU R10, R10
|
|
JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm12B
|
|
|
|
matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm12B:
|
|
RBIT R11, R11
|
|
CLZ R11, R11
|
|
ASR $0x03, R11, R11
|
|
ADD R11, R10, R10
|
|
ADD $8, R10, R10
|
|
MOVWU R10, R10
|
|
JMP repeat_extend_forward_end_encodeSnappyBlockAsm12B
|
|
|
|
matchlen_match8_repeat_extend_encodeSnappyBlockAsm12B:
|
|
CMPW $0x08, R7
|
|
BLO matchlen_match4_repeat_extend_encodeSnappyBlockAsm12B
|
|
MOVD (R8)(R10), R9
|
|
MOVD (R5)(R10), R16
|
|
EOR R16, R9, R9
|
|
TST R9, R9
|
|
BNE matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm12B
|
|
SUB $8, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $8, R10, R10
|
|
MOVWU R10, R10
|
|
JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm12B
|
|
|
|
matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm12B:
|
|
RBIT R9, R9
|
|
CLZ R9, R9
|
|
ASR $0x03, R9, R9
|
|
ADD R9, R10, R10
|
|
MOVWU R10, R10
|
|
JMP repeat_extend_forward_end_encodeSnappyBlockAsm12B
|
|
|
|
matchlen_match4_repeat_extend_encodeSnappyBlockAsm12B:
|
|
CMPW $0x04, R7
|
|
BLO matchlen_match2_repeat_extend_encodeSnappyBlockAsm12B
|
|
MOVWU (R8)(R10), R9
|
|
MOVWU (R5)(R10), R16
|
|
CMPW R9, R16
|
|
BNE matchlen_match2_repeat_extend_encodeSnappyBlockAsm12B
|
|
SUB $4, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $4, R10, R10
|
|
MOVWU R10, R10
|
|
|
|
matchlen_match2_repeat_extend_encodeSnappyBlockAsm12B:
|
|
CMPW $0x01, R7
|
|
BEQ matchlen_match1_repeat_extend_encodeSnappyBlockAsm12B
|
|
BLO repeat_extend_forward_end_encodeSnappyBlockAsm12B
|
|
MOVHU (R8)(R10), R16
|
|
BFI $0, R16, $16, R9
|
|
ADD R10, R5, R15
|
|
MOVHU (R15), R15
|
|
AND $0xffff, R9, R16
|
|
CMP R16, R15
|
|
BNE matchlen_match1_repeat_extend_encodeSnappyBlockAsm12B
|
|
ADD $2, R10, R10
|
|
MOVWU R10, R10
|
|
SUBSW $0x02, R7, R7
|
|
BEQ repeat_extend_forward_end_encodeSnappyBlockAsm12B
|
|
|
|
matchlen_match1_repeat_extend_encodeSnappyBlockAsm12B:
|
|
MOVBU (R8)(R10), R16
|
|
BFI $0, R16, $8, R9
|
|
ADD R10, R5, R15
|
|
MOVBU (R15), R15
|
|
AND $0xff, R9, R16
|
|
CMP R16, R15
|
|
BNE repeat_extend_forward_end_encodeSnappyBlockAsm12B
|
|
ADD $1, R10, R10
|
|
MOVWU R10, R10
|
|
|
|
repeat_extend_forward_end_encodeSnappyBlockAsm12B:
|
|
ADDW R10, R2, R2
|
|
MOVWU R2, R5
|
|
SUBW R6, R5, R5
|
|
MOVWU 24(RSP), R6
|
|
|
|
// emitCopy
|
|
two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm12B:
|
|
CMPW $0x40, R5
|
|
BLS two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm12B
|
|
MOVD $0xee, R16
|
|
MOVB R16, (R1)
|
|
MOVH R6, 1(R1)
|
|
SUB $60, R5, R5
|
|
MOVWU R5, R5
|
|
ADD $0x03, R1, R1
|
|
JMP two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm12B
|
|
|
|
two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm12B:
|
|
MOVWU R5, R7
|
|
LSLW $0x02, R7, R7
|
|
CMPW $0x0c, R5
|
|
BHS emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm12B
|
|
CMPW $0x00000800, R6
|
|
BHS emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm12B
|
|
SUB $15, R7, R7
|
|
MOVWU R7, R7
|
|
MOVB R6, 1(R1)
|
|
LSRW $0x08, R6, R6
|
|
LSLW $0x05, R6, R6
|
|
ORRW R6, R7, R7
|
|
MOVB R7, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeSnappyBlockAsm12B
|
|
|
|
emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm12B:
|
|
SUB $2, R7, R7
|
|
MOVWU R7, R7
|
|
MOVB R7, (R1)
|
|
MOVH R6, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
|
|
repeat_end_emit_encodeSnappyBlockAsm12B:
|
|
MOVW R2, 20(RSP)
|
|
JMP search_loop_encodeSnappyBlockAsm12B
|
|
|
|
no_repeat_found_encodeSnappyBlockAsm12B:
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ candidate_match_encodeSnappyBlockAsm12B
|
|
LSR $0x08, R6, R6
|
|
MOVWU (R0)(R9<<2), R5
|
|
ADD $2, R2, R8
|
|
MOVWU R8, R8
|
|
MOVWU (R3)(R7), R16
|
|
CMPW R6, R16
|
|
BEQ candidate2_match_encodeSnappyBlockAsm12B
|
|
MOVW R8, (R0)(R9<<2)
|
|
LSR $0x08, R6, R6
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ candidate3_match_encodeSnappyBlockAsm12B
|
|
MOVWU 28(RSP), R2
|
|
JMP search_loop_encodeSnappyBlockAsm12B
|
|
|
|
candidate3_match_encodeSnappyBlockAsm12B:
|
|
ADDW $0x02, R2, R2
|
|
JMP candidate_match_encodeSnappyBlockAsm12B
|
|
|
|
candidate2_match_encodeSnappyBlockAsm12B:
|
|
MOVW R8, (R0)(R9<<2)
|
|
ADDW $1, R2, R2
|
|
MOVWU R7, R5
|
|
|
|
candidate_match_encodeSnappyBlockAsm12B:
|
|
MOVWU 20(RSP), R6
|
|
TSTW R5, R5
|
|
BEQ match_extend_back_end_encodeSnappyBlockAsm12B
|
|
|
|
match_extend_back_loop_encodeSnappyBlockAsm12B:
|
|
CMPW R6, R2
|
|
BLS match_extend_back_end_encodeSnappyBlockAsm12B
|
|
ADD R5, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R7
|
|
ADD R2, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R8
|
|
AND $0xff, R8, R16
|
|
AND $0xff, R7, R15
|
|
CMP R16, R15
|
|
BNE match_extend_back_end_encodeSnappyBlockAsm12B
|
|
SUB $1, R2, R2
|
|
MOVWU R2, R2
|
|
SUBSW $1, R5, R5
|
|
BEQ match_extend_back_end_encodeSnappyBlockAsm12B
|
|
JMP match_extend_back_loop_encodeSnappyBlockAsm12B
|
|
|
|
match_extend_back_end_encodeSnappyBlockAsm12B:
|
|
MOVWU R2, R6
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R6, R6
|
|
ADD R6, R1, R6
|
|
ADD $3, R6, R6
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R6
|
|
BLO match_dst_size_check_encodeSnappyBlockAsm12B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeSnappyBlockAsm12B:
|
|
MOVWU R2, R6
|
|
MOVWU 20(RSP), R7
|
|
CMPW R6, R7
|
|
BEQ emit_literal_done_match_emit_encodeSnappyBlockAsm12B
|
|
MOVWU R6, R8
|
|
MOVW R6, 20(RSP)
|
|
ADD R7, R3, R6
|
|
SUBW R7, R8, R8
|
|
SUB $1, R8, R7
|
|
MOVWU R7, R7
|
|
CMPW $0x3c, R7
|
|
BLO one_byte_match_emit_encodeSnappyBlockAsm12B
|
|
CMPW $0x00000100, R7
|
|
BLO two_bytes_match_emit_encodeSnappyBlockAsm12B
|
|
BLO three_bytes_match_emit_encodeSnappyBlockAsm12B
|
|
|
|
three_bytes_match_emit_encodeSnappyBlockAsm12B:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R7, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_match_emit_encodeSnappyBlockAsm12B
|
|
|
|
two_bytes_match_emit_encodeSnappyBlockAsm12B:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R7, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R7
|
|
BLO memmove_match_emit_encodeSnappyBlockAsm12B
|
|
JMP memmove_long_match_emit_encodeSnappyBlockAsm12B
|
|
|
|
one_byte_match_emit_encodeSnappyBlockAsm12B:
|
|
LSLW $0x02, R7, R16
|
|
BFI $0, R16, $8, R7
|
|
MOVB R7, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_match_emit_encodeSnappyBlockAsm12B:
|
|
ADD R8, R1, R7
|
|
|
|
// genMemMoveShort
|
|
CMP $0x08, R8
|
|
BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_8
|
|
CMP $0x10, R8
|
|
BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_8through16
|
|
CMP $0x20, R8
|
|
BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_8:
|
|
MOVD (R6), R9
|
|
MOVD R9, (R1)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm12B
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_8through16:
|
|
MOVD (R6), R9
|
|
ADD R8, R6, R15
|
|
MOVD -8(R15), R6
|
|
MOVD R9, (R1)
|
|
ADD R8, R1, R15
|
|
MOVD R6, -8(R15)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm12B
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_17through32:
|
|
FMOVQ (R6), F0
|
|
ADD R8, R6, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm12B
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_33through64:
|
|
FMOVQ (R6), F0
|
|
FMOVQ 16(R6), F1
|
|
ADD R8, R6, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R8, R6, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_match_emit_encodeSnappyBlockAsm12B:
|
|
MOVD R7, R1
|
|
JMP emit_literal_done_match_emit_encodeSnappyBlockAsm12B
|
|
|
|
memmove_long_match_emit_encodeSnappyBlockAsm12B:
|
|
ADD R8, R1, R7
|
|
|
|
// genMemMoveLong
|
|
MOVD R6, R9
|
|
MOVD R1, R10
|
|
MOVD R8, R11
|
|
|
|
emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm12Blarge_big_loop_back:
|
|
FMOVQ (R9), F0
|
|
FMOVQ 16(R9), F1
|
|
FMOVQ F0, (R10)
|
|
FMOVQ F1, 16(R10)
|
|
ADD $0x20, R9, R9
|
|
ADD $0x20, R10, R10
|
|
SUB $0x20, R11, R11
|
|
CMP $0x20, R11
|
|
BHS emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm12Blarge_big_loop_back
|
|
ADD R8, R6, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R8, R6, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R8, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R7, R1
|
|
|
|
emit_literal_done_match_emit_encodeSnappyBlockAsm12B:
|
|
match_nolit_loop_encodeSnappyBlockAsm12B:
|
|
MOVWU R2, R6
|
|
SUBW R5, R6, R6
|
|
MOVW R6, 24(RSP)
|
|
ADDW $0x04, R2, R2
|
|
ADDW $0x04, R5, R5
|
|
MOVD src_len+32(FP), R6
|
|
SUBW R2, R6, R6
|
|
ADD R2, R3, R7
|
|
ADD R5, R3, R5
|
|
|
|
// matchLen
|
|
MOVD $0, R9
|
|
|
|
matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm12B:
|
|
CMPW $0x10, R6
|
|
BLO matchlen_match8_match_nolit_encodeSnappyBlockAsm12B
|
|
MOVD (R7)(R9), R8
|
|
ADD R9, R7, R15
|
|
MOVD 8(R15), R10
|
|
MOVD (R5)(R9), R16
|
|
EOR R16, R8, R8
|
|
TST R8, R8
|
|
BNE matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm12B
|
|
ADD R9, R5, R15
|
|
MOVD 8(R15), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_16match_nolit_encodeSnappyBlockAsm12B
|
|
SUB $16, R6, R6
|
|
MOVWU R6, R6
|
|
ADD $16, R9, R9
|
|
MOVWU R9, R9
|
|
JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm12B
|
|
|
|
matchlen_bsf_16match_nolit_encodeSnappyBlockAsm12B:
|
|
RBIT R10, R10
|
|
CLZ R10, R10
|
|
ASR $0x03, R10, R10
|
|
ADD R10, R9, R9
|
|
ADD $8, R9, R9
|
|
MOVWU R9, R9
|
|
JMP match_nolit_end_encodeSnappyBlockAsm12B
|
|
|
|
matchlen_match8_match_nolit_encodeSnappyBlockAsm12B:
|
|
CMPW $0x08, R6
|
|
BLO matchlen_match4_match_nolit_encodeSnappyBlockAsm12B
|
|
MOVD (R7)(R9), R8
|
|
MOVD (R5)(R9), R16
|
|
EOR R16, R8, R8
|
|
TST R8, R8
|
|
BNE matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm12B
|
|
SUB $8, R6, R6
|
|
MOVWU R6, R6
|
|
ADD $8, R9, R9
|
|
MOVWU R9, R9
|
|
JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm12B
|
|
|
|
matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm12B:
|
|
RBIT R8, R8
|
|
CLZ R8, R8
|
|
ASR $0x03, R8, R8
|
|
ADD R8, R9, R9
|
|
MOVWU R9, R9
|
|
JMP match_nolit_end_encodeSnappyBlockAsm12B
|
|
|
|
matchlen_match4_match_nolit_encodeSnappyBlockAsm12B:
|
|
CMPW $0x04, R6
|
|
BLO matchlen_match2_match_nolit_encodeSnappyBlockAsm12B
|
|
MOVWU (R7)(R9), R8
|
|
MOVWU (R5)(R9), R16
|
|
CMPW R8, R16
|
|
BNE matchlen_match2_match_nolit_encodeSnappyBlockAsm12B
|
|
SUB $4, R6, R6
|
|
MOVWU R6, R6
|
|
ADD $4, R9, R9
|
|
MOVWU R9, R9
|
|
|
|
matchlen_match2_match_nolit_encodeSnappyBlockAsm12B:
|
|
CMPW $0x01, R6
|
|
BEQ matchlen_match1_match_nolit_encodeSnappyBlockAsm12B
|
|
BLO match_nolit_end_encodeSnappyBlockAsm12B
|
|
MOVHU (R7)(R9), R16
|
|
BFI $0, R16, $16, R8
|
|
ADD R9, R5, R15
|
|
MOVHU (R15), R15
|
|
AND $0xffff, R8, R16
|
|
CMP R16, R15
|
|
BNE matchlen_match1_match_nolit_encodeSnappyBlockAsm12B
|
|
ADD $2, R9, R9
|
|
MOVWU R9, R9
|
|
SUBSW $0x02, R6, R6
|
|
BEQ match_nolit_end_encodeSnappyBlockAsm12B
|
|
|
|
matchlen_match1_match_nolit_encodeSnappyBlockAsm12B:
|
|
MOVBU (R7)(R9), R16
|
|
BFI $0, R16, $8, R8
|
|
ADD R9, R5, R15
|
|
MOVBU (R15), R15
|
|
AND $0xff, R8, R16
|
|
CMP R16, R15
|
|
BNE match_nolit_end_encodeSnappyBlockAsm12B
|
|
ADD $1, R9, R9
|
|
MOVWU R9, R9
|
|
|
|
match_nolit_end_encodeSnappyBlockAsm12B:
|
|
ADDW R9, R2, R2
|
|
MOVWU 24(RSP), R5
|
|
ADDW $0x04, R9, R9
|
|
MOVW R2, 20(RSP)
|
|
|
|
// emitCopy
|
|
two_byte_offset_match_nolit_encodeSnappyBlockAsm12B:
|
|
CMPW $0x40, R9
|
|
BLS two_byte_offset_short_match_nolit_encodeSnappyBlockAsm12B
|
|
MOVD $0xee, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
SUB $60, R9, R9
|
|
MOVWU R9, R9
|
|
ADD $0x03, R1, R1
|
|
JMP two_byte_offset_match_nolit_encodeSnappyBlockAsm12B
|
|
|
|
two_byte_offset_short_match_nolit_encodeSnappyBlockAsm12B:
|
|
MOVWU R9, R6
|
|
LSLW $0x02, R6, R6
|
|
CMPW $0x0c, R9
|
|
BHS emit_copy_three_match_nolit_encodeSnappyBlockAsm12B
|
|
CMPW $0x00000800, R5
|
|
BHS emit_copy_three_match_nolit_encodeSnappyBlockAsm12B
|
|
SUB $15, R6, R6
|
|
MOVWU R6, R6
|
|
MOVB R5, 1(R1)
|
|
LSRW $0x08, R5, R5
|
|
LSLW $0x05, R5, R5
|
|
ORRW R5, R6, R6
|
|
MOVB R6, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeSnappyBlockAsm12B
|
|
|
|
emit_copy_three_match_nolit_encodeSnappyBlockAsm12B:
|
|
SUB $2, R6, R6
|
|
MOVWU R6, R6
|
|
MOVB R6, (R1)
|
|
MOVH R5, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
|
|
match_nolit_emitcopy_end_encodeSnappyBlockAsm12B:
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R2
|
|
BHS emit_remainder_encodeSnappyBlockAsm12B
|
|
ADD R2, R3, R15
|
|
MOVD -2(R15), R6
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R1
|
|
BLO match_nolit_dst_ok_encodeSnappyBlockAsm12B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeSnappyBlockAsm12B:
|
|
MOVD $0x000000cf1bbcdcbb, R8
|
|
MOVD R6, R7
|
|
LSR $0x10, R6, R6
|
|
MOVD R6, R5
|
|
LSL $0x18, R7, R7
|
|
MUL R8, R7, R7
|
|
LSR $0x34, R7, R7
|
|
LSL $0x18, R5, R5
|
|
MUL R8, R5, R5
|
|
LSR $0x34, R5, R5
|
|
SUB $2, R2, R8
|
|
MOVWU R8, R8
|
|
ADD R5<<2, R0, R9
|
|
MOVWU (R9), R5
|
|
MOVW R8, (R0)(R7<<2)
|
|
MOVW R2, (R9)
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ match_nolit_loop_encodeSnappyBlockAsm12B
|
|
ADDW $1, R2, R2
|
|
JMP search_loop_encodeSnappyBlockAsm12B
|
|
|
|
emit_remainder_encodeSnappyBlockAsm12B:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R0, R0
|
|
ADD R0, R1, R0
|
|
ADD $3, R0, R0
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R0
|
|
BLO emit_remainder_ok_encodeSnappyBlockAsm12B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeSnappyBlockAsm12B:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R2
|
|
CMPW R0, R2
|
|
BEQ emit_literal_done_emit_remainder_encodeSnappyBlockAsm12B
|
|
MOVWU R0, R5
|
|
MOVW R0, 20(RSP)
|
|
ADD R2, R3, R0
|
|
SUBW R2, R5, R5
|
|
SUB $1, R5, R2
|
|
MOVWU R2, R2
|
|
CMPW $0x3c, R2
|
|
BLO one_byte_emit_remainder_encodeSnappyBlockAsm12B
|
|
CMPW $0x00000100, R2
|
|
BLO two_bytes_emit_remainder_encodeSnappyBlockAsm12B
|
|
BLO three_bytes_emit_remainder_encodeSnappyBlockAsm12B
|
|
|
|
three_bytes_emit_remainder_encodeSnappyBlockAsm12B:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R2, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_emit_remainder_encodeSnappyBlockAsm12B
|
|
|
|
two_bytes_emit_remainder_encodeSnappyBlockAsm12B:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R2, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R2
|
|
BLO memmove_emit_remainder_encodeSnappyBlockAsm12B
|
|
JMP memmove_long_emit_remainder_encodeSnappyBlockAsm12B
|
|
|
|
one_byte_emit_remainder_encodeSnappyBlockAsm12B:
|
|
LSLW $0x02, R2, R16
|
|
BFI $0, R16, $8, R2
|
|
MOVB R2, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_emit_remainder_encodeSnappyBlockAsm12B:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveShort
|
|
CMP $0x03, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_1or2
|
|
BEQ emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_3
|
|
CMP $0x08, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_4through7
|
|
CMP $0x10, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_8through16
|
|
CMP $0x20, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_1or2:
|
|
MOVBU (R0), R16
|
|
BFI $0, R16, $8, R5
|
|
ADD R3, R0, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVB R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVB R0, -1(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_3:
|
|
MOVHU (R0), R16
|
|
BFI $0, R16, $16, R5
|
|
MOVBU 2(R0), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVH R5, (R1)
|
|
MOVB R0, 2(R1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_4through7:
|
|
MOVWU (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVWU -4(R15), R0
|
|
MOVW R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVW R0, -4(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_8through16:
|
|
MOVD (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVD -8(R15), R0
|
|
MOVD R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVD R0, -8(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_17through32:
|
|
FMOVQ (R0), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_33through64:
|
|
FMOVQ (R0), F0
|
|
FMOVQ 16(R0), F1
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_emit_remainder_encodeSnappyBlockAsm12B:
|
|
MOVD R2, R1
|
|
JMP emit_literal_done_emit_remainder_encodeSnappyBlockAsm12B
|
|
|
|
memmove_long_emit_remainder_encodeSnappyBlockAsm12B:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveLong
|
|
MOVD R0, R5
|
|
MOVD R1, R6
|
|
MOVD R3, R7
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm12Blarge_big_loop_back:
|
|
FMOVQ (R5), F0
|
|
FMOVQ 16(R5), F1
|
|
FMOVQ F0, (R6)
|
|
FMOVQ F1, 16(R6)
|
|
ADD $0x20, R5, R5
|
|
ADD $0x20, R6, R6
|
|
SUB $0x20, R7, R7
|
|
CMP $0x20, R7
|
|
BHS emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm12Blarge_big_loop_back
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R3, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R2, R1
|
|
|
|
emit_literal_done_emit_remainder_encodeSnappyBlockAsm12B:
|
|
MOVD dst_base+0(FP), R0
|
|
SUB R0, R1, R1
|
|
MOVD R1, ret+56(FP)
|
|
RET
|
|
|
|
// func encodeSnappyBlockAsm10B(dst []byte, src []byte, tmp *[4096]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeSnappyBlockAsm10B(SB), $24-64
|
|
MOVD tmp+48(FP), R0
|
|
MOVD dst_base+0(FP), R1
|
|
MOVD $0x00000020, R2
|
|
MOVD R0, R3
|
|
VEOR V0.B16, V0.B16, V0.B16
|
|
|
|
zero_loop_encodeSnappyBlockAsm10B:
|
|
FMOVQ F0, (R3)
|
|
FMOVQ F0, 16(R3)
|
|
FMOVQ F0, 32(R3)
|
|
FMOVQ F0, 48(R3)
|
|
FMOVQ F0, 64(R3)
|
|
FMOVQ F0, 80(R3)
|
|
FMOVQ F0, 96(R3)
|
|
FMOVQ F0, 112(R3)
|
|
ADD $0x80, R3, R3
|
|
SUBS $1, R2, R2
|
|
BNE zero_loop_encodeSnappyBlockAsm10B
|
|
MOVD $0x00000000, R16
|
|
MOVW R16, 20(RSP)
|
|
MOVD src_len+32(FP), R2
|
|
SUB $9, R2, R3
|
|
SUB $8, R2, R5
|
|
MOVW R5, 16(RSP)
|
|
LSR $0x05, R2, R2
|
|
SUBW R2, R3, R3
|
|
ADD R3, R1, R3
|
|
MOVD R3, 8(RSP)
|
|
MOVD $0x00000001, R2
|
|
MOVW R2, 24(RSP)
|
|
MOVD src_base+24(FP), R3
|
|
|
|
search_loop_encodeSnappyBlockAsm10B:
|
|
MOVWU R2, R5
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R5, R5
|
|
LSRW $0x05, R5, R5
|
|
ADD R5, R2, R5
|
|
ADD $4, R5, R5
|
|
MOVWU R5, R5
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R5
|
|
BHS emit_remainder_encodeSnappyBlockAsm10B
|
|
MOVD (R3)(R2), R6
|
|
MOVW R5, 28(RSP)
|
|
MOVD $0x9e3779b1, R8
|
|
MOVD R6, R9
|
|
MOVD R6, R10
|
|
LSR $0x08, R10, R10
|
|
LSL $0x20, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x36, R9, R9
|
|
LSL $0x20, R10, R10
|
|
MUL R8, R10, R10
|
|
LSR $0x36, R10, R10
|
|
MOVWU (R0)(R9<<2), R5
|
|
MOVWU (R0)(R10<<2), R7
|
|
MOVW R2, (R0)(R9<<2)
|
|
ADD $1, R2, R9
|
|
MOVWU R9, R9
|
|
MOVW R9, (R0)(R10<<2)
|
|
MOVD R6, R9
|
|
LSR $0x10, R9, R9
|
|
LSL $0x20, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x36, R9, R9
|
|
MOVWU R2, R8
|
|
MOVWU 24(RSP), R16
|
|
SUBW R16, R8, R8
|
|
ADD R8, R3, R15
|
|
MOVWU 1(R15), R10
|
|
MOVD R6, R8
|
|
LSR $0x08, R8, R8
|
|
CMPW R10, R8
|
|
BNE no_repeat_found_encodeSnappyBlockAsm10B
|
|
ADD $1, R2, R6
|
|
MOVWU R6, R6
|
|
MOVWU 20(RSP), R5
|
|
MOVWU R6, R7
|
|
MOVWU 24(RSP), R16
|
|
SUBSW R16, R7, R7
|
|
BEQ repeat_extend_back_end_encodeSnappyBlockAsm10B
|
|
|
|
repeat_extend_back_loop_encodeSnappyBlockAsm10B:
|
|
CMPW R5, R6
|
|
BLS repeat_extend_back_end_encodeSnappyBlockAsm10B
|
|
ADD R7, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R8
|
|
ADD R6, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R9
|
|
AND $0xff, R9, R16
|
|
AND $0xff, R8, R15
|
|
CMP R16, R15
|
|
BNE repeat_extend_back_end_encodeSnappyBlockAsm10B
|
|
SUB $1, R6, R6
|
|
MOVWU R6, R6
|
|
SUBSW $1, R7, R7
|
|
BNE repeat_extend_back_loop_encodeSnappyBlockAsm10B
|
|
|
|
repeat_extend_back_end_encodeSnappyBlockAsm10B:
|
|
MOVWU R6, R5
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R5, R5
|
|
ADD R5, R1, R5
|
|
ADD $3, R5, R5
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R5
|
|
BLO repeat_dst_size_check_encodeSnappyBlockAsm10B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
repeat_dst_size_check_encodeSnappyBlockAsm10B:
|
|
MOVWU 20(RSP), R5
|
|
CMPW R6, R5
|
|
BEQ emit_literal_done_repeat_emit_encodeSnappyBlockAsm10B
|
|
MOVWU R6, R7
|
|
MOVW R6, 20(RSP)
|
|
ADD R5, R3, R8
|
|
SUBW R5, R7, R7
|
|
SUB $1, R7, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x3c, R5
|
|
BLO one_byte_repeat_emit_encodeSnappyBlockAsm10B
|
|
CMPW $0x00000100, R5
|
|
BLO two_bytes_repeat_emit_encodeSnappyBlockAsm10B
|
|
BLO three_bytes_repeat_emit_encodeSnappyBlockAsm10B
|
|
|
|
three_bytes_repeat_emit_encodeSnappyBlockAsm10B:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_repeat_emit_encodeSnappyBlockAsm10B
|
|
|
|
two_bytes_repeat_emit_encodeSnappyBlockAsm10B:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R5, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R5
|
|
BLO memmove_repeat_emit_encodeSnappyBlockAsm10B
|
|
JMP memmove_long_repeat_emit_encodeSnappyBlockAsm10B
|
|
|
|
one_byte_repeat_emit_encodeSnappyBlockAsm10B:
|
|
LSLW $0x02, R5, R16
|
|
BFI $0, R16, $8, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_repeat_emit_encodeSnappyBlockAsm10B:
|
|
ADD R7, R1, R5
|
|
|
|
// genMemMoveShort
|
|
CMP $0x08, R7
|
|
BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_8
|
|
CMP $0x10, R7
|
|
BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_8through16
|
|
CMP $0x20, R7
|
|
BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_17through32
|
|
JMP emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_8:
|
|
MOVD (R8), R9
|
|
MOVD R9, (R1)
|
|
JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm10B
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_8through16:
|
|
MOVD (R8), R9
|
|
ADD R7, R8, R15
|
|
MOVD -8(R15), R8
|
|
MOVD R9, (R1)
|
|
ADD R7, R1, R15
|
|
MOVD R8, -8(R15)
|
|
JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm10B
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_17through32:
|
|
FMOVQ (R8), F0
|
|
ADD R7, R8, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R7, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm10B
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_33through64:
|
|
FMOVQ (R8), F0
|
|
FMOVQ 16(R8), F1
|
|
ADD R7, R8, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R7, R8, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R7, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R7, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_repeat_emit_encodeSnappyBlockAsm10B:
|
|
MOVD R5, R1
|
|
JMP emit_literal_done_repeat_emit_encodeSnappyBlockAsm10B
|
|
|
|
memmove_long_repeat_emit_encodeSnappyBlockAsm10B:
|
|
ADD R7, R1, R5
|
|
|
|
// genMemMoveLong
|
|
MOVD R8, R9
|
|
MOVD R1, R10
|
|
MOVD R7, R11
|
|
|
|
emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm10Blarge_big_loop_back:
|
|
FMOVQ (R9), F0
|
|
FMOVQ 16(R9), F1
|
|
FMOVQ F0, (R10)
|
|
FMOVQ F1, 16(R10)
|
|
ADD $0x20, R9, R9
|
|
ADD $0x20, R10, R10
|
|
SUB $0x20, R11, R11
|
|
CMP $0x20, R11
|
|
BHS emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm10Blarge_big_loop_back
|
|
ADD R7, R8, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R7, R8, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R7, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R7, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R5, R1
|
|
|
|
emit_literal_done_repeat_emit_encodeSnappyBlockAsm10B:
|
|
ADDW $0x05, R2, R2
|
|
MOVWU R2, R5
|
|
MOVWU 24(RSP), R16
|
|
SUBW R16, R5, R5
|
|
MOVD src_len+32(FP), R7
|
|
SUBW R2, R7, R7
|
|
ADD R2, R3, R8
|
|
ADD R5, R3, R5
|
|
|
|
// matchLen
|
|
MOVD $0, R10
|
|
|
|
matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm10B:
|
|
CMPW $0x10, R7
|
|
BLO matchlen_match8_repeat_extend_encodeSnappyBlockAsm10B
|
|
MOVD (R8)(R10), R9
|
|
ADD R10, R8, R15
|
|
MOVD 8(R15), R11
|
|
MOVD (R5)(R10), R16
|
|
EOR R16, R9, R9
|
|
TST R9, R9
|
|
BNE matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm10B
|
|
ADD R10, R5, R15
|
|
MOVD 8(R15), R16
|
|
EOR R16, R11, R11
|
|
TST R11, R11
|
|
BNE matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm10B
|
|
SUB $16, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $16, R10, R10
|
|
MOVWU R10, R10
|
|
JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm10B
|
|
|
|
matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm10B:
|
|
RBIT R11, R11
|
|
CLZ R11, R11
|
|
ASR $0x03, R11, R11
|
|
ADD R11, R10, R10
|
|
ADD $8, R10, R10
|
|
MOVWU R10, R10
|
|
JMP repeat_extend_forward_end_encodeSnappyBlockAsm10B
|
|
|
|
matchlen_match8_repeat_extend_encodeSnappyBlockAsm10B:
|
|
CMPW $0x08, R7
|
|
BLO matchlen_match4_repeat_extend_encodeSnappyBlockAsm10B
|
|
MOVD (R8)(R10), R9
|
|
MOVD (R5)(R10), R16
|
|
EOR R16, R9, R9
|
|
TST R9, R9
|
|
BNE matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm10B
|
|
SUB $8, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $8, R10, R10
|
|
MOVWU R10, R10
|
|
JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm10B
|
|
|
|
matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm10B:
|
|
RBIT R9, R9
|
|
CLZ R9, R9
|
|
ASR $0x03, R9, R9
|
|
ADD R9, R10, R10
|
|
MOVWU R10, R10
|
|
JMP repeat_extend_forward_end_encodeSnappyBlockAsm10B
|
|
|
|
matchlen_match4_repeat_extend_encodeSnappyBlockAsm10B:
|
|
CMPW $0x04, R7
|
|
BLO matchlen_match2_repeat_extend_encodeSnappyBlockAsm10B
|
|
MOVWU (R8)(R10), R9
|
|
MOVWU (R5)(R10), R16
|
|
CMPW R9, R16
|
|
BNE matchlen_match2_repeat_extend_encodeSnappyBlockAsm10B
|
|
SUB $4, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $4, R10, R10
|
|
MOVWU R10, R10
|
|
|
|
matchlen_match2_repeat_extend_encodeSnappyBlockAsm10B:
|
|
CMPW $0x01, R7
|
|
BEQ matchlen_match1_repeat_extend_encodeSnappyBlockAsm10B
|
|
BLO repeat_extend_forward_end_encodeSnappyBlockAsm10B
|
|
MOVHU (R8)(R10), R16
|
|
BFI $0, R16, $16, R9
|
|
ADD R10, R5, R15
|
|
MOVHU (R15), R15
|
|
AND $0xffff, R9, R16
|
|
CMP R16, R15
|
|
BNE matchlen_match1_repeat_extend_encodeSnappyBlockAsm10B
|
|
ADD $2, R10, R10
|
|
MOVWU R10, R10
|
|
SUBSW $0x02, R7, R7
|
|
BEQ repeat_extend_forward_end_encodeSnappyBlockAsm10B
|
|
|
|
matchlen_match1_repeat_extend_encodeSnappyBlockAsm10B:
|
|
MOVBU (R8)(R10), R16
|
|
BFI $0, R16, $8, R9
|
|
ADD R10, R5, R15
|
|
MOVBU (R15), R15
|
|
AND $0xff, R9, R16
|
|
CMP R16, R15
|
|
BNE repeat_extend_forward_end_encodeSnappyBlockAsm10B
|
|
ADD $1, R10, R10
|
|
MOVWU R10, R10
|
|
|
|
repeat_extend_forward_end_encodeSnappyBlockAsm10B:
|
|
ADDW R10, R2, R2
|
|
MOVWU R2, R5
|
|
SUBW R6, R5, R5
|
|
MOVWU 24(RSP), R6
|
|
|
|
// emitCopy
|
|
two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm10B:
|
|
CMPW $0x40, R5
|
|
BLS two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm10B
|
|
MOVD $0xee, R16
|
|
MOVB R16, (R1)
|
|
MOVH R6, 1(R1)
|
|
SUB $60, R5, R5
|
|
MOVWU R5, R5
|
|
ADD $0x03, R1, R1
|
|
JMP two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm10B
|
|
|
|
two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm10B:
|
|
MOVWU R5, R7
|
|
LSLW $0x02, R7, R7
|
|
CMPW $0x0c, R5
|
|
BHS emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm10B
|
|
CMPW $0x00000800, R6
|
|
BHS emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm10B
|
|
SUB $15, R7, R7
|
|
MOVWU R7, R7
|
|
MOVB R6, 1(R1)
|
|
LSRW $0x08, R6, R6
|
|
LSLW $0x05, R6, R6
|
|
ORRW R6, R7, R7
|
|
MOVB R7, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeSnappyBlockAsm10B
|
|
|
|
emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm10B:
|
|
SUB $2, R7, R7
|
|
MOVWU R7, R7
|
|
MOVB R7, (R1)
|
|
MOVH R6, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
|
|
repeat_end_emit_encodeSnappyBlockAsm10B:
|
|
MOVW R2, 20(RSP)
|
|
JMP search_loop_encodeSnappyBlockAsm10B
|
|
|
|
no_repeat_found_encodeSnappyBlockAsm10B:
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ candidate_match_encodeSnappyBlockAsm10B
|
|
LSR $0x08, R6, R6
|
|
MOVWU (R0)(R9<<2), R5
|
|
ADD $2, R2, R8
|
|
MOVWU R8, R8
|
|
MOVWU (R3)(R7), R16
|
|
CMPW R6, R16
|
|
BEQ candidate2_match_encodeSnappyBlockAsm10B
|
|
MOVW R8, (R0)(R9<<2)
|
|
LSR $0x08, R6, R6
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ candidate3_match_encodeSnappyBlockAsm10B
|
|
MOVWU 28(RSP), R2
|
|
JMP search_loop_encodeSnappyBlockAsm10B
|
|
|
|
candidate3_match_encodeSnappyBlockAsm10B:
|
|
ADDW $0x02, R2, R2
|
|
JMP candidate_match_encodeSnappyBlockAsm10B
|
|
|
|
candidate2_match_encodeSnappyBlockAsm10B:
|
|
MOVW R8, (R0)(R9<<2)
|
|
ADDW $1, R2, R2
|
|
MOVWU R7, R5
|
|
|
|
candidate_match_encodeSnappyBlockAsm10B:
|
|
MOVWU 20(RSP), R6
|
|
TSTW R5, R5
|
|
BEQ match_extend_back_end_encodeSnappyBlockAsm10B
|
|
|
|
match_extend_back_loop_encodeSnappyBlockAsm10B:
|
|
CMPW R6, R2
|
|
BLS match_extend_back_end_encodeSnappyBlockAsm10B
|
|
ADD R5, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R7
|
|
ADD R2, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R8
|
|
AND $0xff, R8, R16
|
|
AND $0xff, R7, R15
|
|
CMP R16, R15
|
|
BNE match_extend_back_end_encodeSnappyBlockAsm10B
|
|
SUB $1, R2, R2
|
|
MOVWU R2, R2
|
|
SUBSW $1, R5, R5
|
|
BEQ match_extend_back_end_encodeSnappyBlockAsm10B
|
|
JMP match_extend_back_loop_encodeSnappyBlockAsm10B
|
|
|
|
match_extend_back_end_encodeSnappyBlockAsm10B:
|
|
MOVWU R2, R6
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R6, R6
|
|
ADD R6, R1, R6
|
|
ADD $3, R6, R6
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R6
|
|
BLO match_dst_size_check_encodeSnappyBlockAsm10B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeSnappyBlockAsm10B:
|
|
MOVWU R2, R6
|
|
MOVWU 20(RSP), R7
|
|
CMPW R6, R7
|
|
BEQ emit_literal_done_match_emit_encodeSnappyBlockAsm10B
|
|
MOVWU R6, R8
|
|
MOVW R6, 20(RSP)
|
|
ADD R7, R3, R6
|
|
SUBW R7, R8, R8
|
|
SUB $1, R8, R7
|
|
MOVWU R7, R7
|
|
CMPW $0x3c, R7
|
|
BLO one_byte_match_emit_encodeSnappyBlockAsm10B
|
|
CMPW $0x00000100, R7
|
|
BLO two_bytes_match_emit_encodeSnappyBlockAsm10B
|
|
BLO three_bytes_match_emit_encodeSnappyBlockAsm10B
|
|
|
|
three_bytes_match_emit_encodeSnappyBlockAsm10B:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R7, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_match_emit_encodeSnappyBlockAsm10B
|
|
|
|
two_bytes_match_emit_encodeSnappyBlockAsm10B:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R7, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R7
|
|
BLO memmove_match_emit_encodeSnappyBlockAsm10B
|
|
JMP memmove_long_match_emit_encodeSnappyBlockAsm10B
|
|
|
|
one_byte_match_emit_encodeSnappyBlockAsm10B:
|
|
LSLW $0x02, R7, R16
|
|
BFI $0, R16, $8, R7
|
|
MOVB R7, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_match_emit_encodeSnappyBlockAsm10B:
|
|
ADD R8, R1, R7
|
|
|
|
// genMemMoveShort
|
|
CMP $0x08, R8
|
|
BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_8
|
|
CMP $0x10, R8
|
|
BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_8through16
|
|
CMP $0x20, R8
|
|
BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_8:
|
|
MOVD (R6), R9
|
|
MOVD R9, (R1)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm10B
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_8through16:
|
|
MOVD (R6), R9
|
|
ADD R8, R6, R15
|
|
MOVD -8(R15), R6
|
|
MOVD R9, (R1)
|
|
ADD R8, R1, R15
|
|
MOVD R6, -8(R15)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm10B
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_17through32:
|
|
FMOVQ (R6), F0
|
|
ADD R8, R6, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm10B
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_33through64:
|
|
FMOVQ (R6), F0
|
|
FMOVQ 16(R6), F1
|
|
ADD R8, R6, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R8, R6, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_match_emit_encodeSnappyBlockAsm10B:
|
|
MOVD R7, R1
|
|
JMP emit_literal_done_match_emit_encodeSnappyBlockAsm10B
|
|
|
|
memmove_long_match_emit_encodeSnappyBlockAsm10B:
|
|
ADD R8, R1, R7
|
|
|
|
// genMemMoveLong
|
|
MOVD R6, R9
|
|
MOVD R1, R10
|
|
MOVD R8, R11
|
|
|
|
emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm10Blarge_big_loop_back:
|
|
FMOVQ (R9), F0
|
|
FMOVQ 16(R9), F1
|
|
FMOVQ F0, (R10)
|
|
FMOVQ F1, 16(R10)
|
|
ADD $0x20, R9, R9
|
|
ADD $0x20, R10, R10
|
|
SUB $0x20, R11, R11
|
|
CMP $0x20, R11
|
|
BHS emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm10Blarge_big_loop_back
|
|
ADD R8, R6, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R8, R6, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R8, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R7, R1
|
|
|
|
emit_literal_done_match_emit_encodeSnappyBlockAsm10B:
|
|
match_nolit_loop_encodeSnappyBlockAsm10B:
|
|
MOVWU R2, R6
|
|
SUBW R5, R6, R6
|
|
MOVW R6, 24(RSP)
|
|
ADDW $0x04, R2, R2
|
|
ADDW $0x04, R5, R5
|
|
MOVD src_len+32(FP), R6
|
|
SUBW R2, R6, R6
|
|
ADD R2, R3, R7
|
|
ADD R5, R3, R5
|
|
|
|
// matchLen
|
|
MOVD $0, R9
|
|
|
|
matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm10B:
|
|
CMPW $0x10, R6
|
|
BLO matchlen_match8_match_nolit_encodeSnappyBlockAsm10B
|
|
MOVD (R7)(R9), R8
|
|
ADD R9, R7, R15
|
|
MOVD 8(R15), R10
|
|
MOVD (R5)(R9), R16
|
|
EOR R16, R8, R8
|
|
TST R8, R8
|
|
BNE matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm10B
|
|
ADD R9, R5, R15
|
|
MOVD 8(R15), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_16match_nolit_encodeSnappyBlockAsm10B
|
|
SUB $16, R6, R6
|
|
MOVWU R6, R6
|
|
ADD $16, R9, R9
|
|
MOVWU R9, R9
|
|
JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm10B
|
|
|
|
matchlen_bsf_16match_nolit_encodeSnappyBlockAsm10B:
|
|
RBIT R10, R10
|
|
CLZ R10, R10
|
|
ASR $0x03, R10, R10
|
|
ADD R10, R9, R9
|
|
ADD $8, R9, R9
|
|
MOVWU R9, R9
|
|
JMP match_nolit_end_encodeSnappyBlockAsm10B
|
|
|
|
matchlen_match8_match_nolit_encodeSnappyBlockAsm10B:
|
|
CMPW $0x08, R6
|
|
BLO matchlen_match4_match_nolit_encodeSnappyBlockAsm10B
|
|
MOVD (R7)(R9), R8
|
|
MOVD (R5)(R9), R16
|
|
EOR R16, R8, R8
|
|
TST R8, R8
|
|
BNE matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm10B
|
|
SUB $8, R6, R6
|
|
MOVWU R6, R6
|
|
ADD $8, R9, R9
|
|
MOVWU R9, R9
|
|
JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm10B
|
|
|
|
matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm10B:
|
|
RBIT R8, R8
|
|
CLZ R8, R8
|
|
ASR $0x03, R8, R8
|
|
ADD R8, R9, R9
|
|
MOVWU R9, R9
|
|
JMP match_nolit_end_encodeSnappyBlockAsm10B
|
|
|
|
matchlen_match4_match_nolit_encodeSnappyBlockAsm10B:
|
|
CMPW $0x04, R6
|
|
BLO matchlen_match2_match_nolit_encodeSnappyBlockAsm10B
|
|
MOVWU (R7)(R9), R8
|
|
MOVWU (R5)(R9), R16
|
|
CMPW R8, R16
|
|
BNE matchlen_match2_match_nolit_encodeSnappyBlockAsm10B
|
|
SUB $4, R6, R6
|
|
MOVWU R6, R6
|
|
ADD $4, R9, R9
|
|
MOVWU R9, R9
|
|
|
|
matchlen_match2_match_nolit_encodeSnappyBlockAsm10B:
|
|
CMPW $0x01, R6
|
|
BEQ matchlen_match1_match_nolit_encodeSnappyBlockAsm10B
|
|
BLO match_nolit_end_encodeSnappyBlockAsm10B
|
|
MOVHU (R7)(R9), R16
|
|
BFI $0, R16, $16, R8
|
|
ADD R9, R5, R15
|
|
MOVHU (R15), R15
|
|
AND $0xffff, R8, R16
|
|
CMP R16, R15
|
|
BNE matchlen_match1_match_nolit_encodeSnappyBlockAsm10B
|
|
ADD $2, R9, R9
|
|
MOVWU R9, R9
|
|
SUBSW $0x02, R6, R6
|
|
BEQ match_nolit_end_encodeSnappyBlockAsm10B
|
|
|
|
matchlen_match1_match_nolit_encodeSnappyBlockAsm10B:
|
|
MOVBU (R7)(R9), R16
|
|
BFI $0, R16, $8, R8
|
|
ADD R9, R5, R15
|
|
MOVBU (R15), R15
|
|
AND $0xff, R8, R16
|
|
CMP R16, R15
|
|
BNE match_nolit_end_encodeSnappyBlockAsm10B
|
|
ADD $1, R9, R9
|
|
MOVWU R9, R9
|
|
|
|
match_nolit_end_encodeSnappyBlockAsm10B:
|
|
ADDW R9, R2, R2
|
|
MOVWU 24(RSP), R5
|
|
ADDW $0x04, R9, R9
|
|
MOVW R2, 20(RSP)
|
|
|
|
// emitCopy
|
|
two_byte_offset_match_nolit_encodeSnappyBlockAsm10B:
|
|
CMPW $0x40, R9
|
|
BLS two_byte_offset_short_match_nolit_encodeSnappyBlockAsm10B
|
|
MOVD $0xee, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
SUB $60, R9, R9
|
|
MOVWU R9, R9
|
|
ADD $0x03, R1, R1
|
|
JMP two_byte_offset_match_nolit_encodeSnappyBlockAsm10B
|
|
|
|
two_byte_offset_short_match_nolit_encodeSnappyBlockAsm10B:
|
|
MOVWU R9, R6
|
|
LSLW $0x02, R6, R6
|
|
CMPW $0x0c, R9
|
|
BHS emit_copy_three_match_nolit_encodeSnappyBlockAsm10B
|
|
CMPW $0x00000800, R5
|
|
BHS emit_copy_three_match_nolit_encodeSnappyBlockAsm10B
|
|
SUB $15, R6, R6
|
|
MOVWU R6, R6
|
|
MOVB R5, 1(R1)
|
|
LSRW $0x08, R5, R5
|
|
LSLW $0x05, R5, R5
|
|
ORRW R5, R6, R6
|
|
MOVB R6, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeSnappyBlockAsm10B
|
|
|
|
emit_copy_three_match_nolit_encodeSnappyBlockAsm10B:
|
|
SUB $2, R6, R6
|
|
MOVWU R6, R6
|
|
MOVB R6, (R1)
|
|
MOVH R5, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
|
|
match_nolit_emitcopy_end_encodeSnappyBlockAsm10B:
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R2
|
|
BHS emit_remainder_encodeSnappyBlockAsm10B
|
|
ADD R2, R3, R15
|
|
MOVD -2(R15), R6
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R1
|
|
BLO match_nolit_dst_ok_encodeSnappyBlockAsm10B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeSnappyBlockAsm10B:
|
|
MOVD $0x9e3779b1, R8
|
|
MOVD R6, R7
|
|
LSR $0x10, R6, R6
|
|
MOVD R6, R5
|
|
LSL $0x20, R7, R7
|
|
MUL R8, R7, R7
|
|
LSR $0x36, R7, R7
|
|
LSL $0x20, R5, R5
|
|
MUL R8, R5, R5
|
|
LSR $0x36, R5, R5
|
|
SUB $2, R2, R8
|
|
MOVWU R8, R8
|
|
ADD R5<<2, R0, R9
|
|
MOVWU (R9), R5
|
|
MOVW R8, (R0)(R7<<2)
|
|
MOVW R2, (R9)
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ match_nolit_loop_encodeSnappyBlockAsm10B
|
|
ADDW $1, R2, R2
|
|
JMP search_loop_encodeSnappyBlockAsm10B
|
|
|
|
emit_remainder_encodeSnappyBlockAsm10B:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R0, R0
|
|
ADD R0, R1, R0
|
|
ADD $3, R0, R0
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R0
|
|
BLO emit_remainder_ok_encodeSnappyBlockAsm10B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeSnappyBlockAsm10B:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R2
|
|
CMPW R0, R2
|
|
BEQ emit_literal_done_emit_remainder_encodeSnappyBlockAsm10B
|
|
MOVWU R0, R5
|
|
MOVW R0, 20(RSP)
|
|
ADD R2, R3, R0
|
|
SUBW R2, R5, R5
|
|
SUB $1, R5, R2
|
|
MOVWU R2, R2
|
|
CMPW $0x3c, R2
|
|
BLO one_byte_emit_remainder_encodeSnappyBlockAsm10B
|
|
CMPW $0x00000100, R2
|
|
BLO two_bytes_emit_remainder_encodeSnappyBlockAsm10B
|
|
BLO three_bytes_emit_remainder_encodeSnappyBlockAsm10B
|
|
|
|
three_bytes_emit_remainder_encodeSnappyBlockAsm10B:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R2, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_emit_remainder_encodeSnappyBlockAsm10B
|
|
|
|
two_bytes_emit_remainder_encodeSnappyBlockAsm10B:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R2, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R2
|
|
BLO memmove_emit_remainder_encodeSnappyBlockAsm10B
|
|
JMP memmove_long_emit_remainder_encodeSnappyBlockAsm10B
|
|
|
|
one_byte_emit_remainder_encodeSnappyBlockAsm10B:
|
|
LSLW $0x02, R2, R16
|
|
BFI $0, R16, $8, R2
|
|
MOVB R2, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_emit_remainder_encodeSnappyBlockAsm10B:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveShort
|
|
CMP $0x03, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_1or2
|
|
BEQ emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_3
|
|
CMP $0x08, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_4through7
|
|
CMP $0x10, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_8through16
|
|
CMP $0x20, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_1or2:
|
|
MOVBU (R0), R16
|
|
BFI $0, R16, $8, R5
|
|
ADD R3, R0, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVB R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVB R0, -1(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_3:
|
|
MOVHU (R0), R16
|
|
BFI $0, R16, $16, R5
|
|
MOVBU 2(R0), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVH R5, (R1)
|
|
MOVB R0, 2(R1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_4through7:
|
|
MOVWU (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVWU -4(R15), R0
|
|
MOVW R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVW R0, -4(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_8through16:
|
|
MOVD (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVD -8(R15), R0
|
|
MOVD R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVD R0, -8(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_17through32:
|
|
FMOVQ (R0), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_33through64:
|
|
FMOVQ (R0), F0
|
|
FMOVQ 16(R0), F1
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_emit_remainder_encodeSnappyBlockAsm10B:
|
|
MOVD R2, R1
|
|
JMP emit_literal_done_emit_remainder_encodeSnappyBlockAsm10B
|
|
|
|
memmove_long_emit_remainder_encodeSnappyBlockAsm10B:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveLong
|
|
MOVD R0, R5
|
|
MOVD R1, R6
|
|
MOVD R3, R7
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm10Blarge_big_loop_back:
|
|
FMOVQ (R5), F0
|
|
FMOVQ 16(R5), F1
|
|
FMOVQ F0, (R6)
|
|
FMOVQ F1, 16(R6)
|
|
ADD $0x20, R5, R5
|
|
ADD $0x20, R6, R6
|
|
SUB $0x20, R7, R7
|
|
CMP $0x20, R7
|
|
BHS emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm10Blarge_big_loop_back
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R3, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R2, R1
|
|
|
|
emit_literal_done_emit_remainder_encodeSnappyBlockAsm10B:
|
|
MOVD dst_base+0(FP), R0
|
|
SUB R0, R1, R1
|
|
MOVD R1, ret+56(FP)
|
|
RET
|
|
|
|
// func encodeSnappyBlockAsm8B(dst []byte, src []byte, tmp *[1024]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeSnappyBlockAsm8B(SB), $24-64
|
|
MOVD tmp+48(FP), R0
|
|
MOVD dst_base+0(FP), R1
|
|
MOVD $0x00000008, R2
|
|
MOVD R0, R3
|
|
VEOR V0.B16, V0.B16, V0.B16
|
|
|
|
zero_loop_encodeSnappyBlockAsm8B:
|
|
FMOVQ F0, (R3)
|
|
FMOVQ F0, 16(R3)
|
|
FMOVQ F0, 32(R3)
|
|
FMOVQ F0, 48(R3)
|
|
FMOVQ F0, 64(R3)
|
|
FMOVQ F0, 80(R3)
|
|
FMOVQ F0, 96(R3)
|
|
FMOVQ F0, 112(R3)
|
|
ADD $0x80, R3, R3
|
|
SUBS $1, R2, R2
|
|
BNE zero_loop_encodeSnappyBlockAsm8B
|
|
MOVD $0x00000000, R16
|
|
MOVW R16, 20(RSP)
|
|
MOVD src_len+32(FP), R2
|
|
SUB $9, R2, R3
|
|
SUB $8, R2, R5
|
|
MOVW R5, 16(RSP)
|
|
LSR $0x05, R2, R2
|
|
SUBW R2, R3, R3
|
|
ADD R3, R1, R3
|
|
MOVD R3, 8(RSP)
|
|
MOVD $0x00000001, R2
|
|
MOVW R2, 24(RSP)
|
|
MOVD src_base+24(FP), R3
|
|
|
|
search_loop_encodeSnappyBlockAsm8B:
|
|
MOVWU R2, R5
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R5, R5
|
|
LSRW $0x04, R5, R5
|
|
ADD R5, R2, R5
|
|
ADD $4, R5, R5
|
|
MOVWU R5, R5
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R5
|
|
BHS emit_remainder_encodeSnappyBlockAsm8B
|
|
MOVD (R3)(R2), R6
|
|
MOVW R5, 28(RSP)
|
|
MOVD $0x9e3779b1, R8
|
|
MOVD R6, R9
|
|
MOVD R6, R10
|
|
LSR $0x08, R10, R10
|
|
LSL $0x20, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x38, R9, R9
|
|
LSL $0x20, R10, R10
|
|
MUL R8, R10, R10
|
|
LSR $0x38, R10, R10
|
|
MOVWU (R0)(R9<<2), R5
|
|
MOVWU (R0)(R10<<2), R7
|
|
MOVW R2, (R0)(R9<<2)
|
|
ADD $1, R2, R9
|
|
MOVWU R9, R9
|
|
MOVW R9, (R0)(R10<<2)
|
|
MOVD R6, R9
|
|
LSR $0x10, R9, R9
|
|
LSL $0x20, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x38, R9, R9
|
|
MOVWU R2, R8
|
|
MOVWU 24(RSP), R16
|
|
SUBW R16, R8, R8
|
|
ADD R8, R3, R15
|
|
MOVWU 1(R15), R10
|
|
MOVD R6, R8
|
|
LSR $0x08, R8, R8
|
|
CMPW R10, R8
|
|
BNE no_repeat_found_encodeSnappyBlockAsm8B
|
|
ADD $1, R2, R6
|
|
MOVWU R6, R6
|
|
MOVWU 20(RSP), R5
|
|
MOVWU R6, R7
|
|
MOVWU 24(RSP), R16
|
|
SUBSW R16, R7, R7
|
|
BEQ repeat_extend_back_end_encodeSnappyBlockAsm8B
|
|
|
|
repeat_extend_back_loop_encodeSnappyBlockAsm8B:
|
|
CMPW R5, R6
|
|
BLS repeat_extend_back_end_encodeSnappyBlockAsm8B
|
|
ADD R7, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R8
|
|
ADD R6, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R9
|
|
AND $0xff, R9, R16
|
|
AND $0xff, R8, R15
|
|
CMP R16, R15
|
|
BNE repeat_extend_back_end_encodeSnappyBlockAsm8B
|
|
SUB $1, R6, R6
|
|
MOVWU R6, R6
|
|
SUBSW $1, R7, R7
|
|
BNE repeat_extend_back_loop_encodeSnappyBlockAsm8B
|
|
|
|
repeat_extend_back_end_encodeSnappyBlockAsm8B:
|
|
MOVWU R6, R5
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R5, R5
|
|
ADD R5, R1, R5
|
|
ADD $3, R5, R5
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R5
|
|
BLO repeat_dst_size_check_encodeSnappyBlockAsm8B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
repeat_dst_size_check_encodeSnappyBlockAsm8B:
|
|
MOVWU 20(RSP), R5
|
|
CMPW R6, R5
|
|
BEQ emit_literal_done_repeat_emit_encodeSnappyBlockAsm8B
|
|
MOVWU R6, R7
|
|
MOVW R6, 20(RSP)
|
|
ADD R5, R3, R8
|
|
SUBW R5, R7, R7
|
|
SUB $1, R7, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x3c, R5
|
|
BLO one_byte_repeat_emit_encodeSnappyBlockAsm8B
|
|
CMPW $0x00000100, R5
|
|
BLO two_bytes_repeat_emit_encodeSnappyBlockAsm8B
|
|
BLO three_bytes_repeat_emit_encodeSnappyBlockAsm8B
|
|
|
|
three_bytes_repeat_emit_encodeSnappyBlockAsm8B:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_repeat_emit_encodeSnappyBlockAsm8B
|
|
|
|
two_bytes_repeat_emit_encodeSnappyBlockAsm8B:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R5, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R5
|
|
BLO memmove_repeat_emit_encodeSnappyBlockAsm8B
|
|
JMP memmove_long_repeat_emit_encodeSnappyBlockAsm8B
|
|
|
|
one_byte_repeat_emit_encodeSnappyBlockAsm8B:
|
|
LSLW $0x02, R5, R16
|
|
BFI $0, R16, $8, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_repeat_emit_encodeSnappyBlockAsm8B:
|
|
ADD R7, R1, R5
|
|
|
|
// genMemMoveShort
|
|
CMP $0x08, R7
|
|
BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_8
|
|
CMP $0x10, R7
|
|
BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_8through16
|
|
CMP $0x20, R7
|
|
BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_17through32
|
|
JMP emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_8:
|
|
MOVD (R8), R9
|
|
MOVD R9, (R1)
|
|
JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm8B
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_8through16:
|
|
MOVD (R8), R9
|
|
ADD R7, R8, R15
|
|
MOVD -8(R15), R8
|
|
MOVD R9, (R1)
|
|
ADD R7, R1, R15
|
|
MOVD R8, -8(R15)
|
|
JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm8B
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_17through32:
|
|
FMOVQ (R8), F0
|
|
ADD R7, R8, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R7, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm8B
|
|
|
|
emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_33through64:
|
|
FMOVQ (R8), F0
|
|
FMOVQ 16(R8), F1
|
|
ADD R7, R8, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R7, R8, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R7, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R7, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_repeat_emit_encodeSnappyBlockAsm8B:
|
|
MOVD R5, R1
|
|
JMP emit_literal_done_repeat_emit_encodeSnappyBlockAsm8B
|
|
|
|
memmove_long_repeat_emit_encodeSnappyBlockAsm8B:
|
|
ADD R7, R1, R5
|
|
|
|
// genMemMoveLong
|
|
MOVD R8, R9
|
|
MOVD R1, R10
|
|
MOVD R7, R11
|
|
|
|
emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm8Blarge_big_loop_back:
|
|
FMOVQ (R9), F0
|
|
FMOVQ 16(R9), F1
|
|
FMOVQ F0, (R10)
|
|
FMOVQ F1, 16(R10)
|
|
ADD $0x20, R9, R9
|
|
ADD $0x20, R10, R10
|
|
SUB $0x20, R11, R11
|
|
CMP $0x20, R11
|
|
BHS emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm8Blarge_big_loop_back
|
|
ADD R7, R8, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R7, R8, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R7, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R7, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R5, R1
|
|
|
|
emit_literal_done_repeat_emit_encodeSnappyBlockAsm8B:
|
|
ADDW $0x05, R2, R2
|
|
MOVWU R2, R5
|
|
MOVWU 24(RSP), R16
|
|
SUBW R16, R5, R5
|
|
MOVD src_len+32(FP), R7
|
|
SUBW R2, R7, R7
|
|
ADD R2, R3, R8
|
|
ADD R5, R3, R5
|
|
|
|
// matchLen
|
|
MOVD $0, R10
|
|
|
|
matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm8B:
|
|
CMPW $0x10, R7
|
|
BLO matchlen_match8_repeat_extend_encodeSnappyBlockAsm8B
|
|
MOVD (R8)(R10), R9
|
|
ADD R10, R8, R15
|
|
MOVD 8(R15), R11
|
|
MOVD (R5)(R10), R16
|
|
EOR R16, R9, R9
|
|
TST R9, R9
|
|
BNE matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm8B
|
|
ADD R10, R5, R15
|
|
MOVD 8(R15), R16
|
|
EOR R16, R11, R11
|
|
TST R11, R11
|
|
BNE matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm8B
|
|
SUB $16, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $16, R10, R10
|
|
MOVWU R10, R10
|
|
JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm8B
|
|
|
|
matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm8B:
|
|
RBIT R11, R11
|
|
CLZ R11, R11
|
|
ASR $0x03, R11, R11
|
|
ADD R11, R10, R10
|
|
ADD $8, R10, R10
|
|
MOVWU R10, R10
|
|
JMP repeat_extend_forward_end_encodeSnappyBlockAsm8B
|
|
|
|
matchlen_match8_repeat_extend_encodeSnappyBlockAsm8B:
|
|
CMPW $0x08, R7
|
|
BLO matchlen_match4_repeat_extend_encodeSnappyBlockAsm8B
|
|
MOVD (R8)(R10), R9
|
|
MOVD (R5)(R10), R16
|
|
EOR R16, R9, R9
|
|
TST R9, R9
|
|
BNE matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm8B
|
|
SUB $8, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $8, R10, R10
|
|
MOVWU R10, R10
|
|
JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm8B
|
|
|
|
matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm8B:
|
|
RBIT R9, R9
|
|
CLZ R9, R9
|
|
ASR $0x03, R9, R9
|
|
ADD R9, R10, R10
|
|
MOVWU R10, R10
|
|
JMP repeat_extend_forward_end_encodeSnappyBlockAsm8B
|
|
|
|
matchlen_match4_repeat_extend_encodeSnappyBlockAsm8B:
|
|
CMPW $0x04, R7
|
|
BLO matchlen_match2_repeat_extend_encodeSnappyBlockAsm8B
|
|
MOVWU (R8)(R10), R9
|
|
MOVWU (R5)(R10), R16
|
|
CMPW R9, R16
|
|
BNE matchlen_match2_repeat_extend_encodeSnappyBlockAsm8B
|
|
SUB $4, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $4, R10, R10
|
|
MOVWU R10, R10
|
|
|
|
matchlen_match2_repeat_extend_encodeSnappyBlockAsm8B:
|
|
CMPW $0x01, R7
|
|
BEQ matchlen_match1_repeat_extend_encodeSnappyBlockAsm8B
|
|
BLO repeat_extend_forward_end_encodeSnappyBlockAsm8B
|
|
MOVHU (R8)(R10), R16
|
|
BFI $0, R16, $16, R9
|
|
ADD R10, R5, R15
|
|
MOVHU (R15), R15
|
|
AND $0xffff, R9, R16
|
|
CMP R16, R15
|
|
BNE matchlen_match1_repeat_extend_encodeSnappyBlockAsm8B
|
|
ADD $2, R10, R10
|
|
MOVWU R10, R10
|
|
SUBSW $0x02, R7, R7
|
|
BEQ repeat_extend_forward_end_encodeSnappyBlockAsm8B
|
|
|
|
matchlen_match1_repeat_extend_encodeSnappyBlockAsm8B:
|
|
MOVBU (R8)(R10), R16
|
|
BFI $0, R16, $8, R9
|
|
ADD R10, R5, R15
|
|
MOVBU (R15), R15
|
|
AND $0xff, R9, R16
|
|
CMP R16, R15
|
|
BNE repeat_extend_forward_end_encodeSnappyBlockAsm8B
|
|
ADD $1, R10, R10
|
|
MOVWU R10, R10
|
|
|
|
repeat_extend_forward_end_encodeSnappyBlockAsm8B:
|
|
ADDW R10, R2, R2
|
|
MOVWU R2, R5
|
|
SUBW R6, R5, R5
|
|
MOVWU 24(RSP), R6
|
|
|
|
// emitCopy
|
|
two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm8B:
|
|
CMPW $0x40, R5
|
|
BLS two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm8B
|
|
MOVD $0xee, R16
|
|
MOVB R16, (R1)
|
|
MOVH R6, 1(R1)
|
|
SUB $60, R5, R5
|
|
MOVWU R5, R5
|
|
ADD $0x03, R1, R1
|
|
JMP two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm8B
|
|
|
|
two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm8B:
|
|
MOVWU R5, R7
|
|
LSLW $0x02, R7, R7
|
|
CMPW $0x0c, R5
|
|
BHS emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm8B
|
|
SUB $15, R7, R7
|
|
MOVWU R7, R7
|
|
MOVB R6, 1(R1)
|
|
LSRW $0x08, R6, R6
|
|
LSLW $0x05, R6, R6
|
|
ORRW R6, R7, R7
|
|
MOVB R7, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_encodeSnappyBlockAsm8B
|
|
|
|
emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm8B:
|
|
SUB $2, R7, R7
|
|
MOVWU R7, R7
|
|
MOVB R7, (R1)
|
|
MOVH R6, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
|
|
repeat_end_emit_encodeSnappyBlockAsm8B:
|
|
MOVW R2, 20(RSP)
|
|
JMP search_loop_encodeSnappyBlockAsm8B
|
|
|
|
no_repeat_found_encodeSnappyBlockAsm8B:
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ candidate_match_encodeSnappyBlockAsm8B
|
|
LSR $0x08, R6, R6
|
|
MOVWU (R0)(R9<<2), R5
|
|
ADD $2, R2, R8
|
|
MOVWU R8, R8
|
|
MOVWU (R3)(R7), R16
|
|
CMPW R6, R16
|
|
BEQ candidate2_match_encodeSnappyBlockAsm8B
|
|
MOVW R8, (R0)(R9<<2)
|
|
LSR $0x08, R6, R6
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ candidate3_match_encodeSnappyBlockAsm8B
|
|
MOVWU 28(RSP), R2
|
|
JMP search_loop_encodeSnappyBlockAsm8B
|
|
|
|
candidate3_match_encodeSnappyBlockAsm8B:
|
|
ADDW $0x02, R2, R2
|
|
JMP candidate_match_encodeSnappyBlockAsm8B
|
|
|
|
candidate2_match_encodeSnappyBlockAsm8B:
|
|
MOVW R8, (R0)(R9<<2)
|
|
ADDW $1, R2, R2
|
|
MOVWU R7, R5
|
|
|
|
candidate_match_encodeSnappyBlockAsm8B:
|
|
MOVWU 20(RSP), R6
|
|
TSTW R5, R5
|
|
BEQ match_extend_back_end_encodeSnappyBlockAsm8B
|
|
|
|
match_extend_back_loop_encodeSnappyBlockAsm8B:
|
|
CMPW R6, R2
|
|
BLS match_extend_back_end_encodeSnappyBlockAsm8B
|
|
ADD R5, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R7
|
|
ADD R2, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R8
|
|
AND $0xff, R8, R16
|
|
AND $0xff, R7, R15
|
|
CMP R16, R15
|
|
BNE match_extend_back_end_encodeSnappyBlockAsm8B
|
|
SUB $1, R2, R2
|
|
MOVWU R2, R2
|
|
SUBSW $1, R5, R5
|
|
BEQ match_extend_back_end_encodeSnappyBlockAsm8B
|
|
JMP match_extend_back_loop_encodeSnappyBlockAsm8B
|
|
|
|
match_extend_back_end_encodeSnappyBlockAsm8B:
|
|
MOVWU R2, R6
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R6, R6
|
|
ADD R6, R1, R6
|
|
ADD $3, R6, R6
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R6
|
|
BLO match_dst_size_check_encodeSnappyBlockAsm8B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeSnappyBlockAsm8B:
|
|
MOVWU R2, R6
|
|
MOVWU 20(RSP), R7
|
|
CMPW R6, R7
|
|
BEQ emit_literal_done_match_emit_encodeSnappyBlockAsm8B
|
|
MOVWU R6, R8
|
|
MOVW R6, 20(RSP)
|
|
ADD R7, R3, R6
|
|
SUBW R7, R8, R8
|
|
SUB $1, R8, R7
|
|
MOVWU R7, R7
|
|
CMPW $0x3c, R7
|
|
BLO one_byte_match_emit_encodeSnappyBlockAsm8B
|
|
CMPW $0x00000100, R7
|
|
BLO two_bytes_match_emit_encodeSnappyBlockAsm8B
|
|
BLO three_bytes_match_emit_encodeSnappyBlockAsm8B
|
|
|
|
three_bytes_match_emit_encodeSnappyBlockAsm8B:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R7, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_match_emit_encodeSnappyBlockAsm8B
|
|
|
|
two_bytes_match_emit_encodeSnappyBlockAsm8B:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R7, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R7
|
|
BLO memmove_match_emit_encodeSnappyBlockAsm8B
|
|
JMP memmove_long_match_emit_encodeSnappyBlockAsm8B
|
|
|
|
one_byte_match_emit_encodeSnappyBlockAsm8B:
|
|
LSLW $0x02, R7, R16
|
|
BFI $0, R16, $8, R7
|
|
MOVB R7, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_match_emit_encodeSnappyBlockAsm8B:
|
|
ADD R8, R1, R7
|
|
|
|
// genMemMoveShort
|
|
CMP $0x08, R8
|
|
BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_8
|
|
CMP $0x10, R8
|
|
BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_8through16
|
|
CMP $0x20, R8
|
|
BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_8:
|
|
MOVD (R6), R9
|
|
MOVD R9, (R1)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm8B
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_8through16:
|
|
MOVD (R6), R9
|
|
ADD R8, R6, R15
|
|
MOVD -8(R15), R6
|
|
MOVD R9, (R1)
|
|
ADD R8, R1, R15
|
|
MOVD R6, -8(R15)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm8B
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_17through32:
|
|
FMOVQ (R6), F0
|
|
ADD R8, R6, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm8B
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_33through64:
|
|
FMOVQ (R6), F0
|
|
FMOVQ 16(R6), F1
|
|
ADD R8, R6, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R8, R6, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_match_emit_encodeSnappyBlockAsm8B:
|
|
MOVD R7, R1
|
|
JMP emit_literal_done_match_emit_encodeSnappyBlockAsm8B
|
|
|
|
memmove_long_match_emit_encodeSnappyBlockAsm8B:
|
|
ADD R8, R1, R7
|
|
|
|
// genMemMoveLong
|
|
MOVD R6, R9
|
|
MOVD R1, R10
|
|
MOVD R8, R11
|
|
|
|
emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm8Blarge_big_loop_back:
|
|
FMOVQ (R9), F0
|
|
FMOVQ 16(R9), F1
|
|
FMOVQ F0, (R10)
|
|
FMOVQ F1, 16(R10)
|
|
ADD $0x20, R9, R9
|
|
ADD $0x20, R10, R10
|
|
SUB $0x20, R11, R11
|
|
CMP $0x20, R11
|
|
BHS emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm8Blarge_big_loop_back
|
|
ADD R8, R6, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R8, R6, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R8, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R7, R1
|
|
|
|
emit_literal_done_match_emit_encodeSnappyBlockAsm8B:
|
|
match_nolit_loop_encodeSnappyBlockAsm8B:
|
|
MOVWU R2, R6
|
|
SUBW R5, R6, R6
|
|
MOVW R6, 24(RSP)
|
|
ADDW $0x04, R2, R2
|
|
ADDW $0x04, R5, R5
|
|
MOVD src_len+32(FP), R6
|
|
SUBW R2, R6, R6
|
|
ADD R2, R3, R7
|
|
ADD R5, R3, R5
|
|
|
|
// matchLen
|
|
MOVD $0, R9
|
|
|
|
matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm8B:
|
|
CMPW $0x10, R6
|
|
BLO matchlen_match8_match_nolit_encodeSnappyBlockAsm8B
|
|
MOVD (R7)(R9), R8
|
|
ADD R9, R7, R15
|
|
MOVD 8(R15), R10
|
|
MOVD (R5)(R9), R16
|
|
EOR R16, R8, R8
|
|
TST R8, R8
|
|
BNE matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm8B
|
|
ADD R9, R5, R15
|
|
MOVD 8(R15), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_16match_nolit_encodeSnappyBlockAsm8B
|
|
SUB $16, R6, R6
|
|
MOVWU R6, R6
|
|
ADD $16, R9, R9
|
|
MOVWU R9, R9
|
|
JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm8B
|
|
|
|
matchlen_bsf_16match_nolit_encodeSnappyBlockAsm8B:
|
|
RBIT R10, R10
|
|
CLZ R10, R10
|
|
ASR $0x03, R10, R10
|
|
ADD R10, R9, R9
|
|
ADD $8, R9, R9
|
|
MOVWU R9, R9
|
|
JMP match_nolit_end_encodeSnappyBlockAsm8B
|
|
|
|
matchlen_match8_match_nolit_encodeSnappyBlockAsm8B:
|
|
CMPW $0x08, R6
|
|
BLO matchlen_match4_match_nolit_encodeSnappyBlockAsm8B
|
|
MOVD (R7)(R9), R8
|
|
MOVD (R5)(R9), R16
|
|
EOR R16, R8, R8
|
|
TST R8, R8
|
|
BNE matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm8B
|
|
SUB $8, R6, R6
|
|
MOVWU R6, R6
|
|
ADD $8, R9, R9
|
|
MOVWU R9, R9
|
|
JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm8B
|
|
|
|
matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm8B:
|
|
RBIT R8, R8
|
|
CLZ R8, R8
|
|
ASR $0x03, R8, R8
|
|
ADD R8, R9, R9
|
|
MOVWU R9, R9
|
|
JMP match_nolit_end_encodeSnappyBlockAsm8B
|
|
|
|
matchlen_match4_match_nolit_encodeSnappyBlockAsm8B:
|
|
CMPW $0x04, R6
|
|
BLO matchlen_match2_match_nolit_encodeSnappyBlockAsm8B
|
|
MOVWU (R7)(R9), R8
|
|
MOVWU (R5)(R9), R16
|
|
CMPW R8, R16
|
|
BNE matchlen_match2_match_nolit_encodeSnappyBlockAsm8B
|
|
SUB $4, R6, R6
|
|
MOVWU R6, R6
|
|
ADD $4, R9, R9
|
|
MOVWU R9, R9
|
|
|
|
matchlen_match2_match_nolit_encodeSnappyBlockAsm8B:
|
|
CMPW $0x01, R6
|
|
BEQ matchlen_match1_match_nolit_encodeSnappyBlockAsm8B
|
|
BLO match_nolit_end_encodeSnappyBlockAsm8B
|
|
MOVHU (R7)(R9), R16
|
|
BFI $0, R16, $16, R8
|
|
ADD R9, R5, R15
|
|
MOVHU (R15), R15
|
|
AND $0xffff, R8, R16
|
|
CMP R16, R15
|
|
BNE matchlen_match1_match_nolit_encodeSnappyBlockAsm8B
|
|
ADD $2, R9, R9
|
|
MOVWU R9, R9
|
|
SUBSW $0x02, R6, R6
|
|
BEQ match_nolit_end_encodeSnappyBlockAsm8B
|
|
|
|
matchlen_match1_match_nolit_encodeSnappyBlockAsm8B:
|
|
MOVBU (R7)(R9), R16
|
|
BFI $0, R16, $8, R8
|
|
ADD R9, R5, R15
|
|
MOVBU (R15), R15
|
|
AND $0xff, R8, R16
|
|
CMP R16, R15
|
|
BNE match_nolit_end_encodeSnappyBlockAsm8B
|
|
ADD $1, R9, R9
|
|
MOVWU R9, R9
|
|
|
|
match_nolit_end_encodeSnappyBlockAsm8B:
|
|
ADDW R9, R2, R2
|
|
MOVWU 24(RSP), R5
|
|
ADDW $0x04, R9, R9
|
|
MOVW R2, 20(RSP)
|
|
|
|
// emitCopy
|
|
two_byte_offset_match_nolit_encodeSnappyBlockAsm8B:
|
|
CMPW $0x40, R9
|
|
BLS two_byte_offset_short_match_nolit_encodeSnappyBlockAsm8B
|
|
MOVD $0xee, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
SUB $60, R9, R9
|
|
MOVWU R9, R9
|
|
ADD $0x03, R1, R1
|
|
JMP two_byte_offset_match_nolit_encodeSnappyBlockAsm8B
|
|
|
|
two_byte_offset_short_match_nolit_encodeSnappyBlockAsm8B:
|
|
MOVWU R9, R6
|
|
LSLW $0x02, R6, R6
|
|
CMPW $0x0c, R9
|
|
BHS emit_copy_three_match_nolit_encodeSnappyBlockAsm8B
|
|
SUB $15, R6, R6
|
|
MOVWU R6, R6
|
|
MOVB R5, 1(R1)
|
|
LSRW $0x08, R5, R5
|
|
LSLW $0x05, R5, R5
|
|
ORRW R5, R6, R6
|
|
MOVB R6, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeSnappyBlockAsm8B
|
|
|
|
emit_copy_three_match_nolit_encodeSnappyBlockAsm8B:
|
|
SUB $2, R6, R6
|
|
MOVWU R6, R6
|
|
MOVB R6, (R1)
|
|
MOVH R5, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
|
|
match_nolit_emitcopy_end_encodeSnappyBlockAsm8B:
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R2
|
|
BHS emit_remainder_encodeSnappyBlockAsm8B
|
|
ADD R2, R3, R15
|
|
MOVD -2(R15), R6
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R1
|
|
BLO match_nolit_dst_ok_encodeSnappyBlockAsm8B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeSnappyBlockAsm8B:
|
|
MOVD $0x9e3779b1, R8
|
|
MOVD R6, R7
|
|
LSR $0x10, R6, R6
|
|
MOVD R6, R5
|
|
LSL $0x20, R7, R7
|
|
MUL R8, R7, R7
|
|
LSR $0x38, R7, R7
|
|
LSL $0x20, R5, R5
|
|
MUL R8, R5, R5
|
|
LSR $0x38, R5, R5
|
|
SUB $2, R2, R8
|
|
MOVWU R8, R8
|
|
ADD R5<<2, R0, R9
|
|
MOVWU (R9), R5
|
|
MOVW R8, (R0)(R7<<2)
|
|
MOVW R2, (R9)
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ match_nolit_loop_encodeSnappyBlockAsm8B
|
|
ADDW $1, R2, R2
|
|
JMP search_loop_encodeSnappyBlockAsm8B
|
|
|
|
emit_remainder_encodeSnappyBlockAsm8B:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R0, R0
|
|
ADD R0, R1, R0
|
|
ADD $3, R0, R0
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R0
|
|
BLO emit_remainder_ok_encodeSnappyBlockAsm8B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeSnappyBlockAsm8B:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R2
|
|
CMPW R0, R2
|
|
BEQ emit_literal_done_emit_remainder_encodeSnappyBlockAsm8B
|
|
MOVWU R0, R5
|
|
MOVW R0, 20(RSP)
|
|
ADD R2, R3, R0
|
|
SUBW R2, R5, R5
|
|
SUB $1, R5, R2
|
|
MOVWU R2, R2
|
|
CMPW $0x3c, R2
|
|
BLO one_byte_emit_remainder_encodeSnappyBlockAsm8B
|
|
CMPW $0x00000100, R2
|
|
BLO two_bytes_emit_remainder_encodeSnappyBlockAsm8B
|
|
BLO three_bytes_emit_remainder_encodeSnappyBlockAsm8B
|
|
|
|
three_bytes_emit_remainder_encodeSnappyBlockAsm8B:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R2, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_emit_remainder_encodeSnappyBlockAsm8B
|
|
|
|
two_bytes_emit_remainder_encodeSnappyBlockAsm8B:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R2, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R2
|
|
BLO memmove_emit_remainder_encodeSnappyBlockAsm8B
|
|
JMP memmove_long_emit_remainder_encodeSnappyBlockAsm8B
|
|
|
|
one_byte_emit_remainder_encodeSnappyBlockAsm8B:
|
|
LSLW $0x02, R2, R16
|
|
BFI $0, R16, $8, R2
|
|
MOVB R2, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_emit_remainder_encodeSnappyBlockAsm8B:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveShort
|
|
CMP $0x03, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_1or2
|
|
BEQ emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_3
|
|
CMP $0x08, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_4through7
|
|
CMP $0x10, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_8through16
|
|
CMP $0x20, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_1or2:
|
|
MOVBU (R0), R16
|
|
BFI $0, R16, $8, R5
|
|
ADD R3, R0, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVB R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVB R0, -1(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_3:
|
|
MOVHU (R0), R16
|
|
BFI $0, R16, $16, R5
|
|
MOVBU 2(R0), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVH R5, (R1)
|
|
MOVB R0, 2(R1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_4through7:
|
|
MOVWU (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVWU -4(R15), R0
|
|
MOVW R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVW R0, -4(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_8through16:
|
|
MOVD (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVD -8(R15), R0
|
|
MOVD R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVD R0, -8(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_17through32:
|
|
FMOVQ (R0), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_33through64:
|
|
FMOVQ (R0), F0
|
|
FMOVQ 16(R0), F1
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_emit_remainder_encodeSnappyBlockAsm8B:
|
|
MOVD R2, R1
|
|
JMP emit_literal_done_emit_remainder_encodeSnappyBlockAsm8B
|
|
|
|
memmove_long_emit_remainder_encodeSnappyBlockAsm8B:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveLong
|
|
MOVD R0, R5
|
|
MOVD R1, R6
|
|
MOVD R3, R7
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm8Blarge_big_loop_back:
|
|
FMOVQ (R5), F0
|
|
FMOVQ 16(R5), F1
|
|
FMOVQ F0, (R6)
|
|
FMOVQ F1, 16(R6)
|
|
ADD $0x20, R5, R5
|
|
ADD $0x20, R6, R6
|
|
SUB $0x20, R7, R7
|
|
CMP $0x20, R7
|
|
BHS emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm8Blarge_big_loop_back
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R3, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R2, R1
|
|
|
|
emit_literal_done_emit_remainder_encodeSnappyBlockAsm8B:
|
|
MOVD dst_base+0(FP), R0
|
|
SUB R0, R1, R1
|
|
MOVD R1, ret+56(FP)
|
|
RET
|
|
|
|
// func encodeSnappyBetterBlockAsm(dst []byte, src []byte, tmp *[589824]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeSnappyBetterBlockAsm(SB), $24-64
|
|
MOVD tmp+48(FP), R0
|
|
MOVD dst_base+0(FP), R1
|
|
MOVD $0x00001200, R2
|
|
MOVD R0, R3
|
|
VEOR V0.B16, V0.B16, V0.B16
|
|
|
|
zero_loop_encodeSnappyBetterBlockAsm:
|
|
FMOVQ F0, (R3)
|
|
FMOVQ F0, 16(R3)
|
|
FMOVQ F0, 32(R3)
|
|
FMOVQ F0, 48(R3)
|
|
FMOVQ F0, 64(R3)
|
|
FMOVQ F0, 80(R3)
|
|
FMOVQ F0, 96(R3)
|
|
FMOVQ F0, 112(R3)
|
|
ADD $0x80, R3, R3
|
|
SUBS $1, R2, R2
|
|
BNE zero_loop_encodeSnappyBetterBlockAsm
|
|
MOVD $0x00000000, R16
|
|
MOVW R16, 20(RSP)
|
|
MOVD src_len+32(FP), R2
|
|
SUB $9, R2, R3
|
|
SUB $8, R2, R5
|
|
MOVW R5, 16(RSP)
|
|
LSR $0x05, R2, R2
|
|
SUBW R2, R3, R3
|
|
ADD R3, R1, R3
|
|
MOVD R3, 8(RSP)
|
|
MOVD $0x00000001, R2
|
|
MOVD $0x00000000, R16
|
|
MOVW R16, 24(RSP)
|
|
MOVD src_base+24(FP), R3
|
|
|
|
search_loop_encodeSnappyBetterBlockAsm:
|
|
MOVWU R2, R5
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R5, R5
|
|
LSRW $0x07, R5, R5
|
|
CMPW $0x63, R5
|
|
BLS check_maxskip_ok_encodeSnappyBetterBlockAsm
|
|
ADD $100, R2, R5
|
|
MOVWU R5, R5
|
|
JMP check_maxskip_cont_encodeSnappyBetterBlockAsm
|
|
|
|
check_maxskip_ok_encodeSnappyBetterBlockAsm:
|
|
ADD R5, R2, R5
|
|
ADD $1, R5, R5
|
|
MOVWU R5, R5
|
|
|
|
check_maxskip_cont_encodeSnappyBetterBlockAsm:
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R5
|
|
BHS emit_remainder_encodeSnappyBetterBlockAsm
|
|
MOVD (R3)(R2), R6
|
|
MOVW R5, 28(RSP)
|
|
MOVD $0x00cf1bbcdcbfa563, R8
|
|
MOVD $0x9e3779b1, R5
|
|
MOVD R6, R9
|
|
MOVD R6, R10
|
|
LSL $0x08, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x2f, R9, R9
|
|
LSL $0x20, R10, R10
|
|
MUL R5, R10, R10
|
|
LSR $0x32, R10, R10
|
|
MOVWU (R0)(R9<<2), R5
|
|
ADD R10<<2, R0, R15
|
|
MOVWU 524288(R15), R7
|
|
MOVW R2, (R0)(R9<<2)
|
|
ADD R10<<2, R0, R15
|
|
MOVW R2, 524288(R15)
|
|
MOVD (R3)(R5), R9
|
|
MOVD (R3)(R7), R10
|
|
CMP R6, R9
|
|
BEQ candidate_match_encodeSnappyBetterBlockAsm
|
|
CMP R6, R10
|
|
BNE no_short_found_encodeSnappyBetterBlockAsm
|
|
MOVWU R7, R5
|
|
JMP candidate_match_encodeSnappyBetterBlockAsm
|
|
|
|
no_short_found_encodeSnappyBetterBlockAsm:
|
|
CMPW R6, R9
|
|
BEQ candidate_match_encodeSnappyBetterBlockAsm
|
|
CMPW R6, R10
|
|
BEQ candidateS_match_encodeSnappyBetterBlockAsm
|
|
MOVWU 28(RSP), R2
|
|
JMP search_loop_encodeSnappyBetterBlockAsm
|
|
|
|
candidateS_match_encodeSnappyBetterBlockAsm:
|
|
LSR $0x08, R6, R6
|
|
MOVD R6, R9
|
|
LSL $0x08, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x2f, R9, R9
|
|
MOVWU (R0)(R9<<2), R5
|
|
ADDW $1, R2, R2
|
|
MOVW R2, (R0)(R9<<2)
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ candidate_match_encodeSnappyBetterBlockAsm
|
|
SUBW $1, R2, R2
|
|
MOVWU R7, R5
|
|
|
|
candidate_match_encodeSnappyBetterBlockAsm:
|
|
MOVWU 20(RSP), R6
|
|
TSTW R5, R5
|
|
BEQ match_extend_back_end_encodeSnappyBetterBlockAsm
|
|
|
|
match_extend_back_loop_encodeSnappyBetterBlockAsm:
|
|
CMPW R6, R2
|
|
BLS match_extend_back_end_encodeSnappyBetterBlockAsm
|
|
ADD R5, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R7
|
|
ADD R2, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R8
|
|
AND $0xff, R8, R16
|
|
AND $0xff, R7, R15
|
|
CMP R16, R15
|
|
BNE match_extend_back_end_encodeSnappyBetterBlockAsm
|
|
SUB $1, R2, R2
|
|
MOVWU R2, R2
|
|
SUBSW $1, R5, R5
|
|
BEQ match_extend_back_end_encodeSnappyBetterBlockAsm
|
|
JMP match_extend_back_loop_encodeSnappyBetterBlockAsm
|
|
|
|
match_extend_back_end_encodeSnappyBetterBlockAsm:
|
|
MOVWU R2, R6
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R6, R6
|
|
ADD R6, R1, R6
|
|
ADD $5, R6, R6
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R6
|
|
BLO match_dst_size_check_encodeSnappyBetterBlockAsm
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeSnappyBetterBlockAsm:
|
|
MOVWU R2, R6
|
|
ADDW $0x04, R2, R2
|
|
ADDW $0x04, R5, R5
|
|
MOVD src_len+32(FP), R7
|
|
SUBW R2, R7, R7
|
|
ADD R2, R3, R8
|
|
ADD R5, R3, R9
|
|
|
|
// matchLen
|
|
MOVD $0, R11
|
|
|
|
matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm:
|
|
CMPW $0x10, R7
|
|
BLO matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm
|
|
MOVD (R8)(R11), R10
|
|
ADD R11, R8, R15
|
|
MOVD 8(R15), R12
|
|
MOVD (R9)(R11), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm
|
|
ADD R11, R9, R15
|
|
MOVD 8(R15), R16
|
|
EOR R16, R12, R12
|
|
TST R12, R12
|
|
BNE matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm
|
|
SUB $16, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $16, R11, R11
|
|
MOVWU R11, R11
|
|
JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm
|
|
|
|
matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm:
|
|
RBIT R12, R12
|
|
CLZ R12, R12
|
|
ASR $0x03, R12, R12
|
|
ADD R12, R11, R11
|
|
ADD $8, R11, R11
|
|
MOVWU R11, R11
|
|
JMP match_nolit_end_encodeSnappyBetterBlockAsm
|
|
|
|
matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm:
|
|
CMPW $0x08, R7
|
|
BLO matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm
|
|
MOVD (R8)(R11), R10
|
|
MOVD (R9)(R11), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm
|
|
SUB $8, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $8, R11, R11
|
|
MOVWU R11, R11
|
|
JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm
|
|
|
|
matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm:
|
|
RBIT R10, R10
|
|
CLZ R10, R10
|
|
ASR $0x03, R10, R10
|
|
ADD R10, R11, R11
|
|
MOVWU R11, R11
|
|
JMP match_nolit_end_encodeSnappyBetterBlockAsm
|
|
|
|
matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm:
|
|
CMPW $0x04, R7
|
|
BLO matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm
|
|
MOVWU (R8)(R11), R10
|
|
MOVWU (R9)(R11), R16
|
|
CMPW R10, R16
|
|
BNE matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm
|
|
SUB $4, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $4, R11, R11
|
|
MOVWU R11, R11
|
|
|
|
matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm:
|
|
CMPW $0x01, R7
|
|
BEQ matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm
|
|
BLO match_nolit_end_encodeSnappyBetterBlockAsm
|
|
MOVHU (R8)(R11), R16
|
|
BFI $0, R16, $16, R10
|
|
ADD R11, R9, R15
|
|
MOVHU (R15), R15
|
|
AND $0xffff, R10, R16
|
|
CMP R16, R15
|
|
BNE matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm
|
|
ADD $2, R11, R11
|
|
MOVWU R11, R11
|
|
SUBSW $0x02, R7, R7
|
|
BEQ match_nolit_end_encodeSnappyBetterBlockAsm
|
|
|
|
matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm:
|
|
MOVBU (R8)(R11), R16
|
|
BFI $0, R16, $8, R10
|
|
ADD R11, R9, R15
|
|
MOVBU (R15), R15
|
|
AND $0xff, R10, R16
|
|
CMP R16, R15
|
|
BNE match_nolit_end_encodeSnappyBetterBlockAsm
|
|
ADD $1, R11, R11
|
|
MOVWU R11, R11
|
|
|
|
match_nolit_end_encodeSnappyBetterBlockAsm:
|
|
MOVWU R2, R7
|
|
SUBW R5, R7, R7
|
|
|
|
// Check if repeat
|
|
CMPW $0x01, R11
|
|
BHI match_length_ok_encodeSnappyBetterBlockAsm
|
|
CMPW $0x0000ffff, R7
|
|
BLS match_length_ok_encodeSnappyBetterBlockAsm
|
|
MOVWU 28(RSP), R2
|
|
ADDW $1, R2, R2
|
|
JMP search_loop_encodeSnappyBetterBlockAsm
|
|
|
|
match_length_ok_encodeSnappyBetterBlockAsm:
|
|
MOVW R7, 24(RSP)
|
|
MOVWU 20(RSP), R5
|
|
CMPW R6, R5
|
|
BEQ emit_literal_done_match_emit_encodeSnappyBetterBlockAsm
|
|
MOVWU R6, R8
|
|
MOVW R6, 20(RSP)
|
|
ADD R5, R3, R9
|
|
SUBW R5, R8, R8
|
|
SUB $1, R8, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x3c, R5
|
|
BLO one_byte_match_emit_encodeSnappyBetterBlockAsm
|
|
CMPW $0x00000100, R5
|
|
BLO two_bytes_match_emit_encodeSnappyBetterBlockAsm
|
|
CMPW $0x00010000, R5
|
|
BLO three_bytes_match_emit_encodeSnappyBetterBlockAsm
|
|
CMPW $0x01000000, R5
|
|
BLO four_bytes_match_emit_encodeSnappyBetterBlockAsm
|
|
MOVD $0xfc, R16
|
|
MOVB R16, (R1)
|
|
MOVW R5, 1(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm
|
|
|
|
four_bytes_match_emit_encodeSnappyBetterBlockAsm:
|
|
MOVWU R5, R10
|
|
LSRW $0x10, R10, R10
|
|
MOVD $0xf8, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
MOVB R10, 3(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm
|
|
|
|
three_bytes_match_emit_encodeSnappyBetterBlockAsm:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm
|
|
|
|
two_bytes_match_emit_encodeSnappyBetterBlockAsm:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R5, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R5
|
|
BLO memmove_match_emit_encodeSnappyBetterBlockAsm
|
|
JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm
|
|
|
|
one_byte_match_emit_encodeSnappyBetterBlockAsm:
|
|
LSLW $0x02, R5, R16
|
|
BFI $0, R16, $8, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_match_emit_encodeSnappyBetterBlockAsm:
|
|
ADD R8, R1, R5
|
|
|
|
// genMemMoveShort
|
|
CMP $0x08, R8
|
|
BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_8
|
|
CMP $0x10, R8
|
|
BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_8through16
|
|
CMP $0x20, R8
|
|
BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_8:
|
|
MOVD (R9), R10
|
|
MOVD R10, (R1)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_8through16:
|
|
MOVD (R9), R10
|
|
ADD R8, R9, R15
|
|
MOVD -8(R15), R9
|
|
MOVD R10, (R1)
|
|
ADD R8, R1, R15
|
|
MOVD R9, -8(R15)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_17through32:
|
|
FMOVQ (R9), F0
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_33through64:
|
|
FMOVQ (R9), F0
|
|
FMOVQ 16(R9), F1
|
|
ADD R8, R9, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm:
|
|
MOVD R5, R1
|
|
JMP emit_literal_done_match_emit_encodeSnappyBetterBlockAsm
|
|
|
|
memmove_long_match_emit_encodeSnappyBetterBlockAsm:
|
|
ADD R8, R1, R5
|
|
|
|
// genMemMoveLong
|
|
MOVD R9, R10
|
|
MOVD R1, R12
|
|
MOVD R8, R13
|
|
|
|
emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsmlarge_big_loop_back:
|
|
FMOVQ (R10), F0
|
|
FMOVQ 16(R10), F1
|
|
FMOVQ F0, (R12)
|
|
FMOVQ F1, 16(R12)
|
|
ADD $0x20, R10, R10
|
|
ADD $0x20, R12, R12
|
|
SUB $0x20, R13, R13
|
|
CMP $0x20, R13
|
|
BHS emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsmlarge_big_loop_back
|
|
ADD R8, R9, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R8, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R5, R1
|
|
|
|
emit_literal_done_match_emit_encodeSnappyBetterBlockAsm:
|
|
ADDW R11, R2, R2
|
|
ADDW $0x04, R11, R11
|
|
MOVW R2, 20(RSP)
|
|
|
|
// emitCopy
|
|
CMPW $0x00010000, R7
|
|
BLO two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm
|
|
|
|
four_bytes_loop_back_match_nolit_encodeSnappyBetterBlockAsm:
|
|
CMPW $0x40, R11
|
|
BLS four_bytes_remain_match_nolit_encodeSnappyBetterBlockAsm
|
|
MOVD $0xff, R16
|
|
MOVB R16, (R1)
|
|
MOVW R7, 1(R1)
|
|
SUB $64, R11, R11
|
|
MOVWU R11, R11
|
|
ADD $0x05, R1, R1
|
|
CMPW $0x04, R11
|
|
BLO four_bytes_remain_match_nolit_encodeSnappyBetterBlockAsm
|
|
JMP four_bytes_loop_back_match_nolit_encodeSnappyBetterBlockAsm
|
|
|
|
four_bytes_remain_match_nolit_encodeSnappyBetterBlockAsm:
|
|
TSTW R11, R11
|
|
BEQ match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm
|
|
MOVD $0, R5
|
|
ADD R11<<2, R5, R11
|
|
SUB $1, R11, R11
|
|
MOVWU R11, R11
|
|
MOVB R11, (R1)
|
|
MOVW R7, 1(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm
|
|
|
|
two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm:
|
|
CMPW $0x40, R11
|
|
BLS two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm
|
|
MOVD $0xee, R16
|
|
MOVB R16, (R1)
|
|
MOVH R7, 1(R1)
|
|
SUB $60, R11, R11
|
|
MOVWU R11, R11
|
|
ADD $0x03, R1, R1
|
|
JMP two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm
|
|
|
|
two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm:
|
|
MOVWU R11, R5
|
|
LSLW $0x02, R5, R5
|
|
CMPW $0x0c, R11
|
|
BHS emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm
|
|
CMPW $0x00000800, R7
|
|
BHS emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm
|
|
SUB $15, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R7, 1(R1)
|
|
LSRW $0x08, R7, R7
|
|
LSLW $0x05, R7, R7
|
|
ORRW R7, R5, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm
|
|
|
|
emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm:
|
|
SUB $2, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R5, (R1)
|
|
MOVH R7, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
|
|
match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm:
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R2
|
|
BHS emit_remainder_encodeSnappyBetterBlockAsm
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R1
|
|
BLO match_nolit_dst_ok_encodeSnappyBetterBlockAsm
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeSnappyBetterBlockAsm:
|
|
MOVD $0x00cf1bbcdcbfa563, R5
|
|
MOVD $0x9e3779b1, R7
|
|
ADD $1, R6, R6
|
|
SUB $2, R2, R8
|
|
MOVD (R3)(R6), R9
|
|
ADD R6, R3, R15
|
|
MOVD 1(R15), R10
|
|
MOVD (R3)(R8), R11
|
|
ADD R8, R3, R15
|
|
MOVD 1(R15), R12
|
|
LSL $0x08, R9, R9
|
|
MUL R5, R9, R9
|
|
LSR $0x2f, R9, R9
|
|
LSL $0x20, R10, R10
|
|
MUL R7, R10, R10
|
|
LSR $0x32, R10, R10
|
|
LSL $0x08, R11, R11
|
|
MUL R5, R11, R11
|
|
LSR $0x2f, R11, R11
|
|
LSL $0x20, R12, R12
|
|
MUL R7, R12, R12
|
|
LSR $0x32, R12, R12
|
|
ADD $1, R6, R7
|
|
ADD $1, R8, R13
|
|
MOVW R6, (R0)(R9<<2)
|
|
MOVW R8, (R0)(R11<<2)
|
|
ADD R10<<2, R0, R15
|
|
MOVW R7, 524288(R15)
|
|
ADD R12<<2, R0, R15
|
|
MOVW R13, 524288(R15)
|
|
ADD R6, R8, R7
|
|
ADD $1, R7, R7
|
|
LSR $0x01, R7, R7
|
|
ADD $0x01, R6, R6
|
|
SUB $0x01, R8, R8
|
|
|
|
index_loop_encodeSnappyBetterBlockAsm:
|
|
CMP R8, R7
|
|
BHS search_loop_encodeSnappyBetterBlockAsm
|
|
MOVD (R3)(R6), R9
|
|
MOVD (R3)(R7), R10
|
|
LSL $0x08, R9, R9
|
|
MUL R5, R9, R9
|
|
LSR $0x2f, R9, R9
|
|
LSL $0x08, R10, R10
|
|
MUL R5, R10, R10
|
|
LSR $0x2f, R10, R10
|
|
MOVW R6, (R0)(R9<<2)
|
|
MOVW R7, (R0)(R10<<2)
|
|
ADD $0x02, R6, R6
|
|
ADD $0x02, R7, R7
|
|
JMP index_loop_encodeSnappyBetterBlockAsm
|
|
|
|
emit_remainder_encodeSnappyBetterBlockAsm:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R0, R0
|
|
ADD R0, R1, R0
|
|
ADD $5, R0, R0
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R0
|
|
BLO emit_remainder_ok_encodeSnappyBetterBlockAsm
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeSnappyBetterBlockAsm:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R2
|
|
CMPW R0, R2
|
|
BEQ emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm
|
|
MOVWU R0, R5
|
|
MOVW R0, 20(RSP)
|
|
ADD R2, R3, R0
|
|
SUBW R2, R5, R5
|
|
SUB $1, R5, R2
|
|
MOVWU R2, R2
|
|
CMPW $0x3c, R2
|
|
BLO one_byte_emit_remainder_encodeSnappyBetterBlockAsm
|
|
CMPW $0x00000100, R2
|
|
BLO two_bytes_emit_remainder_encodeSnappyBetterBlockAsm
|
|
CMPW $0x00010000, R2
|
|
BLO three_bytes_emit_remainder_encodeSnappyBetterBlockAsm
|
|
CMPW $0x01000000, R2
|
|
BLO four_bytes_emit_remainder_encodeSnappyBetterBlockAsm
|
|
MOVD $0xfc, R16
|
|
MOVB R16, (R1)
|
|
MOVW R2, 1(R1)
|
|
ADD $0x05, R1, R1
|
|
JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm
|
|
|
|
four_bytes_emit_remainder_encodeSnappyBetterBlockAsm:
|
|
MOVWU R2, R3
|
|
LSRW $0x10, R3, R3
|
|
MOVD $0xf8, R16
|
|
MOVB R16, (R1)
|
|
MOVH R2, 1(R1)
|
|
MOVB R3, 3(R1)
|
|
ADD $0x04, R1, R1
|
|
JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm
|
|
|
|
three_bytes_emit_remainder_encodeSnappyBetterBlockAsm:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R2, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm
|
|
|
|
two_bytes_emit_remainder_encodeSnappyBetterBlockAsm:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R2, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R2
|
|
BLO memmove_emit_remainder_encodeSnappyBetterBlockAsm
|
|
JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm
|
|
|
|
one_byte_emit_remainder_encodeSnappyBetterBlockAsm:
|
|
LSLW $0x02, R2, R16
|
|
BFI $0, R16, $8, R2
|
|
MOVB R2, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_emit_remainder_encodeSnappyBetterBlockAsm:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveShort
|
|
CMP $0x03, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_1or2
|
|
BEQ emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_3
|
|
CMP $0x08, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_4through7
|
|
CMP $0x10, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_8through16
|
|
CMP $0x20, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_1or2:
|
|
MOVBU (R0), R16
|
|
BFI $0, R16, $8, R5
|
|
ADD R3, R0, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVB R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVB R0, -1(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_3:
|
|
MOVHU (R0), R16
|
|
BFI $0, R16, $16, R5
|
|
MOVBU 2(R0), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVH R5, (R1)
|
|
MOVB R0, 2(R1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_4through7:
|
|
MOVWU (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVWU -4(R15), R0
|
|
MOVW R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVW R0, -4(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_8through16:
|
|
MOVD (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVD -8(R15), R0
|
|
MOVD R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVD R0, -8(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_17through32:
|
|
FMOVQ (R0), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_33through64:
|
|
FMOVQ (R0), F0
|
|
FMOVQ 16(R0), F1
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm:
|
|
MOVD R2, R1
|
|
JMP emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm
|
|
|
|
memmove_long_emit_remainder_encodeSnappyBetterBlockAsm:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveLong
|
|
MOVD R0, R5
|
|
MOVD R1, R6
|
|
MOVD R3, R7
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsmlarge_big_loop_back:
|
|
FMOVQ (R5), F0
|
|
FMOVQ 16(R5), F1
|
|
FMOVQ F0, (R6)
|
|
FMOVQ F1, 16(R6)
|
|
ADD $0x20, R5, R5
|
|
ADD $0x20, R6, R6
|
|
SUB $0x20, R7, R7
|
|
CMP $0x20, R7
|
|
BHS emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsmlarge_big_loop_back
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R3, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R2, R1
|
|
|
|
emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm:
|
|
MOVD dst_base+0(FP), R0
|
|
SUB R0, R1, R1
|
|
MOVD R1, ret+56(FP)
|
|
RET
|
|
|
|
// func encodeSnappyBetterBlockAsm64K(dst []byte, src []byte, tmp *[294912]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeSnappyBetterBlockAsm64K(SB), $24-64
|
|
MOVD tmp+48(FP), R0
|
|
MOVD dst_base+0(FP), R1
|
|
MOVD $0x00000900, R2
|
|
MOVD R0, R3
|
|
VEOR V0.B16, V0.B16, V0.B16
|
|
|
|
zero_loop_encodeSnappyBetterBlockAsm64K:
|
|
FMOVQ F0, (R3)
|
|
FMOVQ F0, 16(R3)
|
|
FMOVQ F0, 32(R3)
|
|
FMOVQ F0, 48(R3)
|
|
FMOVQ F0, 64(R3)
|
|
FMOVQ F0, 80(R3)
|
|
FMOVQ F0, 96(R3)
|
|
FMOVQ F0, 112(R3)
|
|
ADD $0x80, R3, R3
|
|
SUBS $1, R2, R2
|
|
BNE zero_loop_encodeSnappyBetterBlockAsm64K
|
|
MOVD $0x00000000, R16
|
|
MOVW R16, 20(RSP)
|
|
MOVD src_len+32(FP), R2
|
|
SUB $9, R2, R3
|
|
SUB $8, R2, R5
|
|
MOVW R5, 16(RSP)
|
|
LSR $0x05, R2, R2
|
|
SUBW R2, R3, R3
|
|
ADD R3, R1, R3
|
|
MOVD R3, 8(RSP)
|
|
MOVD $0x00000001, R2
|
|
MOVD $0x00000000, R16
|
|
MOVW R16, 24(RSP)
|
|
MOVD src_base+24(FP), R3
|
|
|
|
search_loop_encodeSnappyBetterBlockAsm64K:
|
|
MOVWU R2, R5
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R5, R5
|
|
LSRW $0x07, R5, R5
|
|
ADD R5, R2, R5
|
|
ADD $1, R5, R5
|
|
MOVWU R5, R5
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R5
|
|
BHS emit_remainder_encodeSnappyBetterBlockAsm64K
|
|
MOVD (R3)(R2), R6
|
|
MOVW R5, 28(RSP)
|
|
MOVD $0x00cf1bbcdcbfa563, R8
|
|
MOVD $0x9e3779b1, R5
|
|
MOVD R6, R9
|
|
MOVD R6, R10
|
|
LSL $0x08, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x30, R9, R9
|
|
LSL $0x20, R10, R10
|
|
MUL R5, R10, R10
|
|
LSR $0x33, R10, R10
|
|
MOVWU (R0)(R9<<2), R5
|
|
ADD R10<<2, R0, R15
|
|
MOVWU 262144(R15), R7
|
|
MOVW R2, (R0)(R9<<2)
|
|
ADD R10<<2, R0, R15
|
|
MOVW R2, 262144(R15)
|
|
MOVD (R3)(R5), R9
|
|
MOVD (R3)(R7), R10
|
|
CMP R6, R9
|
|
BEQ candidate_match_encodeSnappyBetterBlockAsm64K
|
|
CMP R6, R10
|
|
BNE no_short_found_encodeSnappyBetterBlockAsm64K
|
|
MOVWU R7, R5
|
|
JMP candidate_match_encodeSnappyBetterBlockAsm64K
|
|
|
|
no_short_found_encodeSnappyBetterBlockAsm64K:
|
|
CMPW R6, R9
|
|
BEQ candidate_match_encodeSnappyBetterBlockAsm64K
|
|
CMPW R6, R10
|
|
BEQ candidateS_match_encodeSnappyBetterBlockAsm64K
|
|
MOVWU 28(RSP), R2
|
|
JMP search_loop_encodeSnappyBetterBlockAsm64K
|
|
|
|
candidateS_match_encodeSnappyBetterBlockAsm64K:
|
|
LSR $0x08, R6, R6
|
|
MOVD R6, R9
|
|
LSL $0x08, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x30, R9, R9
|
|
MOVWU (R0)(R9<<2), R5
|
|
ADDW $1, R2, R2
|
|
MOVW R2, (R0)(R9<<2)
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ candidate_match_encodeSnappyBetterBlockAsm64K
|
|
SUBW $1, R2, R2
|
|
MOVWU R7, R5
|
|
|
|
candidate_match_encodeSnappyBetterBlockAsm64K:
|
|
MOVWU 20(RSP), R6
|
|
TSTW R5, R5
|
|
BEQ match_extend_back_end_encodeSnappyBetterBlockAsm64K
|
|
|
|
match_extend_back_loop_encodeSnappyBetterBlockAsm64K:
|
|
CMPW R6, R2
|
|
BLS match_extend_back_end_encodeSnappyBetterBlockAsm64K
|
|
ADD R5, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R7
|
|
ADD R2, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R8
|
|
AND $0xff, R8, R16
|
|
AND $0xff, R7, R15
|
|
CMP R16, R15
|
|
BNE match_extend_back_end_encodeSnappyBetterBlockAsm64K
|
|
SUB $1, R2, R2
|
|
MOVWU R2, R2
|
|
SUBSW $1, R5, R5
|
|
BEQ match_extend_back_end_encodeSnappyBetterBlockAsm64K
|
|
JMP match_extend_back_loop_encodeSnappyBetterBlockAsm64K
|
|
|
|
match_extend_back_end_encodeSnappyBetterBlockAsm64K:
|
|
MOVWU R2, R6
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R6, R6
|
|
ADD R6, R1, R6
|
|
ADD $3, R6, R6
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R6
|
|
BLO match_dst_size_check_encodeSnappyBetterBlockAsm64K
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeSnappyBetterBlockAsm64K:
|
|
MOVWU R2, R6
|
|
ADDW $0x04, R2, R2
|
|
ADDW $0x04, R5, R5
|
|
MOVD src_len+32(FP), R7
|
|
SUBW R2, R7, R7
|
|
ADD R2, R3, R8
|
|
ADD R5, R3, R9
|
|
|
|
// matchLen
|
|
MOVD $0, R11
|
|
|
|
matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm64K:
|
|
CMPW $0x10, R7
|
|
BLO matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm64K
|
|
MOVD (R8)(R11), R10
|
|
ADD R11, R8, R15
|
|
MOVD 8(R15), R12
|
|
MOVD (R9)(R11), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm64K
|
|
ADD R11, R9, R15
|
|
MOVD 8(R15), R16
|
|
EOR R16, R12, R12
|
|
TST R12, R12
|
|
BNE matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm64K
|
|
SUB $16, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $16, R11, R11
|
|
MOVWU R11, R11
|
|
JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm64K
|
|
|
|
matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm64K:
|
|
RBIT R12, R12
|
|
CLZ R12, R12
|
|
ASR $0x03, R12, R12
|
|
ADD R12, R11, R11
|
|
ADD $8, R11, R11
|
|
MOVWU R11, R11
|
|
JMP match_nolit_end_encodeSnappyBetterBlockAsm64K
|
|
|
|
matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm64K:
|
|
CMPW $0x08, R7
|
|
BLO matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm64K
|
|
MOVD (R8)(R11), R10
|
|
MOVD (R9)(R11), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm64K
|
|
SUB $8, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $8, R11, R11
|
|
MOVWU R11, R11
|
|
JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm64K
|
|
|
|
matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm64K:
|
|
RBIT R10, R10
|
|
CLZ R10, R10
|
|
ASR $0x03, R10, R10
|
|
ADD R10, R11, R11
|
|
MOVWU R11, R11
|
|
JMP match_nolit_end_encodeSnappyBetterBlockAsm64K
|
|
|
|
matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm64K:
|
|
CMPW $0x04, R7
|
|
BLO matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm64K
|
|
MOVWU (R8)(R11), R10
|
|
MOVWU (R9)(R11), R16
|
|
CMPW R10, R16
|
|
BNE matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm64K
|
|
SUB $4, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $4, R11, R11
|
|
MOVWU R11, R11
|
|
|
|
matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm64K:
|
|
CMPW $0x01, R7
|
|
BEQ matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm64K
|
|
BLO match_nolit_end_encodeSnappyBetterBlockAsm64K
|
|
MOVHU (R8)(R11), R16
|
|
BFI $0, R16, $16, R10
|
|
ADD R11, R9, R15
|
|
MOVHU (R15), R15
|
|
AND $0xffff, R10, R16
|
|
CMP R16, R15
|
|
BNE matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm64K
|
|
ADD $2, R11, R11
|
|
MOVWU R11, R11
|
|
SUBSW $0x02, R7, R7
|
|
BEQ match_nolit_end_encodeSnappyBetterBlockAsm64K
|
|
|
|
matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm64K:
|
|
MOVBU (R8)(R11), R16
|
|
BFI $0, R16, $8, R10
|
|
ADD R11, R9, R15
|
|
MOVBU (R15), R15
|
|
AND $0xff, R10, R16
|
|
CMP R16, R15
|
|
BNE match_nolit_end_encodeSnappyBetterBlockAsm64K
|
|
ADD $1, R11, R11
|
|
MOVWU R11, R11
|
|
|
|
match_nolit_end_encodeSnappyBetterBlockAsm64K:
|
|
MOVWU R2, R7
|
|
SUBW R5, R7, R7
|
|
|
|
// Check if repeat
|
|
MOVW R7, 24(RSP)
|
|
MOVWU 20(RSP), R5
|
|
CMPW R6, R5
|
|
BEQ emit_literal_done_match_emit_encodeSnappyBetterBlockAsm64K
|
|
MOVWU R6, R8
|
|
MOVW R6, 20(RSP)
|
|
ADD R5, R3, R9
|
|
SUBW R5, R8, R8
|
|
SUB $1, R8, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x3c, R5
|
|
BLO one_byte_match_emit_encodeSnappyBetterBlockAsm64K
|
|
CMPW $0x00000100, R5
|
|
BLO two_bytes_match_emit_encodeSnappyBetterBlockAsm64K
|
|
BLO three_bytes_match_emit_encodeSnappyBetterBlockAsm64K
|
|
|
|
three_bytes_match_emit_encodeSnappyBetterBlockAsm64K:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm64K
|
|
|
|
two_bytes_match_emit_encodeSnappyBetterBlockAsm64K:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R5, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R5
|
|
BLO memmove_match_emit_encodeSnappyBetterBlockAsm64K
|
|
JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm64K
|
|
|
|
one_byte_match_emit_encodeSnappyBetterBlockAsm64K:
|
|
LSLW $0x02, R5, R16
|
|
BFI $0, R16, $8, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_match_emit_encodeSnappyBetterBlockAsm64K:
|
|
ADD R8, R1, R5
|
|
|
|
// genMemMoveShort
|
|
CMP $0x08, R8
|
|
BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_8
|
|
CMP $0x10, R8
|
|
BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_8through16
|
|
CMP $0x20, R8
|
|
BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_8:
|
|
MOVD (R9), R10
|
|
MOVD R10, (R1)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm64K
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_8through16:
|
|
MOVD (R9), R10
|
|
ADD R8, R9, R15
|
|
MOVD -8(R15), R9
|
|
MOVD R10, (R1)
|
|
ADD R8, R1, R15
|
|
MOVD R9, -8(R15)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm64K
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_17through32:
|
|
FMOVQ (R9), F0
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm64K
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_33through64:
|
|
FMOVQ (R9), F0
|
|
FMOVQ 16(R9), F1
|
|
ADD R8, R9, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm64K:
|
|
MOVD R5, R1
|
|
JMP emit_literal_done_match_emit_encodeSnappyBetterBlockAsm64K
|
|
|
|
memmove_long_match_emit_encodeSnappyBetterBlockAsm64K:
|
|
ADD R8, R1, R5
|
|
|
|
// genMemMoveLong
|
|
MOVD R9, R10
|
|
MOVD R1, R12
|
|
MOVD R8, R13
|
|
|
|
emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm64Klarge_big_loop_back:
|
|
FMOVQ (R10), F0
|
|
FMOVQ 16(R10), F1
|
|
FMOVQ F0, (R12)
|
|
FMOVQ F1, 16(R12)
|
|
ADD $0x20, R10, R10
|
|
ADD $0x20, R12, R12
|
|
SUB $0x20, R13, R13
|
|
CMP $0x20, R13
|
|
BHS emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm64Klarge_big_loop_back
|
|
ADD R8, R9, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R8, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R5, R1
|
|
|
|
emit_literal_done_match_emit_encodeSnappyBetterBlockAsm64K:
|
|
ADDW R11, R2, R2
|
|
ADDW $0x04, R11, R11
|
|
MOVW R2, 20(RSP)
|
|
|
|
// emitCopy
|
|
two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm64K:
|
|
CMPW $0x40, R11
|
|
BLS two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm64K
|
|
MOVD $0xee, R16
|
|
MOVB R16, (R1)
|
|
MOVH R7, 1(R1)
|
|
SUB $60, R11, R11
|
|
MOVWU R11, R11
|
|
ADD $0x03, R1, R1
|
|
JMP two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm64K
|
|
|
|
two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm64K:
|
|
MOVWU R11, R5
|
|
LSLW $0x02, R5, R5
|
|
CMPW $0x0c, R11
|
|
BHS emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm64K
|
|
CMPW $0x00000800, R7
|
|
BHS emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm64K
|
|
SUB $15, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R7, 1(R1)
|
|
LSRW $0x08, R7, R7
|
|
LSLW $0x05, R7, R7
|
|
ORRW R7, R5, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm64K
|
|
|
|
emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm64K:
|
|
SUB $2, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R5, (R1)
|
|
MOVH R7, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
|
|
match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm64K:
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R2
|
|
BHS emit_remainder_encodeSnappyBetterBlockAsm64K
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R1
|
|
BLO match_nolit_dst_ok_encodeSnappyBetterBlockAsm64K
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeSnappyBetterBlockAsm64K:
|
|
MOVD $0x00cf1bbcdcbfa563, R5
|
|
MOVD $0x9e3779b1, R7
|
|
ADD $1, R6, R6
|
|
SUB $2, R2, R8
|
|
MOVD (R3)(R6), R9
|
|
ADD R6, R3, R15
|
|
MOVD 1(R15), R10
|
|
MOVD (R3)(R8), R11
|
|
ADD R8, R3, R15
|
|
MOVD 1(R15), R12
|
|
LSL $0x08, R9, R9
|
|
MUL R5, R9, R9
|
|
LSR $0x30, R9, R9
|
|
LSL $0x20, R10, R10
|
|
MUL R7, R10, R10
|
|
LSR $0x33, R10, R10
|
|
LSL $0x08, R11, R11
|
|
MUL R5, R11, R11
|
|
LSR $0x30, R11, R11
|
|
LSL $0x20, R12, R12
|
|
MUL R7, R12, R12
|
|
LSR $0x33, R12, R12
|
|
ADD $1, R6, R7
|
|
ADD $1, R8, R13
|
|
MOVW R6, (R0)(R9<<2)
|
|
MOVW R8, (R0)(R11<<2)
|
|
ADD R10<<2, R0, R15
|
|
MOVW R7, 262144(R15)
|
|
ADD R12<<2, R0, R15
|
|
MOVW R13, 262144(R15)
|
|
ADD R6, R8, R7
|
|
ADD $1, R7, R7
|
|
LSR $0x01, R7, R7
|
|
ADD $0x01, R6, R6
|
|
SUB $0x01, R8, R8
|
|
|
|
index_loop_encodeSnappyBetterBlockAsm64K:
|
|
CMP R8, R7
|
|
BHS search_loop_encodeSnappyBetterBlockAsm64K
|
|
MOVD (R3)(R6), R9
|
|
MOVD (R3)(R7), R10
|
|
LSL $0x08, R9, R9
|
|
MUL R5, R9, R9
|
|
LSR $0x30, R9, R9
|
|
LSL $0x08, R10, R10
|
|
MUL R5, R10, R10
|
|
LSR $0x30, R10, R10
|
|
MOVW R6, (R0)(R9<<2)
|
|
MOVW R7, (R0)(R10<<2)
|
|
ADD $0x02, R6, R6
|
|
ADD $0x02, R7, R7
|
|
JMP index_loop_encodeSnappyBetterBlockAsm64K
|
|
|
|
emit_remainder_encodeSnappyBetterBlockAsm64K:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R0, R0
|
|
ADD R0, R1, R0
|
|
ADD $3, R0, R0
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R0
|
|
BLO emit_remainder_ok_encodeSnappyBetterBlockAsm64K
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeSnappyBetterBlockAsm64K:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R2
|
|
CMPW R0, R2
|
|
BEQ emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm64K
|
|
MOVWU R0, R5
|
|
MOVW R0, 20(RSP)
|
|
ADD R2, R3, R0
|
|
SUBW R2, R5, R5
|
|
SUB $1, R5, R2
|
|
MOVWU R2, R2
|
|
CMPW $0x3c, R2
|
|
BLO one_byte_emit_remainder_encodeSnappyBetterBlockAsm64K
|
|
CMPW $0x00000100, R2
|
|
BLO two_bytes_emit_remainder_encodeSnappyBetterBlockAsm64K
|
|
BLO three_bytes_emit_remainder_encodeSnappyBetterBlockAsm64K
|
|
|
|
three_bytes_emit_remainder_encodeSnappyBetterBlockAsm64K:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R2, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64K
|
|
|
|
two_bytes_emit_remainder_encodeSnappyBetterBlockAsm64K:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R2, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R2
|
|
BLO memmove_emit_remainder_encodeSnappyBetterBlockAsm64K
|
|
JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64K
|
|
|
|
one_byte_emit_remainder_encodeSnappyBetterBlockAsm64K:
|
|
LSLW $0x02, R2, R16
|
|
BFI $0, R16, $8, R2
|
|
MOVB R2, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_emit_remainder_encodeSnappyBetterBlockAsm64K:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveShort
|
|
CMP $0x03, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_1or2
|
|
BEQ emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_3
|
|
CMP $0x08, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_4through7
|
|
CMP $0x10, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_8through16
|
|
CMP $0x20, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_1or2:
|
|
MOVBU (R0), R16
|
|
BFI $0, R16, $8, R5
|
|
ADD R3, R0, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVB R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVB R0, -1(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm64K
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_3:
|
|
MOVHU (R0), R16
|
|
BFI $0, R16, $16, R5
|
|
MOVBU 2(R0), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVH R5, (R1)
|
|
MOVB R0, 2(R1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm64K
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_4through7:
|
|
MOVWU (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVWU -4(R15), R0
|
|
MOVW R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVW R0, -4(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm64K
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_8through16:
|
|
MOVD (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVD -8(R15), R0
|
|
MOVD R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVD R0, -8(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm64K
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_17through32:
|
|
FMOVQ (R0), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm64K
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_33through64:
|
|
FMOVQ (R0), F0
|
|
FMOVQ 16(R0), F1
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm64K:
|
|
MOVD R2, R1
|
|
JMP emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm64K
|
|
|
|
memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64K:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveLong
|
|
MOVD R0, R5
|
|
MOVD R1, R6
|
|
MOVD R3, R7
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64Klarge_big_loop_back:
|
|
FMOVQ (R5), F0
|
|
FMOVQ 16(R5), F1
|
|
FMOVQ F0, (R6)
|
|
FMOVQ F1, 16(R6)
|
|
ADD $0x20, R5, R5
|
|
ADD $0x20, R6, R6
|
|
SUB $0x20, R7, R7
|
|
CMP $0x20, R7
|
|
BHS emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64Klarge_big_loop_back
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R3, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R2, R1
|
|
|
|
emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm64K:
|
|
MOVD dst_base+0(FP), R0
|
|
SUB R0, R1, R1
|
|
MOVD R1, ret+56(FP)
|
|
RET
|
|
|
|
// func encodeSnappyBetterBlockAsm12B(dst []byte, src []byte, tmp *[81920]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeSnappyBetterBlockAsm12B(SB), $24-64
|
|
MOVD tmp+48(FP), R0
|
|
MOVD dst_base+0(FP), R1
|
|
MOVD $0x00000280, R2
|
|
MOVD R0, R3
|
|
VEOR V0.B16, V0.B16, V0.B16
|
|
|
|
zero_loop_encodeSnappyBetterBlockAsm12B:
|
|
FMOVQ F0, (R3)
|
|
FMOVQ F0, 16(R3)
|
|
FMOVQ F0, 32(R3)
|
|
FMOVQ F0, 48(R3)
|
|
FMOVQ F0, 64(R3)
|
|
FMOVQ F0, 80(R3)
|
|
FMOVQ F0, 96(R3)
|
|
FMOVQ F0, 112(R3)
|
|
ADD $0x80, R3, R3
|
|
SUBS $1, R2, R2
|
|
BNE zero_loop_encodeSnappyBetterBlockAsm12B
|
|
MOVD $0x00000000, R16
|
|
MOVW R16, 20(RSP)
|
|
MOVD src_len+32(FP), R2
|
|
SUB $9, R2, R3
|
|
SUB $8, R2, R5
|
|
MOVW R5, 16(RSP)
|
|
LSR $0x05, R2, R2
|
|
SUBW R2, R3, R3
|
|
ADD R3, R1, R3
|
|
MOVD R3, 8(RSP)
|
|
MOVD $0x00000001, R2
|
|
MOVD $0x00000000, R16
|
|
MOVW R16, 24(RSP)
|
|
MOVD src_base+24(FP), R3
|
|
|
|
search_loop_encodeSnappyBetterBlockAsm12B:
|
|
MOVWU R2, R5
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R5, R5
|
|
LSRW $0x06, R5, R5
|
|
ADD R5, R2, R5
|
|
ADD $1, R5, R5
|
|
MOVWU R5, R5
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R5
|
|
BHS emit_remainder_encodeSnappyBetterBlockAsm12B
|
|
MOVD (R3)(R2), R6
|
|
MOVW R5, 28(RSP)
|
|
MOVD $0x0000cf1bbcdcbf9b, R8
|
|
MOVD $0x9e3779b1, R5
|
|
MOVD R6, R9
|
|
MOVD R6, R10
|
|
LSL $0x10, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x32, R9, R9
|
|
LSL $0x20, R10, R10
|
|
MUL R5, R10, R10
|
|
LSR $0x34, R10, R10
|
|
MOVWU (R0)(R9<<2), R5
|
|
ADD R10<<2, R0, R15
|
|
MOVWU 65536(R15), R7
|
|
MOVW R2, (R0)(R9<<2)
|
|
ADD R10<<2, R0, R15
|
|
MOVW R2, 65536(R15)
|
|
MOVD (R3)(R5), R9
|
|
MOVD (R3)(R7), R10
|
|
CMP R6, R9
|
|
BEQ candidate_match_encodeSnappyBetterBlockAsm12B
|
|
CMP R6, R10
|
|
BNE no_short_found_encodeSnappyBetterBlockAsm12B
|
|
MOVWU R7, R5
|
|
JMP candidate_match_encodeSnappyBetterBlockAsm12B
|
|
|
|
no_short_found_encodeSnappyBetterBlockAsm12B:
|
|
CMPW R6, R9
|
|
BEQ candidate_match_encodeSnappyBetterBlockAsm12B
|
|
CMPW R6, R10
|
|
BEQ candidateS_match_encodeSnappyBetterBlockAsm12B
|
|
MOVWU 28(RSP), R2
|
|
JMP search_loop_encodeSnappyBetterBlockAsm12B
|
|
|
|
candidateS_match_encodeSnappyBetterBlockAsm12B:
|
|
LSR $0x08, R6, R6
|
|
MOVD R6, R9
|
|
LSL $0x10, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x32, R9, R9
|
|
MOVWU (R0)(R9<<2), R5
|
|
ADDW $1, R2, R2
|
|
MOVW R2, (R0)(R9<<2)
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ candidate_match_encodeSnappyBetterBlockAsm12B
|
|
SUBW $1, R2, R2
|
|
MOVWU R7, R5
|
|
|
|
candidate_match_encodeSnappyBetterBlockAsm12B:
|
|
MOVWU 20(RSP), R6
|
|
TSTW R5, R5
|
|
BEQ match_extend_back_end_encodeSnappyBetterBlockAsm12B
|
|
|
|
match_extend_back_loop_encodeSnappyBetterBlockAsm12B:
|
|
CMPW R6, R2
|
|
BLS match_extend_back_end_encodeSnappyBetterBlockAsm12B
|
|
ADD R5, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R7
|
|
ADD R2, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R8
|
|
AND $0xff, R8, R16
|
|
AND $0xff, R7, R15
|
|
CMP R16, R15
|
|
BNE match_extend_back_end_encodeSnappyBetterBlockAsm12B
|
|
SUB $1, R2, R2
|
|
MOVWU R2, R2
|
|
SUBSW $1, R5, R5
|
|
BEQ match_extend_back_end_encodeSnappyBetterBlockAsm12B
|
|
JMP match_extend_back_loop_encodeSnappyBetterBlockAsm12B
|
|
|
|
match_extend_back_end_encodeSnappyBetterBlockAsm12B:
|
|
MOVWU R2, R6
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R6, R6
|
|
ADD R6, R1, R6
|
|
ADD $3, R6, R6
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R6
|
|
BLO match_dst_size_check_encodeSnappyBetterBlockAsm12B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeSnappyBetterBlockAsm12B:
|
|
MOVWU R2, R6
|
|
ADDW $0x04, R2, R2
|
|
ADDW $0x04, R5, R5
|
|
MOVD src_len+32(FP), R7
|
|
SUBW R2, R7, R7
|
|
ADD R2, R3, R8
|
|
ADD R5, R3, R9
|
|
|
|
// matchLen
|
|
MOVD $0, R11
|
|
|
|
matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm12B:
|
|
CMPW $0x10, R7
|
|
BLO matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm12B
|
|
MOVD (R8)(R11), R10
|
|
ADD R11, R8, R15
|
|
MOVD 8(R15), R12
|
|
MOVD (R9)(R11), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm12B
|
|
ADD R11, R9, R15
|
|
MOVD 8(R15), R16
|
|
EOR R16, R12, R12
|
|
TST R12, R12
|
|
BNE matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm12B
|
|
SUB $16, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $16, R11, R11
|
|
MOVWU R11, R11
|
|
JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm12B
|
|
|
|
matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm12B:
|
|
RBIT R12, R12
|
|
CLZ R12, R12
|
|
ASR $0x03, R12, R12
|
|
ADD R12, R11, R11
|
|
ADD $8, R11, R11
|
|
MOVWU R11, R11
|
|
JMP match_nolit_end_encodeSnappyBetterBlockAsm12B
|
|
|
|
matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm12B:
|
|
CMPW $0x08, R7
|
|
BLO matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm12B
|
|
MOVD (R8)(R11), R10
|
|
MOVD (R9)(R11), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm12B
|
|
SUB $8, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $8, R11, R11
|
|
MOVWU R11, R11
|
|
JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm12B
|
|
|
|
matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm12B:
|
|
RBIT R10, R10
|
|
CLZ R10, R10
|
|
ASR $0x03, R10, R10
|
|
ADD R10, R11, R11
|
|
MOVWU R11, R11
|
|
JMP match_nolit_end_encodeSnappyBetterBlockAsm12B
|
|
|
|
matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm12B:
|
|
CMPW $0x04, R7
|
|
BLO matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm12B
|
|
MOVWU (R8)(R11), R10
|
|
MOVWU (R9)(R11), R16
|
|
CMPW R10, R16
|
|
BNE matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm12B
|
|
SUB $4, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $4, R11, R11
|
|
MOVWU R11, R11
|
|
|
|
matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm12B:
|
|
CMPW $0x01, R7
|
|
BEQ matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm12B
|
|
BLO match_nolit_end_encodeSnappyBetterBlockAsm12B
|
|
MOVHU (R8)(R11), R16
|
|
BFI $0, R16, $16, R10
|
|
ADD R11, R9, R15
|
|
MOVHU (R15), R15
|
|
AND $0xffff, R10, R16
|
|
CMP R16, R15
|
|
BNE matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm12B
|
|
ADD $2, R11, R11
|
|
MOVWU R11, R11
|
|
SUBSW $0x02, R7, R7
|
|
BEQ match_nolit_end_encodeSnappyBetterBlockAsm12B
|
|
|
|
matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm12B:
|
|
MOVBU (R8)(R11), R16
|
|
BFI $0, R16, $8, R10
|
|
ADD R11, R9, R15
|
|
MOVBU (R15), R15
|
|
AND $0xff, R10, R16
|
|
CMP R16, R15
|
|
BNE match_nolit_end_encodeSnappyBetterBlockAsm12B
|
|
ADD $1, R11, R11
|
|
MOVWU R11, R11
|
|
|
|
match_nolit_end_encodeSnappyBetterBlockAsm12B:
|
|
MOVWU R2, R7
|
|
SUBW R5, R7, R7
|
|
|
|
// Check if repeat
|
|
MOVW R7, 24(RSP)
|
|
MOVWU 20(RSP), R5
|
|
CMPW R6, R5
|
|
BEQ emit_literal_done_match_emit_encodeSnappyBetterBlockAsm12B
|
|
MOVWU R6, R8
|
|
MOVW R6, 20(RSP)
|
|
ADD R5, R3, R9
|
|
SUBW R5, R8, R8
|
|
SUB $1, R8, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x3c, R5
|
|
BLO one_byte_match_emit_encodeSnappyBetterBlockAsm12B
|
|
CMPW $0x00000100, R5
|
|
BLO two_bytes_match_emit_encodeSnappyBetterBlockAsm12B
|
|
BLO three_bytes_match_emit_encodeSnappyBetterBlockAsm12B
|
|
|
|
three_bytes_match_emit_encodeSnappyBetterBlockAsm12B:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm12B
|
|
|
|
two_bytes_match_emit_encodeSnappyBetterBlockAsm12B:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R5, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R5
|
|
BLO memmove_match_emit_encodeSnappyBetterBlockAsm12B
|
|
JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm12B
|
|
|
|
one_byte_match_emit_encodeSnappyBetterBlockAsm12B:
|
|
LSLW $0x02, R5, R16
|
|
BFI $0, R16, $8, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_match_emit_encodeSnappyBetterBlockAsm12B:
|
|
ADD R8, R1, R5
|
|
|
|
// genMemMoveShort
|
|
CMP $0x08, R8
|
|
BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_8
|
|
CMP $0x10, R8
|
|
BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_8through16
|
|
CMP $0x20, R8
|
|
BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_8:
|
|
MOVD (R9), R10
|
|
MOVD R10, (R1)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_8through16:
|
|
MOVD (R9), R10
|
|
ADD R8, R9, R15
|
|
MOVD -8(R15), R9
|
|
MOVD R10, (R1)
|
|
ADD R8, R1, R15
|
|
MOVD R9, -8(R15)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_17through32:
|
|
FMOVQ (R9), F0
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_33through64:
|
|
FMOVQ (R9), F0
|
|
FMOVQ 16(R9), F1
|
|
ADD R8, R9, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm12B:
|
|
MOVD R5, R1
|
|
JMP emit_literal_done_match_emit_encodeSnappyBetterBlockAsm12B
|
|
|
|
memmove_long_match_emit_encodeSnappyBetterBlockAsm12B:
|
|
ADD R8, R1, R5
|
|
|
|
// genMemMoveLong
|
|
MOVD R9, R10
|
|
MOVD R1, R12
|
|
MOVD R8, R13
|
|
|
|
emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm12Blarge_big_loop_back:
|
|
FMOVQ (R10), F0
|
|
FMOVQ 16(R10), F1
|
|
FMOVQ F0, (R12)
|
|
FMOVQ F1, 16(R12)
|
|
ADD $0x20, R10, R10
|
|
ADD $0x20, R12, R12
|
|
SUB $0x20, R13, R13
|
|
CMP $0x20, R13
|
|
BHS emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm12Blarge_big_loop_back
|
|
ADD R8, R9, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R8, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R5, R1
|
|
|
|
emit_literal_done_match_emit_encodeSnappyBetterBlockAsm12B:
|
|
ADDW R11, R2, R2
|
|
ADDW $0x04, R11, R11
|
|
MOVW R2, 20(RSP)
|
|
|
|
// emitCopy
|
|
two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm12B:
|
|
CMPW $0x40, R11
|
|
BLS two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm12B
|
|
MOVD $0xee, R16
|
|
MOVB R16, (R1)
|
|
MOVH R7, 1(R1)
|
|
SUB $60, R11, R11
|
|
MOVWU R11, R11
|
|
ADD $0x03, R1, R1
|
|
JMP two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm12B
|
|
|
|
two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm12B:
|
|
MOVWU R11, R5
|
|
LSLW $0x02, R5, R5
|
|
CMPW $0x0c, R11
|
|
BHS emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm12B
|
|
CMPW $0x00000800, R7
|
|
BHS emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm12B
|
|
SUB $15, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R7, 1(R1)
|
|
LSRW $0x08, R7, R7
|
|
LSLW $0x05, R7, R7
|
|
ORRW R7, R5, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm12B
|
|
|
|
emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm12B:
|
|
SUB $2, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R5, (R1)
|
|
MOVH R7, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
|
|
match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm12B:
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R2
|
|
BHS emit_remainder_encodeSnappyBetterBlockAsm12B
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R1
|
|
BLO match_nolit_dst_ok_encodeSnappyBetterBlockAsm12B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeSnappyBetterBlockAsm12B:
|
|
MOVD $0x0000cf1bbcdcbf9b, R5
|
|
MOVD $0x9e3779b1, R7
|
|
ADD $1, R6, R6
|
|
SUB $2, R2, R8
|
|
MOVD (R3)(R6), R9
|
|
ADD R6, R3, R15
|
|
MOVD 1(R15), R10
|
|
MOVD (R3)(R8), R11
|
|
ADD R8, R3, R15
|
|
MOVD 1(R15), R12
|
|
LSL $0x10, R9, R9
|
|
MUL R5, R9, R9
|
|
LSR $0x32, R9, R9
|
|
LSL $0x20, R10, R10
|
|
MUL R7, R10, R10
|
|
LSR $0x34, R10, R10
|
|
LSL $0x10, R11, R11
|
|
MUL R5, R11, R11
|
|
LSR $0x32, R11, R11
|
|
LSL $0x20, R12, R12
|
|
MUL R7, R12, R12
|
|
LSR $0x34, R12, R12
|
|
ADD $1, R6, R7
|
|
ADD $1, R8, R13
|
|
MOVW R6, (R0)(R9<<2)
|
|
MOVW R8, (R0)(R11<<2)
|
|
ADD R10<<2, R0, R15
|
|
MOVW R7, 65536(R15)
|
|
ADD R12<<2, R0, R15
|
|
MOVW R13, 65536(R15)
|
|
ADD R6, R8, R7
|
|
ADD $1, R7, R7
|
|
LSR $0x01, R7, R7
|
|
ADD $0x01, R6, R6
|
|
SUB $0x01, R8, R8
|
|
|
|
index_loop_encodeSnappyBetterBlockAsm12B:
|
|
CMP R8, R7
|
|
BHS search_loop_encodeSnappyBetterBlockAsm12B
|
|
MOVD (R3)(R6), R9
|
|
MOVD (R3)(R7), R10
|
|
LSL $0x10, R9, R9
|
|
MUL R5, R9, R9
|
|
LSR $0x32, R9, R9
|
|
LSL $0x10, R10, R10
|
|
MUL R5, R10, R10
|
|
LSR $0x32, R10, R10
|
|
MOVW R6, (R0)(R9<<2)
|
|
MOVW R7, (R0)(R10<<2)
|
|
ADD $0x02, R6, R6
|
|
ADD $0x02, R7, R7
|
|
JMP index_loop_encodeSnappyBetterBlockAsm12B
|
|
|
|
emit_remainder_encodeSnappyBetterBlockAsm12B:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R0, R0
|
|
ADD R0, R1, R0
|
|
ADD $3, R0, R0
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R0
|
|
BLO emit_remainder_ok_encodeSnappyBetterBlockAsm12B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeSnappyBetterBlockAsm12B:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R2
|
|
CMPW R0, R2
|
|
BEQ emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm12B
|
|
MOVWU R0, R5
|
|
MOVW R0, 20(RSP)
|
|
ADD R2, R3, R0
|
|
SUBW R2, R5, R5
|
|
SUB $1, R5, R2
|
|
MOVWU R2, R2
|
|
CMPW $0x3c, R2
|
|
BLO one_byte_emit_remainder_encodeSnappyBetterBlockAsm12B
|
|
CMPW $0x00000100, R2
|
|
BLO two_bytes_emit_remainder_encodeSnappyBetterBlockAsm12B
|
|
BLO three_bytes_emit_remainder_encodeSnappyBetterBlockAsm12B
|
|
|
|
three_bytes_emit_remainder_encodeSnappyBetterBlockAsm12B:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R2, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12B
|
|
|
|
two_bytes_emit_remainder_encodeSnappyBetterBlockAsm12B:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R2, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R2
|
|
BLO memmove_emit_remainder_encodeSnappyBetterBlockAsm12B
|
|
JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12B
|
|
|
|
one_byte_emit_remainder_encodeSnappyBetterBlockAsm12B:
|
|
LSLW $0x02, R2, R16
|
|
BFI $0, R16, $8, R2
|
|
MOVB R2, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_emit_remainder_encodeSnappyBetterBlockAsm12B:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveShort
|
|
CMP $0x03, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_1or2
|
|
BEQ emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_3
|
|
CMP $0x08, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_4through7
|
|
CMP $0x10, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_8through16
|
|
CMP $0x20, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_1or2:
|
|
MOVBU (R0), R16
|
|
BFI $0, R16, $8, R5
|
|
ADD R3, R0, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVB R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVB R0, -1(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_3:
|
|
MOVHU (R0), R16
|
|
BFI $0, R16, $16, R5
|
|
MOVBU 2(R0), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVH R5, (R1)
|
|
MOVB R0, 2(R1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_4through7:
|
|
MOVWU (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVWU -4(R15), R0
|
|
MOVW R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVW R0, -4(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_8through16:
|
|
MOVD (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVD -8(R15), R0
|
|
MOVD R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVD R0, -8(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_17through32:
|
|
FMOVQ (R0), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm12B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_33through64:
|
|
FMOVQ (R0), F0
|
|
FMOVQ 16(R0), F1
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm12B:
|
|
MOVD R2, R1
|
|
JMP emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm12B
|
|
|
|
memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12B:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveLong
|
|
MOVD R0, R5
|
|
MOVD R1, R6
|
|
MOVD R3, R7
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12Blarge_big_loop_back:
|
|
FMOVQ (R5), F0
|
|
FMOVQ 16(R5), F1
|
|
FMOVQ F0, (R6)
|
|
FMOVQ F1, 16(R6)
|
|
ADD $0x20, R5, R5
|
|
ADD $0x20, R6, R6
|
|
SUB $0x20, R7, R7
|
|
CMP $0x20, R7
|
|
BHS emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12Blarge_big_loop_back
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R3, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R2, R1
|
|
|
|
emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm12B:
|
|
MOVD dst_base+0(FP), R0
|
|
SUB R0, R1, R1
|
|
MOVD R1, ret+56(FP)
|
|
RET
|
|
|
|
// func encodeSnappyBetterBlockAsm10B(dst []byte, src []byte, tmp *[20480]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeSnappyBetterBlockAsm10B(SB), $24-64
|
|
MOVD tmp+48(FP), R0
|
|
MOVD dst_base+0(FP), R1
|
|
MOVD $0x000000a0, R2
|
|
MOVD R0, R3
|
|
VEOR V0.B16, V0.B16, V0.B16
|
|
|
|
zero_loop_encodeSnappyBetterBlockAsm10B:
|
|
FMOVQ F0, (R3)
|
|
FMOVQ F0, 16(R3)
|
|
FMOVQ F0, 32(R3)
|
|
FMOVQ F0, 48(R3)
|
|
FMOVQ F0, 64(R3)
|
|
FMOVQ F0, 80(R3)
|
|
FMOVQ F0, 96(R3)
|
|
FMOVQ F0, 112(R3)
|
|
ADD $0x80, R3, R3
|
|
SUBS $1, R2, R2
|
|
BNE zero_loop_encodeSnappyBetterBlockAsm10B
|
|
MOVD $0x00000000, R16
|
|
MOVW R16, 20(RSP)
|
|
MOVD src_len+32(FP), R2
|
|
SUB $9, R2, R3
|
|
SUB $8, R2, R5
|
|
MOVW R5, 16(RSP)
|
|
LSR $0x05, R2, R2
|
|
SUBW R2, R3, R3
|
|
ADD R3, R1, R3
|
|
MOVD R3, 8(RSP)
|
|
MOVD $0x00000001, R2
|
|
MOVD $0x00000000, R16
|
|
MOVW R16, 24(RSP)
|
|
MOVD src_base+24(FP), R3
|
|
|
|
search_loop_encodeSnappyBetterBlockAsm10B:
|
|
MOVWU R2, R5
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R5, R5
|
|
LSRW $0x05, R5, R5
|
|
ADD R5, R2, R5
|
|
ADD $1, R5, R5
|
|
MOVWU R5, R5
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R5
|
|
BHS emit_remainder_encodeSnappyBetterBlockAsm10B
|
|
MOVD (R3)(R2), R6
|
|
MOVW R5, 28(RSP)
|
|
MOVD $0x0000cf1bbcdcbf9b, R8
|
|
MOVD $0x9e3779b1, R5
|
|
MOVD R6, R9
|
|
MOVD R6, R10
|
|
LSL $0x10, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x34, R9, R9
|
|
LSL $0x20, R10, R10
|
|
MUL R5, R10, R10
|
|
LSR $0x36, R10, R10
|
|
MOVWU (R0)(R9<<2), R5
|
|
ADD R10<<2, R0, R15
|
|
MOVWU 16384(R15), R7
|
|
MOVW R2, (R0)(R9<<2)
|
|
ADD R10<<2, R0, R15
|
|
MOVW R2, 16384(R15)
|
|
MOVD (R3)(R5), R9
|
|
MOVD (R3)(R7), R10
|
|
CMP R6, R9
|
|
BEQ candidate_match_encodeSnappyBetterBlockAsm10B
|
|
CMP R6, R10
|
|
BNE no_short_found_encodeSnappyBetterBlockAsm10B
|
|
MOVWU R7, R5
|
|
JMP candidate_match_encodeSnappyBetterBlockAsm10B
|
|
|
|
no_short_found_encodeSnappyBetterBlockAsm10B:
|
|
CMPW R6, R9
|
|
BEQ candidate_match_encodeSnappyBetterBlockAsm10B
|
|
CMPW R6, R10
|
|
BEQ candidateS_match_encodeSnappyBetterBlockAsm10B
|
|
MOVWU 28(RSP), R2
|
|
JMP search_loop_encodeSnappyBetterBlockAsm10B
|
|
|
|
candidateS_match_encodeSnappyBetterBlockAsm10B:
|
|
LSR $0x08, R6, R6
|
|
MOVD R6, R9
|
|
LSL $0x10, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x34, R9, R9
|
|
MOVWU (R0)(R9<<2), R5
|
|
ADDW $1, R2, R2
|
|
MOVW R2, (R0)(R9<<2)
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ candidate_match_encodeSnappyBetterBlockAsm10B
|
|
SUBW $1, R2, R2
|
|
MOVWU R7, R5
|
|
|
|
candidate_match_encodeSnappyBetterBlockAsm10B:
|
|
MOVWU 20(RSP), R6
|
|
TSTW R5, R5
|
|
BEQ match_extend_back_end_encodeSnappyBetterBlockAsm10B
|
|
|
|
match_extend_back_loop_encodeSnappyBetterBlockAsm10B:
|
|
CMPW R6, R2
|
|
BLS match_extend_back_end_encodeSnappyBetterBlockAsm10B
|
|
ADD R5, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R7
|
|
ADD R2, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R8
|
|
AND $0xff, R8, R16
|
|
AND $0xff, R7, R15
|
|
CMP R16, R15
|
|
BNE match_extend_back_end_encodeSnappyBetterBlockAsm10B
|
|
SUB $1, R2, R2
|
|
MOVWU R2, R2
|
|
SUBSW $1, R5, R5
|
|
BEQ match_extend_back_end_encodeSnappyBetterBlockAsm10B
|
|
JMP match_extend_back_loop_encodeSnappyBetterBlockAsm10B
|
|
|
|
match_extend_back_end_encodeSnappyBetterBlockAsm10B:
|
|
MOVWU R2, R6
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R6, R6
|
|
ADD R6, R1, R6
|
|
ADD $3, R6, R6
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R6
|
|
BLO match_dst_size_check_encodeSnappyBetterBlockAsm10B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeSnappyBetterBlockAsm10B:
|
|
MOVWU R2, R6
|
|
ADDW $0x04, R2, R2
|
|
ADDW $0x04, R5, R5
|
|
MOVD src_len+32(FP), R7
|
|
SUBW R2, R7, R7
|
|
ADD R2, R3, R8
|
|
ADD R5, R3, R9
|
|
|
|
// matchLen
|
|
MOVD $0, R11
|
|
|
|
matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm10B:
|
|
CMPW $0x10, R7
|
|
BLO matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm10B
|
|
MOVD (R8)(R11), R10
|
|
ADD R11, R8, R15
|
|
MOVD 8(R15), R12
|
|
MOVD (R9)(R11), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm10B
|
|
ADD R11, R9, R15
|
|
MOVD 8(R15), R16
|
|
EOR R16, R12, R12
|
|
TST R12, R12
|
|
BNE matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm10B
|
|
SUB $16, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $16, R11, R11
|
|
MOVWU R11, R11
|
|
JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm10B
|
|
|
|
matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm10B:
|
|
RBIT R12, R12
|
|
CLZ R12, R12
|
|
ASR $0x03, R12, R12
|
|
ADD R12, R11, R11
|
|
ADD $8, R11, R11
|
|
MOVWU R11, R11
|
|
JMP match_nolit_end_encodeSnappyBetterBlockAsm10B
|
|
|
|
matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm10B:
|
|
CMPW $0x08, R7
|
|
BLO matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm10B
|
|
MOVD (R8)(R11), R10
|
|
MOVD (R9)(R11), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm10B
|
|
SUB $8, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $8, R11, R11
|
|
MOVWU R11, R11
|
|
JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm10B
|
|
|
|
matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm10B:
|
|
RBIT R10, R10
|
|
CLZ R10, R10
|
|
ASR $0x03, R10, R10
|
|
ADD R10, R11, R11
|
|
MOVWU R11, R11
|
|
JMP match_nolit_end_encodeSnappyBetterBlockAsm10B
|
|
|
|
matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm10B:
|
|
CMPW $0x04, R7
|
|
BLO matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm10B
|
|
MOVWU (R8)(R11), R10
|
|
MOVWU (R9)(R11), R16
|
|
CMPW R10, R16
|
|
BNE matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm10B
|
|
SUB $4, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $4, R11, R11
|
|
MOVWU R11, R11
|
|
|
|
matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm10B:
|
|
CMPW $0x01, R7
|
|
BEQ matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm10B
|
|
BLO match_nolit_end_encodeSnappyBetterBlockAsm10B
|
|
MOVHU (R8)(R11), R16
|
|
BFI $0, R16, $16, R10
|
|
ADD R11, R9, R15
|
|
MOVHU (R15), R15
|
|
AND $0xffff, R10, R16
|
|
CMP R16, R15
|
|
BNE matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm10B
|
|
ADD $2, R11, R11
|
|
MOVWU R11, R11
|
|
SUBSW $0x02, R7, R7
|
|
BEQ match_nolit_end_encodeSnappyBetterBlockAsm10B
|
|
|
|
matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm10B:
|
|
MOVBU (R8)(R11), R16
|
|
BFI $0, R16, $8, R10
|
|
ADD R11, R9, R15
|
|
MOVBU (R15), R15
|
|
AND $0xff, R10, R16
|
|
CMP R16, R15
|
|
BNE match_nolit_end_encodeSnappyBetterBlockAsm10B
|
|
ADD $1, R11, R11
|
|
MOVWU R11, R11
|
|
|
|
match_nolit_end_encodeSnappyBetterBlockAsm10B:
|
|
MOVWU R2, R7
|
|
SUBW R5, R7, R7
|
|
|
|
// Check if repeat
|
|
MOVW R7, 24(RSP)
|
|
MOVWU 20(RSP), R5
|
|
CMPW R6, R5
|
|
BEQ emit_literal_done_match_emit_encodeSnappyBetterBlockAsm10B
|
|
MOVWU R6, R8
|
|
MOVW R6, 20(RSP)
|
|
ADD R5, R3, R9
|
|
SUBW R5, R8, R8
|
|
SUB $1, R8, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x3c, R5
|
|
BLO one_byte_match_emit_encodeSnappyBetterBlockAsm10B
|
|
CMPW $0x00000100, R5
|
|
BLO two_bytes_match_emit_encodeSnappyBetterBlockAsm10B
|
|
BLO three_bytes_match_emit_encodeSnappyBetterBlockAsm10B
|
|
|
|
three_bytes_match_emit_encodeSnappyBetterBlockAsm10B:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm10B
|
|
|
|
two_bytes_match_emit_encodeSnappyBetterBlockAsm10B:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R5, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R5
|
|
BLO memmove_match_emit_encodeSnappyBetterBlockAsm10B
|
|
JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm10B
|
|
|
|
one_byte_match_emit_encodeSnappyBetterBlockAsm10B:
|
|
LSLW $0x02, R5, R16
|
|
BFI $0, R16, $8, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_match_emit_encodeSnappyBetterBlockAsm10B:
|
|
ADD R8, R1, R5
|
|
|
|
// genMemMoveShort
|
|
CMP $0x08, R8
|
|
BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_8
|
|
CMP $0x10, R8
|
|
BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_8through16
|
|
CMP $0x20, R8
|
|
BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_8:
|
|
MOVD (R9), R10
|
|
MOVD R10, (R1)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_8through16:
|
|
MOVD (R9), R10
|
|
ADD R8, R9, R15
|
|
MOVD -8(R15), R9
|
|
MOVD R10, (R1)
|
|
ADD R8, R1, R15
|
|
MOVD R9, -8(R15)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_17through32:
|
|
FMOVQ (R9), F0
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_33through64:
|
|
FMOVQ (R9), F0
|
|
FMOVQ 16(R9), F1
|
|
ADD R8, R9, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm10B:
|
|
MOVD R5, R1
|
|
JMP emit_literal_done_match_emit_encodeSnappyBetterBlockAsm10B
|
|
|
|
memmove_long_match_emit_encodeSnappyBetterBlockAsm10B:
|
|
ADD R8, R1, R5
|
|
|
|
// genMemMoveLong
|
|
MOVD R9, R10
|
|
MOVD R1, R12
|
|
MOVD R8, R13
|
|
|
|
emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm10Blarge_big_loop_back:
|
|
FMOVQ (R10), F0
|
|
FMOVQ 16(R10), F1
|
|
FMOVQ F0, (R12)
|
|
FMOVQ F1, 16(R12)
|
|
ADD $0x20, R10, R10
|
|
ADD $0x20, R12, R12
|
|
SUB $0x20, R13, R13
|
|
CMP $0x20, R13
|
|
BHS emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm10Blarge_big_loop_back
|
|
ADD R8, R9, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R8, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R5, R1
|
|
|
|
emit_literal_done_match_emit_encodeSnappyBetterBlockAsm10B:
|
|
ADDW R11, R2, R2
|
|
ADDW $0x04, R11, R11
|
|
MOVW R2, 20(RSP)
|
|
|
|
// emitCopy
|
|
two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm10B:
|
|
CMPW $0x40, R11
|
|
BLS two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm10B
|
|
MOVD $0xee, R16
|
|
MOVB R16, (R1)
|
|
MOVH R7, 1(R1)
|
|
SUB $60, R11, R11
|
|
MOVWU R11, R11
|
|
ADD $0x03, R1, R1
|
|
JMP two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm10B
|
|
|
|
two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm10B:
|
|
MOVWU R11, R5
|
|
LSLW $0x02, R5, R5
|
|
CMPW $0x0c, R11
|
|
BHS emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm10B
|
|
CMPW $0x00000800, R7
|
|
BHS emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm10B
|
|
SUB $15, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R7, 1(R1)
|
|
LSRW $0x08, R7, R7
|
|
LSLW $0x05, R7, R7
|
|
ORRW R7, R5, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm10B
|
|
|
|
emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm10B:
|
|
SUB $2, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R5, (R1)
|
|
MOVH R7, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
|
|
match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm10B:
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R2
|
|
BHS emit_remainder_encodeSnappyBetterBlockAsm10B
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R1
|
|
BLO match_nolit_dst_ok_encodeSnappyBetterBlockAsm10B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeSnappyBetterBlockAsm10B:
|
|
MOVD $0x0000cf1bbcdcbf9b, R5
|
|
MOVD $0x9e3779b1, R7
|
|
ADD $1, R6, R6
|
|
SUB $2, R2, R8
|
|
MOVD (R3)(R6), R9
|
|
ADD R6, R3, R15
|
|
MOVD 1(R15), R10
|
|
MOVD (R3)(R8), R11
|
|
ADD R8, R3, R15
|
|
MOVD 1(R15), R12
|
|
LSL $0x10, R9, R9
|
|
MUL R5, R9, R9
|
|
LSR $0x34, R9, R9
|
|
LSL $0x20, R10, R10
|
|
MUL R7, R10, R10
|
|
LSR $0x36, R10, R10
|
|
LSL $0x10, R11, R11
|
|
MUL R5, R11, R11
|
|
LSR $0x34, R11, R11
|
|
LSL $0x20, R12, R12
|
|
MUL R7, R12, R12
|
|
LSR $0x36, R12, R12
|
|
ADD $1, R6, R7
|
|
ADD $1, R8, R13
|
|
MOVW R6, (R0)(R9<<2)
|
|
MOVW R8, (R0)(R11<<2)
|
|
ADD R10<<2, R0, R15
|
|
MOVW R7, 16384(R15)
|
|
ADD R12<<2, R0, R15
|
|
MOVW R13, 16384(R15)
|
|
ADD R6, R8, R7
|
|
ADD $1, R7, R7
|
|
LSR $0x01, R7, R7
|
|
ADD $0x01, R6, R6
|
|
SUB $0x01, R8, R8
|
|
|
|
index_loop_encodeSnappyBetterBlockAsm10B:
|
|
CMP R8, R7
|
|
BHS search_loop_encodeSnappyBetterBlockAsm10B
|
|
MOVD (R3)(R6), R9
|
|
MOVD (R3)(R7), R10
|
|
LSL $0x10, R9, R9
|
|
MUL R5, R9, R9
|
|
LSR $0x34, R9, R9
|
|
LSL $0x10, R10, R10
|
|
MUL R5, R10, R10
|
|
LSR $0x34, R10, R10
|
|
MOVW R6, (R0)(R9<<2)
|
|
MOVW R7, (R0)(R10<<2)
|
|
ADD $0x02, R6, R6
|
|
ADD $0x02, R7, R7
|
|
JMP index_loop_encodeSnappyBetterBlockAsm10B
|
|
|
|
emit_remainder_encodeSnappyBetterBlockAsm10B:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R0, R0
|
|
ADD R0, R1, R0
|
|
ADD $3, R0, R0
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R0
|
|
BLO emit_remainder_ok_encodeSnappyBetterBlockAsm10B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeSnappyBetterBlockAsm10B:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R2
|
|
CMPW R0, R2
|
|
BEQ emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm10B
|
|
MOVWU R0, R5
|
|
MOVW R0, 20(RSP)
|
|
ADD R2, R3, R0
|
|
SUBW R2, R5, R5
|
|
SUB $1, R5, R2
|
|
MOVWU R2, R2
|
|
CMPW $0x3c, R2
|
|
BLO one_byte_emit_remainder_encodeSnappyBetterBlockAsm10B
|
|
CMPW $0x00000100, R2
|
|
BLO two_bytes_emit_remainder_encodeSnappyBetterBlockAsm10B
|
|
BLO three_bytes_emit_remainder_encodeSnappyBetterBlockAsm10B
|
|
|
|
three_bytes_emit_remainder_encodeSnappyBetterBlockAsm10B:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R2, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10B
|
|
|
|
two_bytes_emit_remainder_encodeSnappyBetterBlockAsm10B:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R2, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R2
|
|
BLO memmove_emit_remainder_encodeSnappyBetterBlockAsm10B
|
|
JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10B
|
|
|
|
one_byte_emit_remainder_encodeSnappyBetterBlockAsm10B:
|
|
LSLW $0x02, R2, R16
|
|
BFI $0, R16, $8, R2
|
|
MOVB R2, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_emit_remainder_encodeSnappyBetterBlockAsm10B:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveShort
|
|
CMP $0x03, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_1or2
|
|
BEQ emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_3
|
|
CMP $0x08, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_4through7
|
|
CMP $0x10, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_8through16
|
|
CMP $0x20, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_1or2:
|
|
MOVBU (R0), R16
|
|
BFI $0, R16, $8, R5
|
|
ADD R3, R0, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVB R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVB R0, -1(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_3:
|
|
MOVHU (R0), R16
|
|
BFI $0, R16, $16, R5
|
|
MOVBU 2(R0), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVH R5, (R1)
|
|
MOVB R0, 2(R1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_4through7:
|
|
MOVWU (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVWU -4(R15), R0
|
|
MOVW R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVW R0, -4(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_8through16:
|
|
MOVD (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVD -8(R15), R0
|
|
MOVD R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVD R0, -8(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_17through32:
|
|
FMOVQ (R0), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm10B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_33through64:
|
|
FMOVQ (R0), F0
|
|
FMOVQ 16(R0), F1
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm10B:
|
|
MOVD R2, R1
|
|
JMP emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm10B
|
|
|
|
memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10B:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveLong
|
|
MOVD R0, R5
|
|
MOVD R1, R6
|
|
MOVD R3, R7
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10Blarge_big_loop_back:
|
|
FMOVQ (R5), F0
|
|
FMOVQ 16(R5), F1
|
|
FMOVQ F0, (R6)
|
|
FMOVQ F1, 16(R6)
|
|
ADD $0x20, R5, R5
|
|
ADD $0x20, R6, R6
|
|
SUB $0x20, R7, R7
|
|
CMP $0x20, R7
|
|
BHS emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10Blarge_big_loop_back
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R3, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R2, R1
|
|
|
|
emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm10B:
|
|
MOVD dst_base+0(FP), R0
|
|
SUB R0, R1, R1
|
|
MOVD R1, ret+56(FP)
|
|
RET
|
|
|
|
// func encodeSnappyBetterBlockAsm8B(dst []byte, src []byte, tmp *[5120]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·encodeSnappyBetterBlockAsm8B(SB), $24-64
|
|
MOVD tmp+48(FP), R0
|
|
MOVD dst_base+0(FP), R1
|
|
MOVD $0x00000028, R2
|
|
MOVD R0, R3
|
|
VEOR V0.B16, V0.B16, V0.B16
|
|
|
|
zero_loop_encodeSnappyBetterBlockAsm8B:
|
|
FMOVQ F0, (R3)
|
|
FMOVQ F0, 16(R3)
|
|
FMOVQ F0, 32(R3)
|
|
FMOVQ F0, 48(R3)
|
|
FMOVQ F0, 64(R3)
|
|
FMOVQ F0, 80(R3)
|
|
FMOVQ F0, 96(R3)
|
|
FMOVQ F0, 112(R3)
|
|
ADD $0x80, R3, R3
|
|
SUBS $1, R2, R2
|
|
BNE zero_loop_encodeSnappyBetterBlockAsm8B
|
|
MOVD $0x00000000, R16
|
|
MOVW R16, 20(RSP)
|
|
MOVD src_len+32(FP), R2
|
|
SUB $9, R2, R3
|
|
SUB $8, R2, R5
|
|
MOVW R5, 16(RSP)
|
|
LSR $0x05, R2, R2
|
|
SUBW R2, R3, R3
|
|
ADD R3, R1, R3
|
|
MOVD R3, 8(RSP)
|
|
MOVD $0x00000001, R2
|
|
MOVD $0x00000000, R16
|
|
MOVW R16, 24(RSP)
|
|
MOVD src_base+24(FP), R3
|
|
|
|
search_loop_encodeSnappyBetterBlockAsm8B:
|
|
MOVWU R2, R5
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R5, R5
|
|
LSRW $0x04, R5, R5
|
|
ADD R5, R2, R5
|
|
ADD $1, R5, R5
|
|
MOVWU R5, R5
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R5
|
|
BHS emit_remainder_encodeSnappyBetterBlockAsm8B
|
|
MOVD (R3)(R2), R6
|
|
MOVW R5, 28(RSP)
|
|
MOVD $0x0000cf1bbcdcbf9b, R8
|
|
MOVD $0x9e3779b1, R5
|
|
MOVD R6, R9
|
|
MOVD R6, R10
|
|
LSL $0x10, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x36, R9, R9
|
|
LSL $0x20, R10, R10
|
|
MUL R5, R10, R10
|
|
LSR $0x38, R10, R10
|
|
MOVWU (R0)(R9<<2), R5
|
|
ADD R10<<2, R0, R15
|
|
MOVWU 4096(R15), R7
|
|
MOVW R2, (R0)(R9<<2)
|
|
ADD R10<<2, R0, R15
|
|
MOVW R2, 4096(R15)
|
|
MOVD (R3)(R5), R9
|
|
MOVD (R3)(R7), R10
|
|
CMP R6, R9
|
|
BEQ candidate_match_encodeSnappyBetterBlockAsm8B
|
|
CMP R6, R10
|
|
BNE no_short_found_encodeSnappyBetterBlockAsm8B
|
|
MOVWU R7, R5
|
|
JMP candidate_match_encodeSnappyBetterBlockAsm8B
|
|
|
|
no_short_found_encodeSnappyBetterBlockAsm8B:
|
|
CMPW R6, R9
|
|
BEQ candidate_match_encodeSnappyBetterBlockAsm8B
|
|
CMPW R6, R10
|
|
BEQ candidateS_match_encodeSnappyBetterBlockAsm8B
|
|
MOVWU 28(RSP), R2
|
|
JMP search_loop_encodeSnappyBetterBlockAsm8B
|
|
|
|
candidateS_match_encodeSnappyBetterBlockAsm8B:
|
|
LSR $0x08, R6, R6
|
|
MOVD R6, R9
|
|
LSL $0x10, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x36, R9, R9
|
|
MOVWU (R0)(R9<<2), R5
|
|
ADDW $1, R2, R2
|
|
MOVW R2, (R0)(R9<<2)
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ candidate_match_encodeSnappyBetterBlockAsm8B
|
|
SUBW $1, R2, R2
|
|
MOVWU R7, R5
|
|
|
|
candidate_match_encodeSnappyBetterBlockAsm8B:
|
|
MOVWU 20(RSP), R6
|
|
TSTW R5, R5
|
|
BEQ match_extend_back_end_encodeSnappyBetterBlockAsm8B
|
|
|
|
match_extend_back_loop_encodeSnappyBetterBlockAsm8B:
|
|
CMPW R6, R2
|
|
BLS match_extend_back_end_encodeSnappyBetterBlockAsm8B
|
|
ADD R5, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R7
|
|
ADD R2, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R8
|
|
AND $0xff, R8, R16
|
|
AND $0xff, R7, R15
|
|
CMP R16, R15
|
|
BNE match_extend_back_end_encodeSnappyBetterBlockAsm8B
|
|
SUB $1, R2, R2
|
|
MOVWU R2, R2
|
|
SUBSW $1, R5, R5
|
|
BEQ match_extend_back_end_encodeSnappyBetterBlockAsm8B
|
|
JMP match_extend_back_loop_encodeSnappyBetterBlockAsm8B
|
|
|
|
match_extend_back_end_encodeSnappyBetterBlockAsm8B:
|
|
MOVWU R2, R6
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R6, R6
|
|
ADD R6, R1, R6
|
|
ADD $3, R6, R6
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R6
|
|
BLO match_dst_size_check_encodeSnappyBetterBlockAsm8B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_dst_size_check_encodeSnappyBetterBlockAsm8B:
|
|
MOVWU R2, R6
|
|
ADDW $0x04, R2, R2
|
|
ADDW $0x04, R5, R5
|
|
MOVD src_len+32(FP), R7
|
|
SUBW R2, R7, R7
|
|
ADD R2, R3, R8
|
|
ADD R5, R3, R9
|
|
|
|
// matchLen
|
|
MOVD $0, R11
|
|
|
|
matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm8B:
|
|
CMPW $0x10, R7
|
|
BLO matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm8B
|
|
MOVD (R8)(R11), R10
|
|
ADD R11, R8, R15
|
|
MOVD 8(R15), R12
|
|
MOVD (R9)(R11), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm8B
|
|
ADD R11, R9, R15
|
|
MOVD 8(R15), R16
|
|
EOR R16, R12, R12
|
|
TST R12, R12
|
|
BNE matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm8B
|
|
SUB $16, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $16, R11, R11
|
|
MOVWU R11, R11
|
|
JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm8B
|
|
|
|
matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm8B:
|
|
RBIT R12, R12
|
|
CLZ R12, R12
|
|
ASR $0x03, R12, R12
|
|
ADD R12, R11, R11
|
|
ADD $8, R11, R11
|
|
MOVWU R11, R11
|
|
JMP match_nolit_end_encodeSnappyBetterBlockAsm8B
|
|
|
|
matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm8B:
|
|
CMPW $0x08, R7
|
|
BLO matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm8B
|
|
MOVD (R8)(R11), R10
|
|
MOVD (R9)(R11), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm8B
|
|
SUB $8, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $8, R11, R11
|
|
MOVWU R11, R11
|
|
JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm8B
|
|
|
|
matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm8B:
|
|
RBIT R10, R10
|
|
CLZ R10, R10
|
|
ASR $0x03, R10, R10
|
|
ADD R10, R11, R11
|
|
MOVWU R11, R11
|
|
JMP match_nolit_end_encodeSnappyBetterBlockAsm8B
|
|
|
|
matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm8B:
|
|
CMPW $0x04, R7
|
|
BLO matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm8B
|
|
MOVWU (R8)(R11), R10
|
|
MOVWU (R9)(R11), R16
|
|
CMPW R10, R16
|
|
BNE matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm8B
|
|
SUB $4, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $4, R11, R11
|
|
MOVWU R11, R11
|
|
|
|
matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm8B:
|
|
CMPW $0x01, R7
|
|
BEQ matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm8B
|
|
BLO match_nolit_end_encodeSnappyBetterBlockAsm8B
|
|
MOVHU (R8)(R11), R16
|
|
BFI $0, R16, $16, R10
|
|
ADD R11, R9, R15
|
|
MOVHU (R15), R15
|
|
AND $0xffff, R10, R16
|
|
CMP R16, R15
|
|
BNE matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm8B
|
|
ADD $2, R11, R11
|
|
MOVWU R11, R11
|
|
SUBSW $0x02, R7, R7
|
|
BEQ match_nolit_end_encodeSnappyBetterBlockAsm8B
|
|
|
|
matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm8B:
|
|
MOVBU (R8)(R11), R16
|
|
BFI $0, R16, $8, R10
|
|
ADD R11, R9, R15
|
|
MOVBU (R15), R15
|
|
AND $0xff, R10, R16
|
|
CMP R16, R15
|
|
BNE match_nolit_end_encodeSnappyBetterBlockAsm8B
|
|
ADD $1, R11, R11
|
|
MOVWU R11, R11
|
|
|
|
match_nolit_end_encodeSnappyBetterBlockAsm8B:
|
|
MOVWU R2, R7
|
|
SUBW R5, R7, R7
|
|
|
|
// Check if repeat
|
|
MOVW R7, 24(RSP)
|
|
MOVWU 20(RSP), R5
|
|
CMPW R6, R5
|
|
BEQ emit_literal_done_match_emit_encodeSnappyBetterBlockAsm8B
|
|
MOVWU R6, R8
|
|
MOVW R6, 20(RSP)
|
|
ADD R5, R3, R9
|
|
SUBW R5, R8, R8
|
|
SUB $1, R8, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x3c, R5
|
|
BLO one_byte_match_emit_encodeSnappyBetterBlockAsm8B
|
|
CMPW $0x00000100, R5
|
|
BLO two_bytes_match_emit_encodeSnappyBetterBlockAsm8B
|
|
BLO three_bytes_match_emit_encodeSnappyBetterBlockAsm8B
|
|
|
|
three_bytes_match_emit_encodeSnappyBetterBlockAsm8B:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R5, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm8B
|
|
|
|
two_bytes_match_emit_encodeSnappyBetterBlockAsm8B:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R5, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R5
|
|
BLO memmove_match_emit_encodeSnappyBetterBlockAsm8B
|
|
JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm8B
|
|
|
|
one_byte_match_emit_encodeSnappyBetterBlockAsm8B:
|
|
LSLW $0x02, R5, R16
|
|
BFI $0, R16, $8, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_match_emit_encodeSnappyBetterBlockAsm8B:
|
|
ADD R8, R1, R5
|
|
|
|
// genMemMoveShort
|
|
CMP $0x08, R8
|
|
BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_8
|
|
CMP $0x10, R8
|
|
BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_8through16
|
|
CMP $0x20, R8
|
|
BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_17through32
|
|
JMP emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_8:
|
|
MOVD (R9), R10
|
|
MOVD R10, (R1)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_8through16:
|
|
MOVD (R9), R10
|
|
ADD R8, R9, R15
|
|
MOVD -8(R15), R9
|
|
MOVD R10, (R1)
|
|
ADD R8, R1, R15
|
|
MOVD R9, -8(R15)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_17through32:
|
|
FMOVQ (R9), F0
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_33through64:
|
|
FMOVQ (R9), F0
|
|
FMOVQ 16(R9), F1
|
|
ADD R8, R9, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R8, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm8B:
|
|
MOVD R5, R1
|
|
JMP emit_literal_done_match_emit_encodeSnappyBetterBlockAsm8B
|
|
|
|
memmove_long_match_emit_encodeSnappyBetterBlockAsm8B:
|
|
ADD R8, R1, R5
|
|
|
|
// genMemMoveLong
|
|
MOVD R9, R10
|
|
MOVD R1, R12
|
|
MOVD R8, R13
|
|
|
|
emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm8Blarge_big_loop_back:
|
|
FMOVQ (R10), F0
|
|
FMOVQ 16(R10), F1
|
|
FMOVQ F0, (R12)
|
|
FMOVQ F1, 16(R12)
|
|
ADD $0x20, R10, R10
|
|
ADD $0x20, R12, R12
|
|
SUB $0x20, R13, R13
|
|
CMP $0x20, R13
|
|
BHS emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm8Blarge_big_loop_back
|
|
ADD R8, R9, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R8, R9, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R8, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R8, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R5, R1
|
|
|
|
emit_literal_done_match_emit_encodeSnappyBetterBlockAsm8B:
|
|
ADDW R11, R2, R2
|
|
ADDW $0x04, R11, R11
|
|
MOVW R2, 20(RSP)
|
|
|
|
// emitCopy
|
|
two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm8B:
|
|
CMPW $0x40, R11
|
|
BLS two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm8B
|
|
MOVD $0xee, R16
|
|
MOVB R16, (R1)
|
|
MOVH R7, 1(R1)
|
|
SUB $60, R11, R11
|
|
MOVWU R11, R11
|
|
ADD $0x03, R1, R1
|
|
JMP two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm8B
|
|
|
|
two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm8B:
|
|
MOVWU R11, R5
|
|
LSLW $0x02, R5, R5
|
|
CMPW $0x0c, R11
|
|
BHS emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm8B
|
|
SUB $15, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R7, 1(R1)
|
|
LSRW $0x08, R7, R7
|
|
LSLW $0x05, R7, R7
|
|
ORRW R7, R5, R5
|
|
MOVB R5, (R1)
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm8B
|
|
|
|
emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm8B:
|
|
SUB $2, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R5, (R1)
|
|
MOVH R7, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
|
|
match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm8B:
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R2
|
|
BHS emit_remainder_encodeSnappyBetterBlockAsm8B
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R1
|
|
BLO match_nolit_dst_ok_encodeSnappyBetterBlockAsm8B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_encodeSnappyBetterBlockAsm8B:
|
|
MOVD $0x0000cf1bbcdcbf9b, R5
|
|
MOVD $0x9e3779b1, R7
|
|
ADD $1, R6, R6
|
|
SUB $2, R2, R8
|
|
MOVD (R3)(R6), R9
|
|
ADD R6, R3, R15
|
|
MOVD 1(R15), R10
|
|
MOVD (R3)(R8), R11
|
|
ADD R8, R3, R15
|
|
MOVD 1(R15), R12
|
|
LSL $0x10, R9, R9
|
|
MUL R5, R9, R9
|
|
LSR $0x36, R9, R9
|
|
LSL $0x20, R10, R10
|
|
MUL R7, R10, R10
|
|
LSR $0x38, R10, R10
|
|
LSL $0x10, R11, R11
|
|
MUL R5, R11, R11
|
|
LSR $0x36, R11, R11
|
|
LSL $0x20, R12, R12
|
|
MUL R7, R12, R12
|
|
LSR $0x38, R12, R12
|
|
ADD $1, R6, R7
|
|
ADD $1, R8, R13
|
|
MOVW R6, (R0)(R9<<2)
|
|
MOVW R8, (R0)(R11<<2)
|
|
ADD R10<<2, R0, R15
|
|
MOVW R7, 4096(R15)
|
|
ADD R12<<2, R0, R15
|
|
MOVW R13, 4096(R15)
|
|
ADD R6, R8, R7
|
|
ADD $1, R7, R7
|
|
LSR $0x01, R7, R7
|
|
ADD $0x01, R6, R6
|
|
SUB $0x01, R8, R8
|
|
|
|
index_loop_encodeSnappyBetterBlockAsm8B:
|
|
CMP R8, R7
|
|
BHS search_loop_encodeSnappyBetterBlockAsm8B
|
|
MOVD (R3)(R6), R9
|
|
MOVD (R3)(R7), R10
|
|
LSL $0x10, R9, R9
|
|
MUL R5, R9, R9
|
|
LSR $0x36, R9, R9
|
|
LSL $0x10, R10, R10
|
|
MUL R5, R10, R10
|
|
LSR $0x36, R10, R10
|
|
MOVW R6, (R0)(R9<<2)
|
|
MOVW R7, (R0)(R10<<2)
|
|
ADD $0x02, R6, R6
|
|
ADD $0x02, R7, R7
|
|
JMP index_loop_encodeSnappyBetterBlockAsm8B
|
|
|
|
emit_remainder_encodeSnappyBetterBlockAsm8B:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R0, R0
|
|
ADD R0, R1, R0
|
|
ADD $3, R0, R0
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R0
|
|
BLO emit_remainder_ok_encodeSnappyBetterBlockAsm8B
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+56(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_encodeSnappyBetterBlockAsm8B:
|
|
MOVD src_len+32(FP), R0
|
|
MOVWU 20(RSP), R2
|
|
CMPW R0, R2
|
|
BEQ emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm8B
|
|
MOVWU R0, R5
|
|
MOVW R0, 20(RSP)
|
|
ADD R2, R3, R0
|
|
SUBW R2, R5, R5
|
|
SUB $1, R5, R2
|
|
MOVWU R2, R2
|
|
CMPW $0x3c, R2
|
|
BLO one_byte_emit_remainder_encodeSnappyBetterBlockAsm8B
|
|
CMPW $0x00000100, R2
|
|
BLO two_bytes_emit_remainder_encodeSnappyBetterBlockAsm8B
|
|
BLO three_bytes_emit_remainder_encodeSnappyBetterBlockAsm8B
|
|
|
|
three_bytes_emit_remainder_encodeSnappyBetterBlockAsm8B:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R1)
|
|
MOVH R2, 1(R1)
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8B
|
|
|
|
two_bytes_emit_remainder_encodeSnappyBetterBlockAsm8B:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R1)
|
|
MOVB R2, 1(R1)
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R2
|
|
BLO memmove_emit_remainder_encodeSnappyBetterBlockAsm8B
|
|
JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8B
|
|
|
|
one_byte_emit_remainder_encodeSnappyBetterBlockAsm8B:
|
|
LSLW $0x02, R2, R16
|
|
BFI $0, R16, $8, R2
|
|
MOVB R2, (R1)
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_emit_remainder_encodeSnappyBetterBlockAsm8B:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveShort
|
|
CMP $0x03, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_1or2
|
|
BEQ emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_3
|
|
CMP $0x08, R3
|
|
BLO emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_4through7
|
|
CMP $0x10, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_8through16
|
|
CMP $0x20, R3
|
|
BLS emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_17through32
|
|
JMP emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_33through64
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_1or2:
|
|
MOVBU (R0), R16
|
|
BFI $0, R16, $8, R5
|
|
ADD R3, R0, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVB R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVB R0, -1(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_3:
|
|
MOVHU (R0), R16
|
|
BFI $0, R16, $16, R5
|
|
MOVBU 2(R0), R16
|
|
BFI $0, R16, $8, R0
|
|
MOVH R5, (R1)
|
|
MOVB R0, 2(R1)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_4through7:
|
|
MOVWU (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVWU -4(R15), R0
|
|
MOVW R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVW R0, -4(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_8through16:
|
|
MOVD (R0), R5
|
|
ADD R3, R0, R15
|
|
MOVD -8(R15), R0
|
|
MOVD R5, (R1)
|
|
ADD R3, R1, R15
|
|
MOVD R0, -8(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_17through32:
|
|
FMOVQ (R0), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm8B
|
|
|
|
emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_33through64:
|
|
FMOVQ (R0), F0
|
|
FMOVQ 16(R0), F1
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R1)
|
|
FMOVQ F1, 16(R1)
|
|
ADD R3, R1, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm8B:
|
|
MOVD R2, R1
|
|
JMP emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm8B
|
|
|
|
memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8B:
|
|
ADD R5, R1, R2
|
|
MOVWU R5, R3
|
|
|
|
// genMemMoveLong
|
|
MOVD R0, R5
|
|
MOVD R1, R6
|
|
MOVD R3, R7
|
|
|
|
emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8Blarge_big_loop_back:
|
|
FMOVQ (R5), F0
|
|
FMOVQ 16(R5), F1
|
|
FMOVQ F0, (R6)
|
|
FMOVQ F1, 16(R6)
|
|
ADD $0x20, R5, R5
|
|
ADD $0x20, R6, R6
|
|
SUB $0x20, R7, R7
|
|
CMP $0x20, R7
|
|
BHS emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8Blarge_big_loop_back
|
|
ADD R3, R0, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R3, R0, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R3, R1, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R3, R1, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R2, R1
|
|
|
|
emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm8B:
|
|
MOVD dst_base+0(FP), R0
|
|
SUB R0, R1, R1
|
|
MOVD R1, ret+56(FP)
|
|
RET
|
|
|
|
// func calcBlockSize(src []byte, tmp *[32768]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·calcBlockSize(SB), $24-40
|
|
MOVD tmp+24(FP), R0
|
|
MOVD $0, R1
|
|
MOVD $0x00000100, R2
|
|
MOVD R0, R3
|
|
VEOR V0.B16, V0.B16, V0.B16
|
|
|
|
zero_loop_calcBlockSize:
|
|
FMOVQ F0, (R3)
|
|
FMOVQ F0, 16(R3)
|
|
FMOVQ F0, 32(R3)
|
|
FMOVQ F0, 48(R3)
|
|
FMOVQ F0, 64(R3)
|
|
FMOVQ F0, 80(R3)
|
|
FMOVQ F0, 96(R3)
|
|
FMOVQ F0, 112(R3)
|
|
ADD $0x80, R3, R3
|
|
SUBS $1, R2, R2
|
|
BNE zero_loop_calcBlockSize
|
|
MOVD $0x00000000, R16
|
|
MOVW R16, 20(RSP)
|
|
MOVD src_len+8(FP), R2
|
|
SUB $9, R2, R3
|
|
SUB $8, R2, R5
|
|
MOVW R5, 16(RSP)
|
|
LSR $0x05, R2, R2
|
|
SUBW R2, R3, R3
|
|
ADD R3, R1, R3
|
|
MOVD R3, 8(RSP)
|
|
MOVD $0x00000001, R2
|
|
MOVW R2, 24(RSP)
|
|
MOVD src_base+0(FP), R3
|
|
|
|
search_loop_calcBlockSize:
|
|
MOVWU R2, R5
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R5, R5
|
|
LSRW $0x05, R5, R5
|
|
ADD R5, R2, R5
|
|
ADD $4, R5, R5
|
|
MOVWU R5, R5
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R5
|
|
BHS emit_remainder_calcBlockSize
|
|
MOVD (R3)(R2), R6
|
|
MOVW R5, 28(RSP)
|
|
MOVD $0x0000cf1bbcdcbf9b, R8
|
|
MOVD R6, R9
|
|
MOVD R6, R10
|
|
LSR $0x08, R10, R10
|
|
LSL $0x10, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x33, R9, R9
|
|
LSL $0x10, R10, R10
|
|
MUL R8, R10, R10
|
|
LSR $0x33, R10, R10
|
|
MOVWU (R0)(R9<<2), R5
|
|
MOVWU (R0)(R10<<2), R7
|
|
MOVW R2, (R0)(R9<<2)
|
|
ADD $1, R2, R9
|
|
MOVWU R9, R9
|
|
MOVW R9, (R0)(R10<<2)
|
|
MOVD R6, R9
|
|
LSR $0x10, R9, R9
|
|
LSL $0x10, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x33, R9, R9
|
|
MOVWU R2, R8
|
|
MOVWU 24(RSP), R16
|
|
SUBW R16, R8, R8
|
|
ADD R8, R3, R15
|
|
MOVWU 1(R15), R10
|
|
MOVD R6, R8
|
|
LSR $0x08, R8, R8
|
|
CMPW R10, R8
|
|
BNE no_repeat_found_calcBlockSize
|
|
ADD $1, R2, R6
|
|
MOVWU R6, R6
|
|
MOVWU 20(RSP), R5
|
|
MOVWU R6, R7
|
|
MOVWU 24(RSP), R16
|
|
SUBSW R16, R7, R7
|
|
BEQ repeat_extend_back_end_calcBlockSize
|
|
|
|
repeat_extend_back_loop_calcBlockSize:
|
|
CMPW R5, R6
|
|
BLS repeat_extend_back_end_calcBlockSize
|
|
ADD R7, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R8
|
|
ADD R6, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R9
|
|
AND $0xff, R9, R16
|
|
AND $0xff, R8, R15
|
|
CMP R16, R15
|
|
BNE repeat_extend_back_end_calcBlockSize
|
|
SUB $1, R6, R6
|
|
MOVWU R6, R6
|
|
SUBSW $1, R7, R7
|
|
BNE repeat_extend_back_loop_calcBlockSize
|
|
|
|
repeat_extend_back_end_calcBlockSize:
|
|
MOVWU R6, R5
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R5, R5
|
|
ADD R5, R1, R5
|
|
ADD $5, R5, R5
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R5
|
|
BLO repeat_dst_size_check_calcBlockSize
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+32(FP)
|
|
RET
|
|
|
|
repeat_dst_size_check_calcBlockSize:
|
|
MOVWU 20(RSP), R5
|
|
CMPW R6, R5
|
|
BEQ emit_literal_done_repeat_emit_calcBlockSize
|
|
MOVWU R6, R7
|
|
MOVW R6, 20(RSP)
|
|
ADD R5, R3, R8
|
|
SUBW R5, R7, R7
|
|
SUB $1, R7, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x3c, R5
|
|
BLO one_byte_repeat_emit_calcBlockSize
|
|
CMPW $0x00000100, R5
|
|
BLO two_bytes_repeat_emit_calcBlockSize
|
|
CMPW $0x00010000, R5
|
|
BLO three_bytes_repeat_emit_calcBlockSize
|
|
CMPW $0x01000000, R5
|
|
BLO four_bytes_repeat_emit_calcBlockSize
|
|
ADD $0x05, R1, R1
|
|
JMP memmove_long_repeat_emit_calcBlockSize
|
|
|
|
four_bytes_repeat_emit_calcBlockSize:
|
|
ADD $0x04, R1, R1
|
|
JMP memmove_long_repeat_emit_calcBlockSize
|
|
|
|
three_bytes_repeat_emit_calcBlockSize:
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_repeat_emit_calcBlockSize
|
|
|
|
two_bytes_repeat_emit_calcBlockSize:
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R5
|
|
BLO memmove_repeat_emit_calcBlockSize
|
|
JMP memmove_long_repeat_emit_calcBlockSize
|
|
|
|
one_byte_repeat_emit_calcBlockSize:
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_repeat_emit_calcBlockSize:
|
|
ADD R7, R1, R1
|
|
JMP emit_literal_done_repeat_emit_calcBlockSize
|
|
|
|
memmove_long_repeat_emit_calcBlockSize:
|
|
ADD R7, R1, R1
|
|
|
|
emit_literal_done_repeat_emit_calcBlockSize:
|
|
ADDW $0x05, R2, R2
|
|
MOVWU R2, R5
|
|
MOVWU 24(RSP), R16
|
|
SUBW R16, R5, R5
|
|
MOVD src_len+8(FP), R7
|
|
SUBW R2, R7, R7
|
|
ADD R2, R3, R8
|
|
ADD R5, R3, R5
|
|
|
|
// matchLen
|
|
MOVD $0, R10
|
|
|
|
matchlen_loopback_16_repeat_extend_calcBlockSize:
|
|
CMPW $0x10, R7
|
|
BLO matchlen_match8_repeat_extend_calcBlockSize
|
|
MOVD (R8)(R10), R9
|
|
ADD R10, R8, R15
|
|
MOVD 8(R15), R11
|
|
MOVD (R5)(R10), R16
|
|
EOR R16, R9, R9
|
|
TST R9, R9
|
|
BNE matchlen_bsf_8_repeat_extend_calcBlockSize
|
|
ADD R10, R5, R15
|
|
MOVD 8(R15), R16
|
|
EOR R16, R11, R11
|
|
TST R11, R11
|
|
BNE matchlen_bsf_16repeat_extend_calcBlockSize
|
|
SUB $16, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $16, R10, R10
|
|
MOVWU R10, R10
|
|
JMP matchlen_loopback_16_repeat_extend_calcBlockSize
|
|
|
|
matchlen_bsf_16repeat_extend_calcBlockSize:
|
|
RBIT R11, R11
|
|
CLZ R11, R11
|
|
ASR $0x03, R11, R11
|
|
ADD R11, R10, R10
|
|
ADD $8, R10, R10
|
|
MOVWU R10, R10
|
|
JMP repeat_extend_forward_end_calcBlockSize
|
|
|
|
matchlen_match8_repeat_extend_calcBlockSize:
|
|
CMPW $0x08, R7
|
|
BLO matchlen_match4_repeat_extend_calcBlockSize
|
|
MOVD (R8)(R10), R9
|
|
MOVD (R5)(R10), R16
|
|
EOR R16, R9, R9
|
|
TST R9, R9
|
|
BNE matchlen_bsf_8_repeat_extend_calcBlockSize
|
|
SUB $8, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $8, R10, R10
|
|
MOVWU R10, R10
|
|
JMP matchlen_match4_repeat_extend_calcBlockSize
|
|
|
|
matchlen_bsf_8_repeat_extend_calcBlockSize:
|
|
RBIT R9, R9
|
|
CLZ R9, R9
|
|
ASR $0x03, R9, R9
|
|
ADD R9, R10, R10
|
|
MOVWU R10, R10
|
|
JMP repeat_extend_forward_end_calcBlockSize
|
|
|
|
matchlen_match4_repeat_extend_calcBlockSize:
|
|
CMPW $0x04, R7
|
|
BLO matchlen_match2_repeat_extend_calcBlockSize
|
|
MOVWU (R8)(R10), R9
|
|
MOVWU (R5)(R10), R16
|
|
CMPW R9, R16
|
|
BNE matchlen_match2_repeat_extend_calcBlockSize
|
|
SUB $4, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $4, R10, R10
|
|
MOVWU R10, R10
|
|
|
|
matchlen_match2_repeat_extend_calcBlockSize:
|
|
CMPW $0x01, R7
|
|
BEQ matchlen_match1_repeat_extend_calcBlockSize
|
|
BLO repeat_extend_forward_end_calcBlockSize
|
|
MOVHU (R8)(R10), R16
|
|
BFI $0, R16, $16, R9
|
|
ADD R10, R5, R15
|
|
MOVHU (R15), R15
|
|
AND $0xffff, R9, R16
|
|
CMP R16, R15
|
|
BNE matchlen_match1_repeat_extend_calcBlockSize
|
|
ADD $2, R10, R10
|
|
MOVWU R10, R10
|
|
SUBSW $0x02, R7, R7
|
|
BEQ repeat_extend_forward_end_calcBlockSize
|
|
|
|
matchlen_match1_repeat_extend_calcBlockSize:
|
|
MOVBU (R8)(R10), R16
|
|
BFI $0, R16, $8, R9
|
|
ADD R10, R5, R15
|
|
MOVBU (R15), R15
|
|
AND $0xff, R9, R16
|
|
CMP R16, R15
|
|
BNE repeat_extend_forward_end_calcBlockSize
|
|
ADD $1, R10, R10
|
|
MOVWU R10, R10
|
|
|
|
repeat_extend_forward_end_calcBlockSize:
|
|
ADDW R10, R2, R2
|
|
MOVWU R2, R5
|
|
SUBW R6, R5, R5
|
|
MOVWU 24(RSP), R6
|
|
|
|
// emitCopy
|
|
CMPW $0x00010000, R6
|
|
BLO two_byte_offset_repeat_as_copy_calcBlockSize
|
|
|
|
four_bytes_loop_back_repeat_as_copy_calcBlockSize:
|
|
CMPW $0x40, R5
|
|
BLS four_bytes_remain_repeat_as_copy_calcBlockSize
|
|
SUB $64, R5, R5
|
|
MOVWU R5, R5
|
|
ADD $0x05, R1, R1
|
|
CMPW $0x04, R5
|
|
BLO four_bytes_remain_repeat_as_copy_calcBlockSize
|
|
JMP four_bytes_loop_back_repeat_as_copy_calcBlockSize
|
|
|
|
four_bytes_remain_repeat_as_copy_calcBlockSize:
|
|
TSTW R5, R5
|
|
BEQ repeat_end_emit_calcBlockSize
|
|
MOVD $0, R5
|
|
ADD $0x05, R1, R1
|
|
JMP repeat_end_emit_calcBlockSize
|
|
|
|
two_byte_offset_repeat_as_copy_calcBlockSize:
|
|
CMPW $0x40, R5
|
|
BLS two_byte_offset_short_repeat_as_copy_calcBlockSize
|
|
SUB $60, R5, R5
|
|
MOVWU R5, R5
|
|
ADD $0x03, R1, R1
|
|
JMP two_byte_offset_repeat_as_copy_calcBlockSize
|
|
|
|
two_byte_offset_short_repeat_as_copy_calcBlockSize:
|
|
MOVWU R5, R7
|
|
LSLW $0x02, R7, R7
|
|
CMPW $0x0c, R5
|
|
BHS emit_copy_three_repeat_as_copy_calcBlockSize
|
|
CMPW $0x00000800, R6
|
|
BHS emit_copy_three_repeat_as_copy_calcBlockSize
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_calcBlockSize
|
|
|
|
emit_copy_three_repeat_as_copy_calcBlockSize:
|
|
ADD $0x03, R1, R1
|
|
|
|
repeat_end_emit_calcBlockSize:
|
|
MOVW R2, 20(RSP)
|
|
JMP search_loop_calcBlockSize
|
|
|
|
no_repeat_found_calcBlockSize:
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ candidate_match_calcBlockSize
|
|
LSR $0x08, R6, R6
|
|
MOVWU (R0)(R9<<2), R5
|
|
ADD $2, R2, R8
|
|
MOVWU R8, R8
|
|
MOVWU (R3)(R7), R16
|
|
CMPW R6, R16
|
|
BEQ candidate2_match_calcBlockSize
|
|
MOVW R8, (R0)(R9<<2)
|
|
LSR $0x08, R6, R6
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ candidate3_match_calcBlockSize
|
|
MOVWU 28(RSP), R2
|
|
JMP search_loop_calcBlockSize
|
|
|
|
candidate3_match_calcBlockSize:
|
|
ADDW $0x02, R2, R2
|
|
JMP candidate_match_calcBlockSize
|
|
|
|
candidate2_match_calcBlockSize:
|
|
MOVW R8, (R0)(R9<<2)
|
|
ADDW $1, R2, R2
|
|
MOVWU R7, R5
|
|
|
|
candidate_match_calcBlockSize:
|
|
MOVWU 20(RSP), R6
|
|
TSTW R5, R5
|
|
BEQ match_extend_back_end_calcBlockSize
|
|
|
|
match_extend_back_loop_calcBlockSize:
|
|
CMPW R6, R2
|
|
BLS match_extend_back_end_calcBlockSize
|
|
ADD R5, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R7
|
|
ADD R2, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R8
|
|
AND $0xff, R8, R16
|
|
AND $0xff, R7, R15
|
|
CMP R16, R15
|
|
BNE match_extend_back_end_calcBlockSize
|
|
SUB $1, R2, R2
|
|
MOVWU R2, R2
|
|
SUBSW $1, R5, R5
|
|
BEQ match_extend_back_end_calcBlockSize
|
|
JMP match_extend_back_loop_calcBlockSize
|
|
|
|
match_extend_back_end_calcBlockSize:
|
|
MOVWU R2, R6
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R6, R6
|
|
ADD R6, R1, R6
|
|
ADD $5, R6, R6
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R6
|
|
BLO match_dst_size_check_calcBlockSize
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+32(FP)
|
|
RET
|
|
|
|
match_dst_size_check_calcBlockSize:
|
|
MOVWU R2, R6
|
|
MOVWU 20(RSP), R7
|
|
CMPW R6, R7
|
|
BEQ emit_literal_done_match_emit_calcBlockSize
|
|
MOVWU R6, R8
|
|
MOVW R6, 20(RSP)
|
|
ADD R7, R3, R6
|
|
SUBW R7, R8, R8
|
|
SUB $1, R8, R6
|
|
MOVWU R6, R6
|
|
CMPW $0x3c, R6
|
|
BLO one_byte_match_emit_calcBlockSize
|
|
CMPW $0x00000100, R6
|
|
BLO two_bytes_match_emit_calcBlockSize
|
|
CMPW $0x00010000, R6
|
|
BLO three_bytes_match_emit_calcBlockSize
|
|
CMPW $0x01000000, R6
|
|
BLO four_bytes_match_emit_calcBlockSize
|
|
ADD $0x05, R1, R1
|
|
JMP memmove_long_match_emit_calcBlockSize
|
|
|
|
four_bytes_match_emit_calcBlockSize:
|
|
ADD $0x04, R1, R1
|
|
JMP memmove_long_match_emit_calcBlockSize
|
|
|
|
three_bytes_match_emit_calcBlockSize:
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_match_emit_calcBlockSize
|
|
|
|
two_bytes_match_emit_calcBlockSize:
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R6
|
|
BLO memmove_match_emit_calcBlockSize
|
|
JMP memmove_long_match_emit_calcBlockSize
|
|
|
|
one_byte_match_emit_calcBlockSize:
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_match_emit_calcBlockSize:
|
|
ADD R8, R1, R1
|
|
JMP emit_literal_done_match_emit_calcBlockSize
|
|
|
|
memmove_long_match_emit_calcBlockSize:
|
|
ADD R8, R1, R1
|
|
|
|
emit_literal_done_match_emit_calcBlockSize:
|
|
match_nolit_loop_calcBlockSize:
|
|
MOVWU R2, R6
|
|
SUBW R5, R6, R6
|
|
MOVW R6, 24(RSP)
|
|
ADDW $0x04, R2, R2
|
|
ADDW $0x04, R5, R5
|
|
MOVD src_len+8(FP), R6
|
|
SUBW R2, R6, R6
|
|
ADD R2, R3, R7
|
|
ADD R5, R3, R5
|
|
|
|
// matchLen
|
|
MOVD $0, R9
|
|
|
|
matchlen_loopback_16_match_nolit_calcBlockSize:
|
|
CMPW $0x10, R6
|
|
BLO matchlen_match8_match_nolit_calcBlockSize
|
|
MOVD (R7)(R9), R8
|
|
ADD R9, R7, R15
|
|
MOVD 8(R15), R10
|
|
MOVD (R5)(R9), R16
|
|
EOR R16, R8, R8
|
|
TST R8, R8
|
|
BNE matchlen_bsf_8_match_nolit_calcBlockSize
|
|
ADD R9, R5, R15
|
|
MOVD 8(R15), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_16match_nolit_calcBlockSize
|
|
SUB $16, R6, R6
|
|
MOVWU R6, R6
|
|
ADD $16, R9, R9
|
|
MOVWU R9, R9
|
|
JMP matchlen_loopback_16_match_nolit_calcBlockSize
|
|
|
|
matchlen_bsf_16match_nolit_calcBlockSize:
|
|
RBIT R10, R10
|
|
CLZ R10, R10
|
|
ASR $0x03, R10, R10
|
|
ADD R10, R9, R9
|
|
ADD $8, R9, R9
|
|
MOVWU R9, R9
|
|
JMP match_nolit_end_calcBlockSize
|
|
|
|
matchlen_match8_match_nolit_calcBlockSize:
|
|
CMPW $0x08, R6
|
|
BLO matchlen_match4_match_nolit_calcBlockSize
|
|
MOVD (R7)(R9), R8
|
|
MOVD (R5)(R9), R16
|
|
EOR R16, R8, R8
|
|
TST R8, R8
|
|
BNE matchlen_bsf_8_match_nolit_calcBlockSize
|
|
SUB $8, R6, R6
|
|
MOVWU R6, R6
|
|
ADD $8, R9, R9
|
|
MOVWU R9, R9
|
|
JMP matchlen_match4_match_nolit_calcBlockSize
|
|
|
|
matchlen_bsf_8_match_nolit_calcBlockSize:
|
|
RBIT R8, R8
|
|
CLZ R8, R8
|
|
ASR $0x03, R8, R8
|
|
ADD R8, R9, R9
|
|
MOVWU R9, R9
|
|
JMP match_nolit_end_calcBlockSize
|
|
|
|
matchlen_match4_match_nolit_calcBlockSize:
|
|
CMPW $0x04, R6
|
|
BLO matchlen_match2_match_nolit_calcBlockSize
|
|
MOVWU (R7)(R9), R8
|
|
MOVWU (R5)(R9), R16
|
|
CMPW R8, R16
|
|
BNE matchlen_match2_match_nolit_calcBlockSize
|
|
SUB $4, R6, R6
|
|
MOVWU R6, R6
|
|
ADD $4, R9, R9
|
|
MOVWU R9, R9
|
|
|
|
matchlen_match2_match_nolit_calcBlockSize:
|
|
CMPW $0x01, R6
|
|
BEQ matchlen_match1_match_nolit_calcBlockSize
|
|
BLO match_nolit_end_calcBlockSize
|
|
MOVHU (R7)(R9), R16
|
|
BFI $0, R16, $16, R8
|
|
ADD R9, R5, R15
|
|
MOVHU (R15), R15
|
|
AND $0xffff, R8, R16
|
|
CMP R16, R15
|
|
BNE matchlen_match1_match_nolit_calcBlockSize
|
|
ADD $2, R9, R9
|
|
MOVWU R9, R9
|
|
SUBSW $0x02, R6, R6
|
|
BEQ match_nolit_end_calcBlockSize
|
|
|
|
matchlen_match1_match_nolit_calcBlockSize:
|
|
MOVBU (R7)(R9), R16
|
|
BFI $0, R16, $8, R8
|
|
ADD R9, R5, R15
|
|
MOVBU (R15), R15
|
|
AND $0xff, R8, R16
|
|
CMP R16, R15
|
|
BNE match_nolit_end_calcBlockSize
|
|
ADD $1, R9, R9
|
|
MOVWU R9, R9
|
|
|
|
match_nolit_end_calcBlockSize:
|
|
ADDW R9, R2, R2
|
|
MOVWU 24(RSP), R5
|
|
ADDW $0x04, R9, R9
|
|
MOVW R2, 20(RSP)
|
|
|
|
// emitCopy
|
|
CMPW $0x00010000, R5
|
|
BLO two_byte_offset_match_nolit_calcBlockSize
|
|
|
|
four_bytes_loop_back_match_nolit_calcBlockSize:
|
|
CMPW $0x40, R9
|
|
BLS four_bytes_remain_match_nolit_calcBlockSize
|
|
SUB $64, R9, R9
|
|
MOVWU R9, R9
|
|
ADD $0x05, R1, R1
|
|
CMPW $0x04, R9
|
|
BLO four_bytes_remain_match_nolit_calcBlockSize
|
|
JMP four_bytes_loop_back_match_nolit_calcBlockSize
|
|
|
|
four_bytes_remain_match_nolit_calcBlockSize:
|
|
TSTW R9, R9
|
|
BEQ match_nolit_emitcopy_end_calcBlockSize
|
|
MOVD $0, R5
|
|
ADD $0x05, R1, R1
|
|
JMP match_nolit_emitcopy_end_calcBlockSize
|
|
|
|
two_byte_offset_match_nolit_calcBlockSize:
|
|
CMPW $0x40, R9
|
|
BLS two_byte_offset_short_match_nolit_calcBlockSize
|
|
SUB $60, R9, R9
|
|
MOVWU R9, R9
|
|
ADD $0x03, R1, R1
|
|
JMP two_byte_offset_match_nolit_calcBlockSize
|
|
|
|
two_byte_offset_short_match_nolit_calcBlockSize:
|
|
MOVWU R9, R6
|
|
LSLW $0x02, R6, R6
|
|
CMPW $0x0c, R9
|
|
BHS emit_copy_three_match_nolit_calcBlockSize
|
|
CMPW $0x00000800, R5
|
|
BHS emit_copy_three_match_nolit_calcBlockSize
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_calcBlockSize
|
|
|
|
emit_copy_three_match_nolit_calcBlockSize:
|
|
ADD $0x03, R1, R1
|
|
|
|
match_nolit_emitcopy_end_calcBlockSize:
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R2
|
|
BHS emit_remainder_calcBlockSize
|
|
ADD R2, R3, R15
|
|
MOVD -2(R15), R6
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R1
|
|
BLO match_nolit_dst_ok_calcBlockSize
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+32(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_calcBlockSize:
|
|
MOVD $0x0000cf1bbcdcbf9b, R8
|
|
MOVD R6, R7
|
|
LSR $0x10, R6, R6
|
|
MOVD R6, R5
|
|
LSL $0x10, R7, R7
|
|
MUL R8, R7, R7
|
|
LSR $0x33, R7, R7
|
|
LSL $0x10, R5, R5
|
|
MUL R8, R5, R5
|
|
LSR $0x33, R5, R5
|
|
SUB $2, R2, R8
|
|
MOVWU R8, R8
|
|
ADD R5<<2, R0, R9
|
|
MOVWU (R9), R5
|
|
MOVW R8, (R0)(R7<<2)
|
|
MOVW R2, (R9)
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ match_nolit_loop_calcBlockSize
|
|
ADDW $1, R2, R2
|
|
JMP search_loop_calcBlockSize
|
|
|
|
emit_remainder_calcBlockSize:
|
|
MOVD src_len+8(FP), R0
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R0, R0
|
|
ADD R0, R1, R0
|
|
ADD $5, R0, R0
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R0
|
|
BLO emit_remainder_ok_calcBlockSize
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+32(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_calcBlockSize:
|
|
MOVD src_len+8(FP), R0
|
|
MOVWU 20(RSP), R2
|
|
CMPW R0, R2
|
|
BEQ emit_literal_done_emit_remainder_calcBlockSize
|
|
MOVWU R0, R5
|
|
MOVW R0, 20(RSP)
|
|
ADD R2, R3, R0
|
|
SUBW R2, R5, R5
|
|
SUB $1, R5, R0
|
|
MOVWU R0, R0
|
|
CMPW $0x3c, R0
|
|
BLO one_byte_emit_remainder_calcBlockSize
|
|
CMPW $0x00000100, R0
|
|
BLO two_bytes_emit_remainder_calcBlockSize
|
|
CMPW $0x00010000, R0
|
|
BLO three_bytes_emit_remainder_calcBlockSize
|
|
CMPW $0x01000000, R0
|
|
BLO four_bytes_emit_remainder_calcBlockSize
|
|
ADD $0x05, R1, R1
|
|
JMP memmove_long_emit_remainder_calcBlockSize
|
|
|
|
four_bytes_emit_remainder_calcBlockSize:
|
|
ADD $0x04, R1, R1
|
|
JMP memmove_long_emit_remainder_calcBlockSize
|
|
|
|
three_bytes_emit_remainder_calcBlockSize:
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_emit_remainder_calcBlockSize
|
|
|
|
two_bytes_emit_remainder_calcBlockSize:
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R0
|
|
BLO memmove_emit_remainder_calcBlockSize
|
|
JMP memmove_long_emit_remainder_calcBlockSize
|
|
|
|
one_byte_emit_remainder_calcBlockSize:
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_emit_remainder_calcBlockSize:
|
|
ADD R5, R1, R0
|
|
MOVD R0, R1
|
|
JMP emit_literal_done_emit_remainder_calcBlockSize
|
|
|
|
memmove_long_emit_remainder_calcBlockSize:
|
|
ADD R5, R1, R0
|
|
MOVD R0, R1
|
|
|
|
emit_literal_done_emit_remainder_calcBlockSize:
|
|
MOVD R1, ret+32(FP)
|
|
RET
|
|
|
|
// func calcBlockSizeSmall(src []byte, tmp *[2048]byte) int
|
|
// Requires: BMI, SSE2
|
|
TEXT ·calcBlockSizeSmall(SB), $24-40
|
|
MOVD tmp+24(FP), R0
|
|
MOVD $0, R1
|
|
MOVD $0x00000010, R2
|
|
MOVD R0, R3
|
|
VEOR V0.B16, V0.B16, V0.B16
|
|
|
|
zero_loop_calcBlockSizeSmall:
|
|
FMOVQ F0, (R3)
|
|
FMOVQ F0, 16(R3)
|
|
FMOVQ F0, 32(R3)
|
|
FMOVQ F0, 48(R3)
|
|
FMOVQ F0, 64(R3)
|
|
FMOVQ F0, 80(R3)
|
|
FMOVQ F0, 96(R3)
|
|
FMOVQ F0, 112(R3)
|
|
ADD $0x80, R3, R3
|
|
SUBS $1, R2, R2
|
|
BNE zero_loop_calcBlockSizeSmall
|
|
MOVD $0x00000000, R16
|
|
MOVW R16, 20(RSP)
|
|
MOVD src_len+8(FP), R2
|
|
SUB $9, R2, R3
|
|
SUB $8, R2, R5
|
|
MOVW R5, 16(RSP)
|
|
LSR $0x05, R2, R2
|
|
SUBW R2, R3, R3
|
|
ADD R3, R1, R3
|
|
MOVD R3, 8(RSP)
|
|
MOVD $0x00000001, R2
|
|
MOVW R2, 24(RSP)
|
|
MOVD src_base+0(FP), R3
|
|
|
|
search_loop_calcBlockSizeSmall:
|
|
MOVWU R2, R5
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R5, R5
|
|
LSRW $0x04, R5, R5
|
|
ADD R5, R2, R5
|
|
ADD $4, R5, R5
|
|
MOVWU R5, R5
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R5
|
|
BHS emit_remainder_calcBlockSizeSmall
|
|
MOVD (R3)(R2), R6
|
|
MOVW R5, 28(RSP)
|
|
MOVD $0x9e3779b1, R8
|
|
MOVD R6, R9
|
|
MOVD R6, R10
|
|
LSR $0x08, R10, R10
|
|
LSL $0x20, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x37, R9, R9
|
|
LSL $0x20, R10, R10
|
|
MUL R8, R10, R10
|
|
LSR $0x37, R10, R10
|
|
MOVWU (R0)(R9<<2), R5
|
|
MOVWU (R0)(R10<<2), R7
|
|
MOVW R2, (R0)(R9<<2)
|
|
ADD $1, R2, R9
|
|
MOVWU R9, R9
|
|
MOVW R9, (R0)(R10<<2)
|
|
MOVD R6, R9
|
|
LSR $0x10, R9, R9
|
|
LSL $0x20, R9, R9
|
|
MUL R8, R9, R9
|
|
LSR $0x37, R9, R9
|
|
MOVWU R2, R8
|
|
MOVWU 24(RSP), R16
|
|
SUBW R16, R8, R8
|
|
ADD R8, R3, R15
|
|
MOVWU 1(R15), R10
|
|
MOVD R6, R8
|
|
LSR $0x08, R8, R8
|
|
CMPW R10, R8
|
|
BNE no_repeat_found_calcBlockSizeSmall
|
|
ADD $1, R2, R6
|
|
MOVWU R6, R6
|
|
MOVWU 20(RSP), R5
|
|
MOVWU R6, R7
|
|
MOVWU 24(RSP), R16
|
|
SUBSW R16, R7, R7
|
|
BEQ repeat_extend_back_end_calcBlockSizeSmall
|
|
|
|
repeat_extend_back_loop_calcBlockSizeSmall:
|
|
CMPW R5, R6
|
|
BLS repeat_extend_back_end_calcBlockSizeSmall
|
|
ADD R7, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R8
|
|
ADD R6, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R9
|
|
AND $0xff, R9, R16
|
|
AND $0xff, R8, R15
|
|
CMP R16, R15
|
|
BNE repeat_extend_back_end_calcBlockSizeSmall
|
|
SUB $1, R6, R6
|
|
MOVWU R6, R6
|
|
SUBSW $1, R7, R7
|
|
BNE repeat_extend_back_loop_calcBlockSizeSmall
|
|
|
|
repeat_extend_back_end_calcBlockSizeSmall:
|
|
MOVWU R6, R5
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R5, R5
|
|
ADD R5, R1, R5
|
|
ADD $3, R5, R5
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R5
|
|
BLO repeat_dst_size_check_calcBlockSizeSmall
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+32(FP)
|
|
RET
|
|
|
|
repeat_dst_size_check_calcBlockSizeSmall:
|
|
MOVWU 20(RSP), R5
|
|
CMPW R6, R5
|
|
BEQ emit_literal_done_repeat_emit_calcBlockSizeSmall
|
|
MOVWU R6, R7
|
|
MOVW R6, 20(RSP)
|
|
ADD R5, R3, R8
|
|
SUBW R5, R7, R7
|
|
SUB $1, R7, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x3c, R5
|
|
BLO one_byte_repeat_emit_calcBlockSizeSmall
|
|
CMPW $0x00000100, R5
|
|
BLO two_bytes_repeat_emit_calcBlockSizeSmall
|
|
BLO three_bytes_repeat_emit_calcBlockSizeSmall
|
|
|
|
three_bytes_repeat_emit_calcBlockSizeSmall:
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_repeat_emit_calcBlockSizeSmall
|
|
|
|
two_bytes_repeat_emit_calcBlockSizeSmall:
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R5
|
|
BLO memmove_repeat_emit_calcBlockSizeSmall
|
|
JMP memmove_long_repeat_emit_calcBlockSizeSmall
|
|
|
|
one_byte_repeat_emit_calcBlockSizeSmall:
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_repeat_emit_calcBlockSizeSmall:
|
|
ADD R7, R1, R1
|
|
JMP emit_literal_done_repeat_emit_calcBlockSizeSmall
|
|
|
|
memmove_long_repeat_emit_calcBlockSizeSmall:
|
|
ADD R7, R1, R1
|
|
|
|
emit_literal_done_repeat_emit_calcBlockSizeSmall:
|
|
ADDW $0x05, R2, R2
|
|
MOVWU R2, R5
|
|
MOVWU 24(RSP), R16
|
|
SUBW R16, R5, R5
|
|
MOVD src_len+8(FP), R7
|
|
SUBW R2, R7, R7
|
|
ADD R2, R3, R8
|
|
ADD R5, R3, R5
|
|
|
|
// matchLen
|
|
MOVD $0, R10
|
|
|
|
matchlen_loopback_16_repeat_extend_calcBlockSizeSmall:
|
|
CMPW $0x10, R7
|
|
BLO matchlen_match8_repeat_extend_calcBlockSizeSmall
|
|
MOVD (R8)(R10), R9
|
|
ADD R10, R8, R15
|
|
MOVD 8(R15), R11
|
|
MOVD (R5)(R10), R16
|
|
EOR R16, R9, R9
|
|
TST R9, R9
|
|
BNE matchlen_bsf_8_repeat_extend_calcBlockSizeSmall
|
|
ADD R10, R5, R15
|
|
MOVD 8(R15), R16
|
|
EOR R16, R11, R11
|
|
TST R11, R11
|
|
BNE matchlen_bsf_16repeat_extend_calcBlockSizeSmall
|
|
SUB $16, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $16, R10, R10
|
|
MOVWU R10, R10
|
|
JMP matchlen_loopback_16_repeat_extend_calcBlockSizeSmall
|
|
|
|
matchlen_bsf_16repeat_extend_calcBlockSizeSmall:
|
|
RBIT R11, R11
|
|
CLZ R11, R11
|
|
ASR $0x03, R11, R11
|
|
ADD R11, R10, R10
|
|
ADD $8, R10, R10
|
|
MOVWU R10, R10
|
|
JMP repeat_extend_forward_end_calcBlockSizeSmall
|
|
|
|
matchlen_match8_repeat_extend_calcBlockSizeSmall:
|
|
CMPW $0x08, R7
|
|
BLO matchlen_match4_repeat_extend_calcBlockSizeSmall
|
|
MOVD (R8)(R10), R9
|
|
MOVD (R5)(R10), R16
|
|
EOR R16, R9, R9
|
|
TST R9, R9
|
|
BNE matchlen_bsf_8_repeat_extend_calcBlockSizeSmall
|
|
SUB $8, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $8, R10, R10
|
|
MOVWU R10, R10
|
|
JMP matchlen_match4_repeat_extend_calcBlockSizeSmall
|
|
|
|
matchlen_bsf_8_repeat_extend_calcBlockSizeSmall:
|
|
RBIT R9, R9
|
|
CLZ R9, R9
|
|
ASR $0x03, R9, R9
|
|
ADD R9, R10, R10
|
|
MOVWU R10, R10
|
|
JMP repeat_extend_forward_end_calcBlockSizeSmall
|
|
|
|
matchlen_match4_repeat_extend_calcBlockSizeSmall:
|
|
CMPW $0x04, R7
|
|
BLO matchlen_match2_repeat_extend_calcBlockSizeSmall
|
|
MOVWU (R8)(R10), R9
|
|
MOVWU (R5)(R10), R16
|
|
CMPW R9, R16
|
|
BNE matchlen_match2_repeat_extend_calcBlockSizeSmall
|
|
SUB $4, R7, R7
|
|
MOVWU R7, R7
|
|
ADD $4, R10, R10
|
|
MOVWU R10, R10
|
|
|
|
matchlen_match2_repeat_extend_calcBlockSizeSmall:
|
|
CMPW $0x01, R7
|
|
BEQ matchlen_match1_repeat_extend_calcBlockSizeSmall
|
|
BLO repeat_extend_forward_end_calcBlockSizeSmall
|
|
MOVHU (R8)(R10), R16
|
|
BFI $0, R16, $16, R9
|
|
ADD R10, R5, R15
|
|
MOVHU (R15), R15
|
|
AND $0xffff, R9, R16
|
|
CMP R16, R15
|
|
BNE matchlen_match1_repeat_extend_calcBlockSizeSmall
|
|
ADD $2, R10, R10
|
|
MOVWU R10, R10
|
|
SUBSW $0x02, R7, R7
|
|
BEQ repeat_extend_forward_end_calcBlockSizeSmall
|
|
|
|
matchlen_match1_repeat_extend_calcBlockSizeSmall:
|
|
MOVBU (R8)(R10), R16
|
|
BFI $0, R16, $8, R9
|
|
ADD R10, R5, R15
|
|
MOVBU (R15), R15
|
|
AND $0xff, R9, R16
|
|
CMP R16, R15
|
|
BNE repeat_extend_forward_end_calcBlockSizeSmall
|
|
ADD $1, R10, R10
|
|
MOVWU R10, R10
|
|
|
|
repeat_extend_forward_end_calcBlockSizeSmall:
|
|
ADDW R10, R2, R2
|
|
MOVWU R2, R5
|
|
SUBW R6, R5, R5
|
|
MOVWU 24(RSP), R6
|
|
|
|
// emitCopy
|
|
two_byte_offset_repeat_as_copy_calcBlockSizeSmall:
|
|
CMPW $0x40, R5
|
|
BLS two_byte_offset_short_repeat_as_copy_calcBlockSizeSmall
|
|
SUB $60, R5, R5
|
|
MOVWU R5, R5
|
|
ADD $0x03, R1, R1
|
|
JMP two_byte_offset_repeat_as_copy_calcBlockSizeSmall
|
|
|
|
two_byte_offset_short_repeat_as_copy_calcBlockSizeSmall:
|
|
MOVWU R5, R6
|
|
LSLW $0x02, R6, R6
|
|
CMPW $0x0c, R5
|
|
BHS emit_copy_three_repeat_as_copy_calcBlockSizeSmall
|
|
ADD $0x02, R1, R1
|
|
JMP repeat_end_emit_calcBlockSizeSmall
|
|
|
|
emit_copy_three_repeat_as_copy_calcBlockSizeSmall:
|
|
ADD $0x03, R1, R1
|
|
|
|
repeat_end_emit_calcBlockSizeSmall:
|
|
MOVW R2, 20(RSP)
|
|
JMP search_loop_calcBlockSizeSmall
|
|
|
|
no_repeat_found_calcBlockSizeSmall:
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ candidate_match_calcBlockSizeSmall
|
|
LSR $0x08, R6, R6
|
|
MOVWU (R0)(R9<<2), R5
|
|
ADD $2, R2, R8
|
|
MOVWU R8, R8
|
|
MOVWU (R3)(R7), R16
|
|
CMPW R6, R16
|
|
BEQ candidate2_match_calcBlockSizeSmall
|
|
MOVW R8, (R0)(R9<<2)
|
|
LSR $0x08, R6, R6
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ candidate3_match_calcBlockSizeSmall
|
|
MOVWU 28(RSP), R2
|
|
JMP search_loop_calcBlockSizeSmall
|
|
|
|
candidate3_match_calcBlockSizeSmall:
|
|
ADDW $0x02, R2, R2
|
|
JMP candidate_match_calcBlockSizeSmall
|
|
|
|
candidate2_match_calcBlockSizeSmall:
|
|
MOVW R8, (R0)(R9<<2)
|
|
ADDW $1, R2, R2
|
|
MOVWU R7, R5
|
|
|
|
candidate_match_calcBlockSizeSmall:
|
|
MOVWU 20(RSP), R6
|
|
TSTW R5, R5
|
|
BEQ match_extend_back_end_calcBlockSizeSmall
|
|
|
|
match_extend_back_loop_calcBlockSizeSmall:
|
|
CMPW R6, R2
|
|
BLS match_extend_back_end_calcBlockSizeSmall
|
|
ADD R5, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R7
|
|
ADD R2, R3, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R8
|
|
AND $0xff, R8, R16
|
|
AND $0xff, R7, R15
|
|
CMP R16, R15
|
|
BNE match_extend_back_end_calcBlockSizeSmall
|
|
SUB $1, R2, R2
|
|
MOVWU R2, R2
|
|
SUBSW $1, R5, R5
|
|
BEQ match_extend_back_end_calcBlockSizeSmall
|
|
JMP match_extend_back_loop_calcBlockSizeSmall
|
|
|
|
match_extend_back_end_calcBlockSizeSmall:
|
|
MOVWU R2, R6
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R6, R6
|
|
ADD R6, R1, R6
|
|
ADD $3, R6, R6
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R6
|
|
BLO match_dst_size_check_calcBlockSizeSmall
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+32(FP)
|
|
RET
|
|
|
|
match_dst_size_check_calcBlockSizeSmall:
|
|
MOVWU R2, R6
|
|
MOVWU 20(RSP), R7
|
|
CMPW R6, R7
|
|
BEQ emit_literal_done_match_emit_calcBlockSizeSmall
|
|
MOVWU R6, R8
|
|
MOVW R6, 20(RSP)
|
|
ADD R7, R3, R6
|
|
SUBW R7, R8, R8
|
|
SUB $1, R8, R6
|
|
MOVWU R6, R6
|
|
CMPW $0x3c, R6
|
|
BLO one_byte_match_emit_calcBlockSizeSmall
|
|
CMPW $0x00000100, R6
|
|
BLO two_bytes_match_emit_calcBlockSizeSmall
|
|
BLO three_bytes_match_emit_calcBlockSizeSmall
|
|
|
|
three_bytes_match_emit_calcBlockSizeSmall:
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_match_emit_calcBlockSizeSmall
|
|
|
|
two_bytes_match_emit_calcBlockSizeSmall:
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R6
|
|
BLO memmove_match_emit_calcBlockSizeSmall
|
|
JMP memmove_long_match_emit_calcBlockSizeSmall
|
|
|
|
one_byte_match_emit_calcBlockSizeSmall:
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_match_emit_calcBlockSizeSmall:
|
|
ADD R8, R1, R1
|
|
JMP emit_literal_done_match_emit_calcBlockSizeSmall
|
|
|
|
memmove_long_match_emit_calcBlockSizeSmall:
|
|
ADD R8, R1, R1
|
|
|
|
emit_literal_done_match_emit_calcBlockSizeSmall:
|
|
match_nolit_loop_calcBlockSizeSmall:
|
|
MOVWU R2, R6
|
|
SUBW R5, R6, R6
|
|
MOVW R6, 24(RSP)
|
|
ADDW $0x04, R2, R2
|
|
ADDW $0x04, R5, R5
|
|
MOVD src_len+8(FP), R6
|
|
SUBW R2, R6, R6
|
|
ADD R2, R3, R7
|
|
ADD R5, R3, R5
|
|
|
|
// matchLen
|
|
MOVD $0, R9
|
|
|
|
matchlen_loopback_16_match_nolit_calcBlockSizeSmall:
|
|
CMPW $0x10, R6
|
|
BLO matchlen_match8_match_nolit_calcBlockSizeSmall
|
|
MOVD (R7)(R9), R8
|
|
ADD R9, R7, R15
|
|
MOVD 8(R15), R10
|
|
MOVD (R5)(R9), R16
|
|
EOR R16, R8, R8
|
|
TST R8, R8
|
|
BNE matchlen_bsf_8_match_nolit_calcBlockSizeSmall
|
|
ADD R9, R5, R15
|
|
MOVD 8(R15), R16
|
|
EOR R16, R10, R10
|
|
TST R10, R10
|
|
BNE matchlen_bsf_16match_nolit_calcBlockSizeSmall
|
|
SUB $16, R6, R6
|
|
MOVWU R6, R6
|
|
ADD $16, R9, R9
|
|
MOVWU R9, R9
|
|
JMP matchlen_loopback_16_match_nolit_calcBlockSizeSmall
|
|
|
|
matchlen_bsf_16match_nolit_calcBlockSizeSmall:
|
|
RBIT R10, R10
|
|
CLZ R10, R10
|
|
ASR $0x03, R10, R10
|
|
ADD R10, R9, R9
|
|
ADD $8, R9, R9
|
|
MOVWU R9, R9
|
|
JMP match_nolit_end_calcBlockSizeSmall
|
|
|
|
matchlen_match8_match_nolit_calcBlockSizeSmall:
|
|
CMPW $0x08, R6
|
|
BLO matchlen_match4_match_nolit_calcBlockSizeSmall
|
|
MOVD (R7)(R9), R8
|
|
MOVD (R5)(R9), R16
|
|
EOR R16, R8, R8
|
|
TST R8, R8
|
|
BNE matchlen_bsf_8_match_nolit_calcBlockSizeSmall
|
|
SUB $8, R6, R6
|
|
MOVWU R6, R6
|
|
ADD $8, R9, R9
|
|
MOVWU R9, R9
|
|
JMP matchlen_match4_match_nolit_calcBlockSizeSmall
|
|
|
|
matchlen_bsf_8_match_nolit_calcBlockSizeSmall:
|
|
RBIT R8, R8
|
|
CLZ R8, R8
|
|
ASR $0x03, R8, R8
|
|
ADD R8, R9, R9
|
|
MOVWU R9, R9
|
|
JMP match_nolit_end_calcBlockSizeSmall
|
|
|
|
matchlen_match4_match_nolit_calcBlockSizeSmall:
|
|
CMPW $0x04, R6
|
|
BLO matchlen_match2_match_nolit_calcBlockSizeSmall
|
|
MOVWU (R7)(R9), R8
|
|
MOVWU (R5)(R9), R16
|
|
CMPW R8, R16
|
|
BNE matchlen_match2_match_nolit_calcBlockSizeSmall
|
|
SUB $4, R6, R6
|
|
MOVWU R6, R6
|
|
ADD $4, R9, R9
|
|
MOVWU R9, R9
|
|
|
|
matchlen_match2_match_nolit_calcBlockSizeSmall:
|
|
CMPW $0x01, R6
|
|
BEQ matchlen_match1_match_nolit_calcBlockSizeSmall
|
|
BLO match_nolit_end_calcBlockSizeSmall
|
|
MOVHU (R7)(R9), R16
|
|
BFI $0, R16, $16, R8
|
|
ADD R9, R5, R15
|
|
MOVHU (R15), R15
|
|
AND $0xffff, R8, R16
|
|
CMP R16, R15
|
|
BNE matchlen_match1_match_nolit_calcBlockSizeSmall
|
|
ADD $2, R9, R9
|
|
MOVWU R9, R9
|
|
SUBSW $0x02, R6, R6
|
|
BEQ match_nolit_end_calcBlockSizeSmall
|
|
|
|
matchlen_match1_match_nolit_calcBlockSizeSmall:
|
|
MOVBU (R7)(R9), R16
|
|
BFI $0, R16, $8, R8
|
|
ADD R9, R5, R15
|
|
MOVBU (R15), R15
|
|
AND $0xff, R8, R16
|
|
CMP R16, R15
|
|
BNE match_nolit_end_calcBlockSizeSmall
|
|
ADD $1, R9, R9
|
|
MOVWU R9, R9
|
|
|
|
match_nolit_end_calcBlockSizeSmall:
|
|
ADDW R9, R2, R2
|
|
MOVWU 24(RSP), R5
|
|
ADDW $0x04, R9, R9
|
|
MOVW R2, 20(RSP)
|
|
|
|
// emitCopy
|
|
two_byte_offset_match_nolit_calcBlockSizeSmall:
|
|
CMPW $0x40, R9
|
|
BLS two_byte_offset_short_match_nolit_calcBlockSizeSmall
|
|
SUB $60, R9, R9
|
|
MOVWU R9, R9
|
|
ADD $0x03, R1, R1
|
|
JMP two_byte_offset_match_nolit_calcBlockSizeSmall
|
|
|
|
two_byte_offset_short_match_nolit_calcBlockSizeSmall:
|
|
MOVWU R9, R5
|
|
LSLW $0x02, R5, R5
|
|
CMPW $0x0c, R9
|
|
BHS emit_copy_three_match_nolit_calcBlockSizeSmall
|
|
ADD $0x02, R1, R1
|
|
JMP match_nolit_emitcopy_end_calcBlockSizeSmall
|
|
|
|
emit_copy_three_match_nolit_calcBlockSizeSmall:
|
|
ADD $0x03, R1, R1
|
|
|
|
match_nolit_emitcopy_end_calcBlockSizeSmall:
|
|
MOVWU 16(RSP), R16
|
|
CMPW R16, R2
|
|
BHS emit_remainder_calcBlockSizeSmall
|
|
ADD R2, R3, R15
|
|
MOVD -2(R15), R6
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R1
|
|
BLO match_nolit_dst_ok_calcBlockSizeSmall
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+32(FP)
|
|
RET
|
|
|
|
match_nolit_dst_ok_calcBlockSizeSmall:
|
|
MOVD $0x9e3779b1, R8
|
|
MOVD R6, R7
|
|
LSR $0x10, R6, R6
|
|
MOVD R6, R5
|
|
LSL $0x20, R7, R7
|
|
MUL R8, R7, R7
|
|
LSR $0x37, R7, R7
|
|
LSL $0x20, R5, R5
|
|
MUL R8, R5, R5
|
|
LSR $0x37, R5, R5
|
|
SUB $2, R2, R8
|
|
MOVWU R8, R8
|
|
ADD R5<<2, R0, R9
|
|
MOVWU (R9), R5
|
|
MOVW R8, (R0)(R7<<2)
|
|
MOVW R2, (R9)
|
|
MOVWU (R3)(R5), R16
|
|
CMPW R6, R16
|
|
BEQ match_nolit_loop_calcBlockSizeSmall
|
|
ADDW $1, R2, R2
|
|
JMP search_loop_calcBlockSizeSmall
|
|
|
|
emit_remainder_calcBlockSizeSmall:
|
|
MOVD src_len+8(FP), R0
|
|
MOVWU 20(RSP), R16
|
|
SUBW R16, R0, R0
|
|
ADD R0, R1, R0
|
|
ADD $3, R0, R0
|
|
MOVD 8(RSP), R16
|
|
CMP R16, R0
|
|
BLO emit_remainder_ok_calcBlockSizeSmall
|
|
MOVD $0x00000000, R16
|
|
MOVD R16, ret+32(FP)
|
|
RET
|
|
|
|
emit_remainder_ok_calcBlockSizeSmall:
|
|
MOVD src_len+8(FP), R0
|
|
MOVWU 20(RSP), R2
|
|
CMPW R0, R2
|
|
BEQ emit_literal_done_emit_remainder_calcBlockSizeSmall
|
|
MOVWU R0, R5
|
|
MOVW R0, 20(RSP)
|
|
ADD R2, R3, R0
|
|
SUBW R2, R5, R5
|
|
SUB $1, R5, R0
|
|
MOVWU R0, R0
|
|
CMPW $0x3c, R0
|
|
BLO one_byte_emit_remainder_calcBlockSizeSmall
|
|
CMPW $0x00000100, R0
|
|
BLO two_bytes_emit_remainder_calcBlockSizeSmall
|
|
BLO three_bytes_emit_remainder_calcBlockSizeSmall
|
|
|
|
three_bytes_emit_remainder_calcBlockSizeSmall:
|
|
ADD $0x03, R1, R1
|
|
JMP memmove_long_emit_remainder_calcBlockSizeSmall
|
|
|
|
two_bytes_emit_remainder_calcBlockSizeSmall:
|
|
ADD $0x02, R1, R1
|
|
CMPW $0x40, R0
|
|
BLO memmove_emit_remainder_calcBlockSizeSmall
|
|
JMP memmove_long_emit_remainder_calcBlockSizeSmall
|
|
|
|
one_byte_emit_remainder_calcBlockSizeSmall:
|
|
ADD $0x01, R1, R1
|
|
|
|
memmove_emit_remainder_calcBlockSizeSmall:
|
|
ADD R5, R1, R0
|
|
MOVD R0, R1
|
|
JMP emit_literal_done_emit_remainder_calcBlockSizeSmall
|
|
|
|
memmove_long_emit_remainder_calcBlockSizeSmall:
|
|
ADD R5, R1, R0
|
|
MOVD R0, R1
|
|
|
|
emit_literal_done_emit_remainder_calcBlockSizeSmall:
|
|
MOVD R1, ret+32(FP)
|
|
RET
|
|
|
|
// func emitLiteral(dst []byte, lit []byte) int
|
|
// Requires: SSE2
|
|
TEXT ·emitLiteral(SB), NOSPLIT, $0-56
|
|
MOVD lit_len+32(FP), R2
|
|
MOVD dst_base+0(FP), R0
|
|
MOVD lit_base+24(FP), R1
|
|
TST R2, R2
|
|
BEQ emit_literal_end_standalone_skip
|
|
MOVWU R2, R3
|
|
SUB $1, R2, R5
|
|
MOVWU R5, R5
|
|
CMPW $0x3c, R5
|
|
BLO one_byte_standalone
|
|
CMPW $0x00000100, R5
|
|
BLO two_bytes_standalone
|
|
CMPW $0x00010000, R5
|
|
BLO three_bytes_standalone
|
|
CMPW $0x01000000, R5
|
|
BLO four_bytes_standalone
|
|
MOVD $0xfc, R16
|
|
MOVB R16, (R0)
|
|
MOVW R5, 1(R0)
|
|
ADD $0x05, R3, R3
|
|
ADD $0x05, R0, R0
|
|
JMP memmove_long_standalone
|
|
|
|
four_bytes_standalone:
|
|
MOVWU R5, R6
|
|
LSRW $0x10, R6, R6
|
|
MOVD $0xf8, R16
|
|
MOVB R16, (R0)
|
|
MOVH R5, 1(R0)
|
|
MOVB R6, 3(R0)
|
|
ADD $0x04, R3, R3
|
|
ADD $0x04, R0, R0
|
|
JMP memmove_long_standalone
|
|
|
|
three_bytes_standalone:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R0)
|
|
MOVH R5, 1(R0)
|
|
ADD $0x03, R3, R3
|
|
ADD $0x03, R0, R0
|
|
JMP memmove_long_standalone
|
|
|
|
two_bytes_standalone:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R0)
|
|
MOVB R5, 1(R0)
|
|
ADD $0x02, R3, R3
|
|
ADD $0x02, R0, R0
|
|
CMPW $0x40, R5
|
|
BLO memmove_standalone
|
|
JMP memmove_long_standalone
|
|
|
|
one_byte_standalone:
|
|
LSLW $0x02, R5, R16
|
|
BFI $0, R16, $8, R5
|
|
MOVB R5, (R0)
|
|
ADD $0x01, R3, R3
|
|
ADD $0x01, R0, R0
|
|
|
|
memmove_standalone:
|
|
// genMemMoveShort
|
|
CMP $0x03, R2
|
|
BLO emit_lit_memmove_standalone_memmove_move_1or2
|
|
BEQ emit_lit_memmove_standalone_memmove_move_3
|
|
CMP $0x08, R2
|
|
BLO emit_lit_memmove_standalone_memmove_move_4through7
|
|
CMP $0x10, R2
|
|
BLS emit_lit_memmove_standalone_memmove_move_8through16
|
|
CMP $0x20, R2
|
|
BLS emit_lit_memmove_standalone_memmove_move_17through32
|
|
JMP emit_lit_memmove_standalone_memmove_move_33through64
|
|
|
|
emit_lit_memmove_standalone_memmove_move_1or2:
|
|
MOVBU (R1), R16
|
|
BFI $0, R16, $8, R5
|
|
ADD R2, R1, R15
|
|
MOVBU -1(R15), R16
|
|
BFI $0, R16, $8, R1
|
|
MOVB R5, (R0)
|
|
ADD R2, R0, R15
|
|
MOVB R1, -1(R15)
|
|
JMP emit_literal_end_standalone
|
|
|
|
emit_lit_memmove_standalone_memmove_move_3:
|
|
MOVHU (R1), R16
|
|
BFI $0, R16, $16, R5
|
|
MOVBU 2(R1), R16
|
|
BFI $0, R16, $8, R1
|
|
MOVH R5, (R0)
|
|
MOVB R1, 2(R0)
|
|
JMP emit_literal_end_standalone
|
|
|
|
emit_lit_memmove_standalone_memmove_move_4through7:
|
|
MOVWU (R1), R5
|
|
ADD R2, R1, R15
|
|
MOVWU -4(R15), R1
|
|
MOVW R5, (R0)
|
|
ADD R2, R0, R15
|
|
MOVW R1, -4(R15)
|
|
JMP emit_literal_end_standalone
|
|
|
|
emit_lit_memmove_standalone_memmove_move_8through16:
|
|
MOVD (R1), R5
|
|
ADD R2, R1, R15
|
|
MOVD -8(R15), R1
|
|
MOVD R5, (R0)
|
|
ADD R2, R0, R15
|
|
MOVD R1, -8(R15)
|
|
JMP emit_literal_end_standalone
|
|
|
|
emit_lit_memmove_standalone_memmove_move_17through32:
|
|
FMOVQ (R1), F0
|
|
ADD R2, R1, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R0)
|
|
ADD R2, R0, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP emit_literal_end_standalone
|
|
|
|
emit_lit_memmove_standalone_memmove_move_33through64:
|
|
FMOVQ (R1), F0
|
|
FMOVQ 16(R1), F1
|
|
ADD R2, R1, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R2, R1, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R0)
|
|
FMOVQ F1, 16(R0)
|
|
ADD R2, R0, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R2, R0, R15
|
|
FMOVQ F3, -16(R15)
|
|
JMP emit_literal_end_standalone
|
|
JMP emit_literal_end_standalone
|
|
|
|
memmove_long_standalone:
|
|
// genMemMoveLong
|
|
MOVD R1, R5
|
|
MOVD R0, R6
|
|
MOVD R2, R7
|
|
|
|
emit_lit_memmove_long_standalonelarge_big_loop_back:
|
|
FMOVQ (R5), F0
|
|
FMOVQ 16(R5), F1
|
|
FMOVQ F0, (R6)
|
|
FMOVQ F1, 16(R6)
|
|
ADD $0x20, R5, R5
|
|
ADD $0x20, R6, R6
|
|
SUB $0x20, R7, R7
|
|
CMP $0x20, R7
|
|
BHS emit_lit_memmove_long_standalonelarge_big_loop_back
|
|
ADD R2, R1, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R2, R1, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R2, R0, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R2, R0, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP emit_literal_end_standalone
|
|
JMP emit_literal_end_standalone
|
|
|
|
emit_literal_end_standalone_skip:
|
|
MOVD $0, R3
|
|
|
|
emit_literal_end_standalone:
|
|
MOVD R3, ret+48(FP)
|
|
RET
|
|
|
|
// func emitRepeat(dst []byte, offset int, length int) int
|
|
TEXT ·emitRepeat(SB), NOSPLIT, $0-48
|
|
MOVD $0, R3
|
|
MOVD dst_base+0(FP), R0
|
|
MOVD offset+24(FP), R1
|
|
MOVD length+32(FP), R2
|
|
|
|
// emitRepeat
|
|
emit_repeat_again_standalone:
|
|
MOVWU R2, R5
|
|
SUB $4, R2, R2
|
|
MOVWU R2, R2
|
|
CMPW $0x08, R5
|
|
BLS repeat_two_standalone
|
|
CMPW $0x0c, R5
|
|
BHS cant_repeat_two_offset_standalone
|
|
CMPW $0x00000800, R1
|
|
BLO repeat_two_offset_standalone
|
|
|
|
cant_repeat_two_offset_standalone:
|
|
CMPW $0x00000104, R2
|
|
BLO repeat_three_standalone
|
|
CMPW $0x00010100, R2
|
|
BLO repeat_four_standalone
|
|
CMPW $0x0100ffff, R2
|
|
BLO repeat_five_standalone
|
|
SUB $16842747, R2, R2
|
|
MOVWU R2, R2
|
|
MOVD $0xfffb001d, R16
|
|
MOVW R16, (R0)
|
|
MOVD $0xff, R16
|
|
MOVB R16, 4(R0)
|
|
ADD $0x05, R0, R0
|
|
ADD $0x05, R3, R3
|
|
JMP emit_repeat_again_standalone
|
|
|
|
repeat_five_standalone:
|
|
SUB $65536, R2, R2
|
|
MOVWU R2, R2
|
|
MOVWU R2, R1
|
|
MOVD $0x001d, R16
|
|
MOVH R16, (R0)
|
|
MOVH R2, 2(R0)
|
|
ASRW $0x10, R1, R1
|
|
MOVB R1, 4(R0)
|
|
ADD $0x05, R3, R3
|
|
ADD $0x05, R0, R0
|
|
JMP gen_emit_repeat_end
|
|
|
|
repeat_four_standalone:
|
|
SUB $256, R2, R2
|
|
MOVWU R2, R2
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R0)
|
|
MOVH R2, 2(R0)
|
|
ADD $0x04, R3, R3
|
|
ADD $0x04, R0, R0
|
|
JMP gen_emit_repeat_end
|
|
|
|
repeat_three_standalone:
|
|
SUB $4, R2, R2
|
|
MOVWU R2, R2
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R0)
|
|
MOVB R2, 2(R0)
|
|
ADD $0x03, R3, R3
|
|
ADD $0x03, R0, R0
|
|
JMP gen_emit_repeat_end
|
|
|
|
repeat_two_standalone:
|
|
LSLW $0x02, R2, R2
|
|
ORRW $0x01, R2, R2
|
|
MOVH R2, (R0)
|
|
ADD $0x02, R3, R3
|
|
ADD $0x02, R0, R0
|
|
JMP gen_emit_repeat_end
|
|
|
|
repeat_two_offset_standalone:
|
|
MOVD $0, R5
|
|
ADD R2<<2, R5, R2
|
|
ADD $1, R2, R2
|
|
MOVWU R2, R2
|
|
MOVB R1, 1(R0)
|
|
ASRW $0x08, R1, R1
|
|
LSLW $0x05, R1, R1
|
|
ORRW R1, R2, R2
|
|
MOVB R2, (R0)
|
|
ADD $0x02, R3, R3
|
|
ADD $0x02, R0, R0
|
|
|
|
gen_emit_repeat_end:
|
|
MOVD R3, ret+40(FP)
|
|
RET
|
|
|
|
// func emitCopy(dst []byte, offset int, length int) int
|
|
TEXT ·emitCopy(SB), NOSPLIT, $0-48
|
|
MOVD $0, R3
|
|
MOVD dst_base+0(FP), R0
|
|
MOVD offset+24(FP), R1
|
|
MOVD length+32(FP), R2
|
|
|
|
// emitCopy
|
|
CMPW $0x00010000, R1
|
|
BLO two_byte_offset_standalone
|
|
CMPW $0x40, R2
|
|
BLS four_bytes_remain_standalone
|
|
MOVD $0xff, R16
|
|
MOVB R16, (R0)
|
|
MOVW R1, 1(R0)
|
|
SUB $64, R2, R2
|
|
MOVWU R2, R2
|
|
ADD $0x05, R3, R3
|
|
ADD $0x05, R0, R0
|
|
CMPW $0x04, R2
|
|
BLO four_bytes_remain_standalone
|
|
|
|
// emitRepeat
|
|
emit_repeat_again_standalone_emit_copy:
|
|
MOVWU R2, R5
|
|
SUB $4, R2, R2
|
|
MOVWU R2, R2
|
|
CMPW $0x08, R5
|
|
BLS repeat_two_standalone_emit_copy
|
|
CMPW $0x0c, R5
|
|
BHS cant_repeat_two_offset_standalone_emit_copy
|
|
CMPW $0x00000800, R1
|
|
BLO repeat_two_offset_standalone_emit_copy
|
|
|
|
cant_repeat_two_offset_standalone_emit_copy:
|
|
CMPW $0x00000104, R2
|
|
BLO repeat_three_standalone_emit_copy
|
|
CMPW $0x00010100, R2
|
|
BLO repeat_four_standalone_emit_copy
|
|
CMPW $0x0100ffff, R2
|
|
BLO repeat_five_standalone_emit_copy
|
|
SUB $16842747, R2, R2
|
|
MOVWU R2, R2
|
|
MOVD $0xfffb001d, R16
|
|
MOVW R16, (R0)
|
|
MOVD $0xff, R16
|
|
MOVB R16, 4(R0)
|
|
ADD $0x05, R0, R0
|
|
ADD $0x05, R3, R3
|
|
JMP emit_repeat_again_standalone_emit_copy
|
|
|
|
repeat_five_standalone_emit_copy:
|
|
SUB $65536, R2, R2
|
|
MOVWU R2, R2
|
|
MOVWU R2, R1
|
|
MOVD $0x001d, R16
|
|
MOVH R16, (R0)
|
|
MOVH R2, 2(R0)
|
|
ASRW $0x10, R1, R1
|
|
MOVB R1, 4(R0)
|
|
ADD $0x05, R3, R3
|
|
ADD $0x05, R0, R0
|
|
JMP gen_emit_copy_end
|
|
|
|
repeat_four_standalone_emit_copy:
|
|
SUB $256, R2, R2
|
|
MOVWU R2, R2
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R0)
|
|
MOVH R2, 2(R0)
|
|
ADD $0x04, R3, R3
|
|
ADD $0x04, R0, R0
|
|
JMP gen_emit_copy_end
|
|
|
|
repeat_three_standalone_emit_copy:
|
|
SUB $4, R2, R2
|
|
MOVWU R2, R2
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R0)
|
|
MOVB R2, 2(R0)
|
|
ADD $0x03, R3, R3
|
|
ADD $0x03, R0, R0
|
|
JMP gen_emit_copy_end
|
|
|
|
repeat_two_standalone_emit_copy:
|
|
LSLW $0x02, R2, R2
|
|
ORRW $0x01, R2, R2
|
|
MOVH R2, (R0)
|
|
ADD $0x02, R3, R3
|
|
ADD $0x02, R0, R0
|
|
JMP gen_emit_copy_end
|
|
|
|
repeat_two_offset_standalone_emit_copy:
|
|
MOVD $0, R5
|
|
ADD R2<<2, R5, R2
|
|
ADD $1, R2, R2
|
|
MOVWU R2, R2
|
|
MOVB R1, 1(R0)
|
|
ASRW $0x08, R1, R1
|
|
LSLW $0x05, R1, R1
|
|
ORRW R1, R2, R2
|
|
MOVB R2, (R0)
|
|
ADD $0x02, R3, R3
|
|
ADD $0x02, R0, R0
|
|
JMP gen_emit_copy_end
|
|
|
|
four_bytes_remain_standalone:
|
|
TSTW R2, R2
|
|
BEQ gen_emit_copy_end
|
|
MOVD $0, R5
|
|
ADD R2<<2, R5, R2
|
|
SUB $1, R2, R2
|
|
MOVWU R2, R2
|
|
MOVB R2, (R0)
|
|
MOVW R1, 1(R0)
|
|
ADD $0x05, R3, R3
|
|
ADD $0x05, R0, R0
|
|
JMP gen_emit_copy_end
|
|
|
|
two_byte_offset_standalone:
|
|
CMPW $0x40, R2
|
|
BLS two_byte_offset_short_standalone
|
|
CMPW $0x00000800, R1
|
|
BHS long_offset_short_standalone
|
|
MOVD $0x00000001, R5
|
|
ADD $16, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R1, 1(R0)
|
|
MOVWU R1, R6
|
|
LSRW $0x08, R6, R6
|
|
LSLW $0x05, R6, R6
|
|
ORRW R6, R5, R5
|
|
MOVB R5, (R0)
|
|
ADD $0x02, R3, R3
|
|
ADD $0x02, R0, R0
|
|
SUBW $0x08, R2, R2
|
|
|
|
// emitRepeat
|
|
SUB $4, R2, R2
|
|
MOVWU R2, R2
|
|
JMP cant_repeat_two_offset_standalone_emit_copy_short_2b
|
|
|
|
emit_repeat_again_standalone_emit_copy_short_2b:
|
|
MOVWU R2, R5
|
|
SUB $4, R2, R2
|
|
MOVWU R2, R2
|
|
CMPW $0x08, R5
|
|
BLS repeat_two_standalone_emit_copy_short_2b
|
|
CMPW $0x0c, R5
|
|
BHS cant_repeat_two_offset_standalone_emit_copy_short_2b
|
|
CMPW $0x00000800, R1
|
|
BLO repeat_two_offset_standalone_emit_copy_short_2b
|
|
|
|
cant_repeat_two_offset_standalone_emit_copy_short_2b:
|
|
CMPW $0x00000104, R2
|
|
BLO repeat_three_standalone_emit_copy_short_2b
|
|
CMPW $0x00010100, R2
|
|
BLO repeat_four_standalone_emit_copy_short_2b
|
|
CMPW $0x0100ffff, R2
|
|
BLO repeat_five_standalone_emit_copy_short_2b
|
|
SUB $16842747, R2, R2
|
|
MOVWU R2, R2
|
|
MOVD $0xfffb001d, R16
|
|
MOVW R16, (R0)
|
|
MOVD $0xff, R16
|
|
MOVB R16, 4(R0)
|
|
ADD $0x05, R0, R0
|
|
ADD $0x05, R3, R3
|
|
JMP emit_repeat_again_standalone_emit_copy_short_2b
|
|
|
|
repeat_five_standalone_emit_copy_short_2b:
|
|
SUB $65536, R2, R2
|
|
MOVWU R2, R2
|
|
MOVWU R2, R1
|
|
MOVD $0x001d, R16
|
|
MOVH R16, (R0)
|
|
MOVH R2, 2(R0)
|
|
ASRW $0x10, R1, R1
|
|
MOVB R1, 4(R0)
|
|
ADD $0x05, R3, R3
|
|
ADD $0x05, R0, R0
|
|
JMP gen_emit_copy_end
|
|
|
|
repeat_four_standalone_emit_copy_short_2b:
|
|
SUB $256, R2, R2
|
|
MOVWU R2, R2
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R0)
|
|
MOVH R2, 2(R0)
|
|
ADD $0x04, R3, R3
|
|
ADD $0x04, R0, R0
|
|
JMP gen_emit_copy_end
|
|
|
|
repeat_three_standalone_emit_copy_short_2b:
|
|
SUB $4, R2, R2
|
|
MOVWU R2, R2
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R0)
|
|
MOVB R2, 2(R0)
|
|
ADD $0x03, R3, R3
|
|
ADD $0x03, R0, R0
|
|
JMP gen_emit_copy_end
|
|
|
|
repeat_two_standalone_emit_copy_short_2b:
|
|
LSLW $0x02, R2, R2
|
|
ORRW $0x01, R2, R2
|
|
MOVH R2, (R0)
|
|
ADD $0x02, R3, R3
|
|
ADD $0x02, R0, R0
|
|
JMP gen_emit_copy_end
|
|
|
|
repeat_two_offset_standalone_emit_copy_short_2b:
|
|
MOVD $0, R5
|
|
ADD R2<<2, R5, R2
|
|
ADD $1, R2, R2
|
|
MOVWU R2, R2
|
|
MOVB R1, 1(R0)
|
|
ASRW $0x08, R1, R1
|
|
LSLW $0x05, R1, R1
|
|
ORRW R1, R2, R2
|
|
MOVB R2, (R0)
|
|
ADD $0x02, R3, R3
|
|
ADD $0x02, R0, R0
|
|
JMP gen_emit_copy_end
|
|
|
|
long_offset_short_standalone:
|
|
MOVD $0xee, R16
|
|
MOVB R16, (R0)
|
|
MOVH R1, 1(R0)
|
|
SUB $60, R2, R2
|
|
MOVWU R2, R2
|
|
ADD $0x03, R0, R0
|
|
ADD $0x03, R3, R3
|
|
|
|
// emitRepeat
|
|
emit_repeat_again_standalone_emit_copy_short:
|
|
MOVWU R2, R5
|
|
SUB $4, R2, R2
|
|
MOVWU R2, R2
|
|
CMPW $0x08, R5
|
|
BLS repeat_two_standalone_emit_copy_short
|
|
CMPW $0x0c, R5
|
|
BHS cant_repeat_two_offset_standalone_emit_copy_short
|
|
CMPW $0x00000800, R1
|
|
BLO repeat_two_offset_standalone_emit_copy_short
|
|
|
|
cant_repeat_two_offset_standalone_emit_copy_short:
|
|
CMPW $0x00000104, R2
|
|
BLO repeat_three_standalone_emit_copy_short
|
|
CMPW $0x00010100, R2
|
|
BLO repeat_four_standalone_emit_copy_short
|
|
CMPW $0x0100ffff, R2
|
|
BLO repeat_five_standalone_emit_copy_short
|
|
SUB $16842747, R2, R2
|
|
MOVWU R2, R2
|
|
MOVD $0xfffb001d, R16
|
|
MOVW R16, (R0)
|
|
MOVD $0xff, R16
|
|
MOVB R16, 4(R0)
|
|
ADD $0x05, R0, R0
|
|
ADD $0x05, R3, R3
|
|
JMP emit_repeat_again_standalone_emit_copy_short
|
|
|
|
repeat_five_standalone_emit_copy_short:
|
|
SUB $65536, R2, R2
|
|
MOVWU R2, R2
|
|
MOVWU R2, R1
|
|
MOVD $0x001d, R16
|
|
MOVH R16, (R0)
|
|
MOVH R2, 2(R0)
|
|
ASRW $0x10, R1, R1
|
|
MOVB R1, 4(R0)
|
|
ADD $0x05, R3, R3
|
|
ADD $0x05, R0, R0
|
|
JMP gen_emit_copy_end
|
|
|
|
repeat_four_standalone_emit_copy_short:
|
|
SUB $256, R2, R2
|
|
MOVWU R2, R2
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R0)
|
|
MOVH R2, 2(R0)
|
|
ADD $0x04, R3, R3
|
|
ADD $0x04, R0, R0
|
|
JMP gen_emit_copy_end
|
|
|
|
repeat_three_standalone_emit_copy_short:
|
|
SUB $4, R2, R2
|
|
MOVWU R2, R2
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R0)
|
|
MOVB R2, 2(R0)
|
|
ADD $0x03, R3, R3
|
|
ADD $0x03, R0, R0
|
|
JMP gen_emit_copy_end
|
|
|
|
repeat_two_standalone_emit_copy_short:
|
|
LSLW $0x02, R2, R2
|
|
ORRW $0x01, R2, R2
|
|
MOVH R2, (R0)
|
|
ADD $0x02, R3, R3
|
|
ADD $0x02, R0, R0
|
|
JMP gen_emit_copy_end
|
|
|
|
repeat_two_offset_standalone_emit_copy_short:
|
|
MOVD $0, R5
|
|
ADD R2<<2, R5, R2
|
|
ADD $1, R2, R2
|
|
MOVWU R2, R2
|
|
MOVB R1, 1(R0)
|
|
ASRW $0x08, R1, R1
|
|
LSLW $0x05, R1, R1
|
|
ORRW R1, R2, R2
|
|
MOVB R2, (R0)
|
|
ADD $0x02, R3, R3
|
|
ADD $0x02, R0, R0
|
|
JMP gen_emit_copy_end
|
|
|
|
two_byte_offset_short_standalone:
|
|
MOVWU R2, R5
|
|
LSLW $0x02, R5, R5
|
|
CMPW $0x0c, R2
|
|
BHS emit_copy_three_standalone
|
|
CMPW $0x00000800, R1
|
|
BHS emit_copy_three_standalone
|
|
SUB $15, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R1, 1(R0)
|
|
LSRW $0x08, R1, R1
|
|
LSLW $0x05, R1, R1
|
|
ORRW R1, R5, R5
|
|
MOVB R5, (R0)
|
|
ADD $0x02, R3, R3
|
|
ADD $0x02, R0, R0
|
|
JMP gen_emit_copy_end
|
|
|
|
emit_copy_three_standalone:
|
|
SUB $2, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R5, (R0)
|
|
MOVH R1, 1(R0)
|
|
ADD $0x03, R3, R3
|
|
ADD $0x03, R0, R0
|
|
|
|
gen_emit_copy_end:
|
|
MOVD R3, ret+40(FP)
|
|
RET
|
|
|
|
// func emitCopyNoRepeat(dst []byte, offset int, length int) int
|
|
TEXT ·emitCopyNoRepeat(SB), NOSPLIT, $0-48
|
|
MOVD $0, R3
|
|
MOVD dst_base+0(FP), R0
|
|
MOVD offset+24(FP), R1
|
|
MOVD length+32(FP), R2
|
|
|
|
// emitCopy
|
|
CMPW $0x00010000, R1
|
|
BLO two_byte_offset_standalone_snappy
|
|
|
|
four_bytes_loop_back_standalone_snappy:
|
|
CMPW $0x40, R2
|
|
BLS four_bytes_remain_standalone_snappy
|
|
MOVD $0xff, R16
|
|
MOVB R16, (R0)
|
|
MOVW R1, 1(R0)
|
|
SUB $64, R2, R2
|
|
MOVWU R2, R2
|
|
ADD $0x05, R3, R3
|
|
ADD $0x05, R0, R0
|
|
CMPW $0x04, R2
|
|
BLO four_bytes_remain_standalone_snappy
|
|
JMP four_bytes_loop_back_standalone_snappy
|
|
|
|
four_bytes_remain_standalone_snappy:
|
|
TSTW R2, R2
|
|
BEQ gen_emit_copy_end_snappy
|
|
MOVD $0, R5
|
|
ADD R2<<2, R5, R2
|
|
SUB $1, R2, R2
|
|
MOVWU R2, R2
|
|
MOVB R2, (R0)
|
|
MOVW R1, 1(R0)
|
|
ADD $0x05, R3, R3
|
|
ADD $0x05, R0, R0
|
|
JMP gen_emit_copy_end_snappy
|
|
|
|
two_byte_offset_standalone_snappy:
|
|
CMPW $0x40, R2
|
|
BLS two_byte_offset_short_standalone_snappy
|
|
MOVD $0xee, R16
|
|
MOVB R16, (R0)
|
|
MOVH R1, 1(R0)
|
|
SUB $60, R2, R2
|
|
MOVWU R2, R2
|
|
ADD $0x03, R0, R0
|
|
ADD $0x03, R3, R3
|
|
JMP two_byte_offset_standalone_snappy
|
|
|
|
two_byte_offset_short_standalone_snappy:
|
|
MOVWU R2, R5
|
|
LSLW $0x02, R5, R5
|
|
CMPW $0x0c, R2
|
|
BHS emit_copy_three_standalone_snappy
|
|
CMPW $0x00000800, R1
|
|
BHS emit_copy_three_standalone_snappy
|
|
SUB $15, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R1, 1(R0)
|
|
LSRW $0x08, R1, R1
|
|
LSLW $0x05, R1, R1
|
|
ORRW R1, R5, R5
|
|
MOVB R5, (R0)
|
|
ADD $0x02, R3, R3
|
|
ADD $0x02, R0, R0
|
|
JMP gen_emit_copy_end_snappy
|
|
|
|
emit_copy_three_standalone_snappy:
|
|
SUB $2, R5, R5
|
|
MOVWU R5, R5
|
|
MOVB R5, (R0)
|
|
MOVH R1, 1(R0)
|
|
ADD $0x03, R3, R3
|
|
ADD $0x03, R0, R0
|
|
|
|
gen_emit_copy_end_snappy:
|
|
MOVD R3, ret+40(FP)
|
|
RET
|
|
|
|
// func matchLen(a []byte, b []byte) int
|
|
// Requires: BMI
|
|
TEXT ·matchLen(SB), NOSPLIT, $0-56
|
|
MOVD a_base+0(FP), R0
|
|
MOVD b_base+24(FP), R1
|
|
MOVD a_len+8(FP), R2
|
|
|
|
// matchLen
|
|
MOVD $0, R5
|
|
|
|
matchlen_loopback_16_standalone:
|
|
CMPW $0x10, R2
|
|
BLO matchlen_match8_standalone
|
|
MOVD (R0)(R5), R3
|
|
ADD R5, R0, R15
|
|
MOVD 8(R15), R6
|
|
MOVD (R1)(R5), R16
|
|
EOR R16, R3, R3
|
|
TST R3, R3
|
|
BNE matchlen_bsf_8_standalone
|
|
ADD R5, R1, R15
|
|
MOVD 8(R15), R16
|
|
EOR R16, R6, R6
|
|
TST R6, R6
|
|
BNE matchlen_bsf_16standalone
|
|
SUB $16, R2, R2
|
|
MOVWU R2, R2
|
|
ADD $16, R5, R5
|
|
MOVWU R5, R5
|
|
JMP matchlen_loopback_16_standalone
|
|
|
|
matchlen_bsf_16standalone:
|
|
RBIT R6, R6
|
|
CLZ R6, R6
|
|
ASR $0x03, R6, R6
|
|
ADD R6, R5, R5
|
|
ADD $8, R5, R5
|
|
MOVWU R5, R5
|
|
JMP gen_match_len_end
|
|
|
|
matchlen_match8_standalone:
|
|
CMPW $0x08, R2
|
|
BLO matchlen_match4_standalone
|
|
MOVD (R0)(R5), R3
|
|
MOVD (R1)(R5), R16
|
|
EOR R16, R3, R3
|
|
TST R3, R3
|
|
BNE matchlen_bsf_8_standalone
|
|
SUB $8, R2, R2
|
|
MOVWU R2, R2
|
|
ADD $8, R5, R5
|
|
MOVWU R5, R5
|
|
JMP matchlen_match4_standalone
|
|
|
|
matchlen_bsf_8_standalone:
|
|
RBIT R3, R3
|
|
CLZ R3, R3
|
|
ASR $0x03, R3, R3
|
|
ADD R3, R5, R5
|
|
MOVWU R5, R5
|
|
JMP gen_match_len_end
|
|
|
|
matchlen_match4_standalone:
|
|
CMPW $0x04, R2
|
|
BLO matchlen_match2_standalone
|
|
MOVWU (R0)(R5), R3
|
|
MOVWU (R1)(R5), R16
|
|
CMPW R3, R16
|
|
BNE matchlen_match2_standalone
|
|
SUB $4, R2, R2
|
|
MOVWU R2, R2
|
|
ADD $4, R5, R5
|
|
MOVWU R5, R5
|
|
|
|
matchlen_match2_standalone:
|
|
CMPW $0x01, R2
|
|
BEQ matchlen_match1_standalone
|
|
BLO gen_match_len_end
|
|
MOVHU (R0)(R5), R16
|
|
BFI $0, R16, $16, R3
|
|
ADD R5, R1, R15
|
|
MOVHU (R15), R15
|
|
AND $0xffff, R3, R16
|
|
CMP R16, R15
|
|
BNE matchlen_match1_standalone
|
|
ADD $2, R5, R5
|
|
MOVWU R5, R5
|
|
SUBSW $0x02, R2, R2
|
|
BEQ gen_match_len_end
|
|
|
|
matchlen_match1_standalone:
|
|
MOVBU (R0)(R5), R16
|
|
BFI $0, R16, $8, R3
|
|
ADD R5, R1, R15
|
|
MOVBU (R15), R15
|
|
AND $0xff, R3, R16
|
|
CMP R16, R15
|
|
BNE gen_match_len_end
|
|
ADD $1, R5, R5
|
|
MOVWU R5, R5
|
|
|
|
gen_match_len_end:
|
|
MOVD R5, ret+48(FP)
|
|
RET
|
|
|
|
// func cvtLZ4BlockAsm(dst []byte, src []byte) (uncompressed int, dstUsed int)
|
|
// Requires: SSE2
|
|
TEXT ·cvtLZ4BlockAsm(SB), NOSPLIT, $0-64
|
|
MOVD $0, R5
|
|
MOVD dst_base+0(FP), R0
|
|
MOVD dst_len+8(FP), R1
|
|
MOVD src_base+24(FP), R2
|
|
MOVD src_len+32(FP), R3
|
|
ADD R3, R2, R3
|
|
ADD R1, R0, R1
|
|
SUB $8, R1, R1
|
|
MOVD $0, R6
|
|
|
|
lz4_s2_loop:
|
|
CMP R3, R2
|
|
BHS lz4_s2_corrupt
|
|
CMP R1, R0
|
|
BHS lz4_s2_dstfull
|
|
MOVBU (R2), R7
|
|
MOVD R7, R8
|
|
MOVD R7, R9
|
|
LSR $0x04, R8, R8
|
|
AND $0x0f, R9, R9
|
|
CMP $0xf0, R7
|
|
BLO lz4_s2_ll_end
|
|
|
|
lz4_s2_ll_loop:
|
|
ADD $1, R2, R2
|
|
CMP R3, R2
|
|
BHS lz4_s2_corrupt
|
|
MOVBU (R2), R7
|
|
ADD R7, R8, R8
|
|
CMP $0xff, R7
|
|
BEQ lz4_s2_ll_loop
|
|
|
|
lz4_s2_ll_end:
|
|
ADD R8, R2, R7
|
|
ADD $0x04, R9, R9
|
|
CMP R3, R7
|
|
BHS lz4_s2_corrupt
|
|
ADD $1, R2, R2
|
|
ADD $1, R7, R7
|
|
TST R8, R8
|
|
BEQ lz4_s2_lits_done
|
|
ADD R8, R0, R10
|
|
CMP R1, R10
|
|
BHS lz4_s2_dstfull
|
|
ADD R8, R5, R5
|
|
SUB $1, R8, R10
|
|
MOVWU R10, R10
|
|
CMPW $0x3c, R10
|
|
BLO one_byte_lz4_s2
|
|
CMPW $0x00000100, R10
|
|
BLO two_bytes_lz4_s2
|
|
CMPW $0x00010000, R10
|
|
BLO three_bytes_lz4_s2
|
|
CMPW $0x01000000, R10
|
|
BLO four_bytes_lz4_s2
|
|
MOVD $0xfc, R16
|
|
MOVB R16, (R0)
|
|
MOVW R10, 1(R0)
|
|
ADD $0x05, R0, R0
|
|
JMP memmove_long_lz4_s2
|
|
|
|
four_bytes_lz4_s2:
|
|
MOVWU R10, R11
|
|
LSRW $0x10, R11, R11
|
|
MOVD $0xf8, R16
|
|
MOVB R16, (R0)
|
|
MOVH R10, 1(R0)
|
|
MOVB R11, 3(R0)
|
|
ADD $0x04, R0, R0
|
|
JMP memmove_long_lz4_s2
|
|
|
|
three_bytes_lz4_s2:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R0)
|
|
MOVH R10, 1(R0)
|
|
ADD $0x03, R0, R0
|
|
JMP memmove_long_lz4_s2
|
|
|
|
two_bytes_lz4_s2:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R0)
|
|
MOVB R10, 1(R0)
|
|
ADD $0x02, R0, R0
|
|
CMPW $0x40, R10
|
|
BLO memmove_lz4_s2
|
|
JMP memmove_long_lz4_s2
|
|
|
|
one_byte_lz4_s2:
|
|
LSLW $0x02, R10, R16
|
|
BFI $0, R16, $8, R10
|
|
MOVB R10, (R0)
|
|
ADD $0x01, R0, R0
|
|
|
|
memmove_lz4_s2:
|
|
ADD R8, R0, R10
|
|
|
|
// genMemMoveShort
|
|
CMP $0x08, R8
|
|
BLS emit_lit_memmove_lz4_s2_memmove_move_8
|
|
CMP $0x10, R8
|
|
BLS emit_lit_memmove_lz4_s2_memmove_move_8through16
|
|
CMP $0x20, R8
|
|
BLS emit_lit_memmove_lz4_s2_memmove_move_17through32
|
|
JMP emit_lit_memmove_lz4_s2_memmove_move_33through64
|
|
|
|
emit_lit_memmove_lz4_s2_memmove_move_8:
|
|
MOVD (R2), R11
|
|
MOVD R11, (R0)
|
|
JMP memmove_end_copy_lz4_s2
|
|
|
|
emit_lit_memmove_lz4_s2_memmove_move_8through16:
|
|
MOVD (R2), R11
|
|
ADD R8, R2, R15
|
|
MOVD -8(R15), R2
|
|
MOVD R11, (R0)
|
|
ADD R8, R0, R15
|
|
MOVD R2, -8(R15)
|
|
JMP memmove_end_copy_lz4_s2
|
|
|
|
emit_lit_memmove_lz4_s2_memmove_move_17through32:
|
|
FMOVQ (R2), F0
|
|
ADD R8, R2, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R0)
|
|
ADD R8, R0, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_lz4_s2
|
|
|
|
emit_lit_memmove_lz4_s2_memmove_move_33through64:
|
|
FMOVQ (R2), F0
|
|
FMOVQ 16(R2), F1
|
|
ADD R8, R2, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R8, R2, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R0)
|
|
FMOVQ F1, 16(R0)
|
|
ADD R8, R0, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R8, R0, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_lz4_s2:
|
|
MOVD R10, R0
|
|
JMP lz4_s2_lits_emit_done
|
|
|
|
memmove_long_lz4_s2:
|
|
ADD R8, R0, R10
|
|
|
|
// genMemMoveLong
|
|
MOVD R2, R11
|
|
MOVD R0, R12
|
|
MOVD R8, R13
|
|
|
|
emit_lit_memmove_long_lz4_s2large_big_loop_back:
|
|
FMOVQ (R11), F0
|
|
FMOVQ 16(R11), F1
|
|
FMOVQ F0, (R12)
|
|
FMOVQ F1, 16(R12)
|
|
ADD $0x20, R11, R11
|
|
ADD $0x20, R12, R12
|
|
SUB $0x20, R13, R13
|
|
CMP $0x20, R13
|
|
BHS emit_lit_memmove_long_lz4_s2large_big_loop_back
|
|
ADD R8, R2, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R8, R2, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R8, R0, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R8, R0, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R10, R0
|
|
|
|
lz4_s2_lits_emit_done:
|
|
MOVD R7, R2
|
|
|
|
lz4_s2_lits_done:
|
|
CMP R3, R2
|
|
BNE lz4_s2_match
|
|
CMP $0x04, R9
|
|
BEQ lz4_s2_done
|
|
JMP lz4_s2_corrupt
|
|
|
|
lz4_s2_match:
|
|
ADD $2, R2, R7
|
|
CMP R3, R7
|
|
BHS lz4_s2_corrupt
|
|
MOVHU (R2), R8
|
|
MOVD R7, R2
|
|
TST R8, R8
|
|
BEQ lz4_s2_corrupt
|
|
CMP R5, R8
|
|
BHI lz4_s2_corrupt
|
|
CMP $0x13, R9
|
|
BNE lz4_s2_ml_done
|
|
|
|
lz4_s2_ml_loop:
|
|
MOVBU (R2), R7
|
|
ADD $1, R2, R2
|
|
ADD R7, R9, R9
|
|
CMP R3, R2
|
|
BHS lz4_s2_corrupt
|
|
CMP $0xff, R7
|
|
BEQ lz4_s2_ml_loop
|
|
|
|
lz4_s2_ml_done:
|
|
ADD R9, R5, R5
|
|
CMP R6, R8
|
|
BNE lz4_s2_docopy
|
|
|
|
// emitRepeat
|
|
emit_repeat_again_lz4_s2:
|
|
MOVWU R9, R7
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
CMPW $0x08, R7
|
|
BLS repeat_two_lz4_s2
|
|
CMPW $0x0c, R7
|
|
BHS cant_repeat_two_offset_lz4_s2
|
|
CMPW $0x00000800, R8
|
|
BLO repeat_two_offset_lz4_s2
|
|
|
|
cant_repeat_two_offset_lz4_s2:
|
|
CMPW $0x00000104, R9
|
|
BLO repeat_three_lz4_s2
|
|
CMPW $0x00010100, R9
|
|
BLO repeat_four_lz4_s2
|
|
CMPW $0x0100ffff, R9
|
|
BLO repeat_five_lz4_s2
|
|
SUB $16842747, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0xfffb001d, R16
|
|
MOVW R16, (R0)
|
|
MOVD $0xff, R16
|
|
MOVB R16, 4(R0)
|
|
ADD $0x05, R0, R0
|
|
JMP emit_repeat_again_lz4_s2
|
|
|
|
repeat_five_lz4_s2:
|
|
SUB $65536, R9, R9
|
|
MOVWU R9, R9
|
|
MOVWU R9, R8
|
|
MOVD $0x001d, R16
|
|
MOVH R16, (R0)
|
|
MOVH R9, 2(R0)
|
|
ASRW $0x10, R8, R8
|
|
MOVB R8, 4(R0)
|
|
ADD $0x05, R0, R0
|
|
JMP lz4_s2_loop
|
|
|
|
repeat_four_lz4_s2:
|
|
SUB $256, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R0)
|
|
MOVH R9, 2(R0)
|
|
ADD $0x04, R0, R0
|
|
JMP lz4_s2_loop
|
|
|
|
repeat_three_lz4_s2:
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R0)
|
|
MOVB R9, 2(R0)
|
|
ADD $0x03, R0, R0
|
|
JMP lz4_s2_loop
|
|
|
|
repeat_two_lz4_s2:
|
|
LSLW $0x02, R9, R9
|
|
ORRW $0x01, R9, R9
|
|
MOVH R9, (R0)
|
|
ADD $0x02, R0, R0
|
|
JMP lz4_s2_loop
|
|
|
|
repeat_two_offset_lz4_s2:
|
|
MOVD $0, R7
|
|
ADD R9<<2, R7, R9
|
|
ADD $1, R9, R9
|
|
MOVWU R9, R9
|
|
MOVB R8, 1(R0)
|
|
ASRW $0x08, R8, R8
|
|
LSLW $0x05, R8, R8
|
|
ORRW R8, R9, R9
|
|
MOVB R9, (R0)
|
|
ADD $0x02, R0, R0
|
|
JMP lz4_s2_loop
|
|
|
|
lz4_s2_docopy:
|
|
MOVD R8, R6
|
|
|
|
// emitCopy
|
|
CMPW $0x40, R9
|
|
BLS two_byte_offset_short_lz4_s2
|
|
CMPW $0x00000800, R8
|
|
BHS long_offset_short_lz4_s2
|
|
MOVD $0x00000001, R7
|
|
ADD $16, R7, R7
|
|
MOVWU R7, R7
|
|
MOVB R8, 1(R0)
|
|
MOVWU R8, R10
|
|
LSRW $0x08, R10, R10
|
|
LSLW $0x05, R10, R10
|
|
ORRW R10, R7, R7
|
|
MOVB R7, (R0)
|
|
ADD $0x02, R0, R0
|
|
SUBW $0x08, R9, R9
|
|
|
|
// emitRepeat
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
JMP cant_repeat_two_offset_lz4_s2_emit_copy_short_2b
|
|
|
|
emit_repeat_again_lz4_s2_emit_copy_short_2b:
|
|
MOVWU R9, R7
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
CMPW $0x08, R7
|
|
BLS repeat_two_lz4_s2_emit_copy_short_2b
|
|
CMPW $0x0c, R7
|
|
BHS cant_repeat_two_offset_lz4_s2_emit_copy_short_2b
|
|
CMPW $0x00000800, R8
|
|
BLO repeat_two_offset_lz4_s2_emit_copy_short_2b
|
|
|
|
cant_repeat_two_offset_lz4_s2_emit_copy_short_2b:
|
|
CMPW $0x00000104, R9
|
|
BLO repeat_three_lz4_s2_emit_copy_short_2b
|
|
CMPW $0x00010100, R9
|
|
BLO repeat_four_lz4_s2_emit_copy_short_2b
|
|
CMPW $0x0100ffff, R9
|
|
BLO repeat_five_lz4_s2_emit_copy_short_2b
|
|
SUB $16842747, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0xfffb001d, R16
|
|
MOVW R16, (R0)
|
|
MOVD $0xff, R16
|
|
MOVB R16, 4(R0)
|
|
ADD $0x05, R0, R0
|
|
JMP emit_repeat_again_lz4_s2_emit_copy_short_2b
|
|
|
|
repeat_five_lz4_s2_emit_copy_short_2b:
|
|
SUB $65536, R9, R9
|
|
MOVWU R9, R9
|
|
MOVWU R9, R8
|
|
MOVD $0x001d, R16
|
|
MOVH R16, (R0)
|
|
MOVH R9, 2(R0)
|
|
ASRW $0x10, R8, R8
|
|
MOVB R8, 4(R0)
|
|
ADD $0x05, R0, R0
|
|
JMP lz4_s2_loop
|
|
|
|
repeat_four_lz4_s2_emit_copy_short_2b:
|
|
SUB $256, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R0)
|
|
MOVH R9, 2(R0)
|
|
ADD $0x04, R0, R0
|
|
JMP lz4_s2_loop
|
|
|
|
repeat_three_lz4_s2_emit_copy_short_2b:
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R0)
|
|
MOVB R9, 2(R0)
|
|
ADD $0x03, R0, R0
|
|
JMP lz4_s2_loop
|
|
|
|
repeat_two_lz4_s2_emit_copy_short_2b:
|
|
LSLW $0x02, R9, R9
|
|
ORRW $0x01, R9, R9
|
|
MOVH R9, (R0)
|
|
ADD $0x02, R0, R0
|
|
JMP lz4_s2_loop
|
|
|
|
repeat_two_offset_lz4_s2_emit_copy_short_2b:
|
|
MOVD $0, R7
|
|
ADD R9<<2, R7, R9
|
|
ADD $1, R9, R9
|
|
MOVWU R9, R9
|
|
MOVB R8, 1(R0)
|
|
ASRW $0x08, R8, R8
|
|
LSLW $0x05, R8, R8
|
|
ORRW R8, R9, R9
|
|
MOVB R9, (R0)
|
|
ADD $0x02, R0, R0
|
|
JMP lz4_s2_loop
|
|
|
|
long_offset_short_lz4_s2:
|
|
MOVD $0xee, R16
|
|
MOVB R16, (R0)
|
|
MOVH R8, 1(R0)
|
|
SUB $60, R9, R9
|
|
MOVWU R9, R9
|
|
ADD $0x03, R0, R0
|
|
|
|
// emitRepeat
|
|
emit_repeat_again_lz4_s2_emit_copy_short:
|
|
MOVWU R9, R7
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
CMPW $0x08, R7
|
|
BLS repeat_two_lz4_s2_emit_copy_short
|
|
CMPW $0x0c, R7
|
|
BHS cant_repeat_two_offset_lz4_s2_emit_copy_short
|
|
CMPW $0x00000800, R8
|
|
BLO repeat_two_offset_lz4_s2_emit_copy_short
|
|
|
|
cant_repeat_two_offset_lz4_s2_emit_copy_short:
|
|
CMPW $0x00000104, R9
|
|
BLO repeat_three_lz4_s2_emit_copy_short
|
|
CMPW $0x00010100, R9
|
|
BLO repeat_four_lz4_s2_emit_copy_short
|
|
CMPW $0x0100ffff, R9
|
|
BLO repeat_five_lz4_s2_emit_copy_short
|
|
SUB $16842747, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0xfffb001d, R16
|
|
MOVW R16, (R0)
|
|
MOVD $0xff, R16
|
|
MOVB R16, 4(R0)
|
|
ADD $0x05, R0, R0
|
|
JMP emit_repeat_again_lz4_s2_emit_copy_short
|
|
|
|
repeat_five_lz4_s2_emit_copy_short:
|
|
SUB $65536, R9, R9
|
|
MOVWU R9, R9
|
|
MOVWU R9, R8
|
|
MOVD $0x001d, R16
|
|
MOVH R16, (R0)
|
|
MOVH R9, 2(R0)
|
|
ASRW $0x10, R8, R8
|
|
MOVB R8, 4(R0)
|
|
ADD $0x05, R0, R0
|
|
JMP lz4_s2_loop
|
|
|
|
repeat_four_lz4_s2_emit_copy_short:
|
|
SUB $256, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R0)
|
|
MOVH R9, 2(R0)
|
|
ADD $0x04, R0, R0
|
|
JMP lz4_s2_loop
|
|
|
|
repeat_three_lz4_s2_emit_copy_short:
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R0)
|
|
MOVB R9, 2(R0)
|
|
ADD $0x03, R0, R0
|
|
JMP lz4_s2_loop
|
|
|
|
repeat_two_lz4_s2_emit_copy_short:
|
|
LSLW $0x02, R9, R9
|
|
ORRW $0x01, R9, R9
|
|
MOVH R9, (R0)
|
|
ADD $0x02, R0, R0
|
|
JMP lz4_s2_loop
|
|
|
|
repeat_two_offset_lz4_s2_emit_copy_short:
|
|
MOVD $0, R7
|
|
ADD R9<<2, R7, R9
|
|
ADD $1, R9, R9
|
|
MOVWU R9, R9
|
|
MOVB R8, 1(R0)
|
|
ASRW $0x08, R8, R8
|
|
LSLW $0x05, R8, R8
|
|
ORRW R8, R9, R9
|
|
MOVB R9, (R0)
|
|
ADD $0x02, R0, R0
|
|
JMP lz4_s2_loop
|
|
|
|
two_byte_offset_short_lz4_s2:
|
|
MOVWU R9, R7
|
|
LSLW $0x02, R7, R7
|
|
CMPW $0x0c, R9
|
|
BHS emit_copy_three_lz4_s2
|
|
CMPW $0x00000800, R8
|
|
BHS emit_copy_three_lz4_s2
|
|
SUB $15, R7, R7
|
|
MOVWU R7, R7
|
|
MOVB R8, 1(R0)
|
|
LSRW $0x08, R8, R8
|
|
LSLW $0x05, R8, R8
|
|
ORRW R8, R7, R7
|
|
MOVB R7, (R0)
|
|
ADD $0x02, R0, R0
|
|
JMP lz4_s2_loop
|
|
|
|
emit_copy_three_lz4_s2:
|
|
SUB $2, R7, R7
|
|
MOVWU R7, R7
|
|
MOVB R7, (R0)
|
|
MOVH R8, 1(R0)
|
|
ADD $0x03, R0, R0
|
|
JMP lz4_s2_loop
|
|
|
|
lz4_s2_done:
|
|
MOVD dst_base+0(FP), R1
|
|
SUB R1, R0, R0
|
|
MOVD R5, uncompressed+48(FP)
|
|
MOVD R0, dstUsed+56(FP)
|
|
RET
|
|
|
|
lz4_s2_corrupt:
|
|
MOVD $0, R0
|
|
SUB $1, R0, R5
|
|
MOVD R5, uncompressed+48(FP)
|
|
RET
|
|
|
|
lz4_s2_dstfull:
|
|
MOVD $0, R0
|
|
SUB $2, R0, R5
|
|
MOVD R5, uncompressed+48(FP)
|
|
RET
|
|
|
|
// func cvtLZ4sBlockAsm(dst []byte, src []byte) (uncompressed int, dstUsed int)
|
|
// Requires: SSE2
|
|
TEXT ·cvtLZ4sBlockAsm(SB), NOSPLIT, $0-64
|
|
MOVD $0, R5
|
|
MOVD dst_base+0(FP), R0
|
|
MOVD dst_len+8(FP), R1
|
|
MOVD src_base+24(FP), R2
|
|
MOVD src_len+32(FP), R3
|
|
ADD R3, R2, R3
|
|
ADD R1, R0, R1
|
|
SUB $8, R1, R1
|
|
MOVD $0, R6
|
|
|
|
lz4s_s2_loop:
|
|
CMP R3, R2
|
|
BHS lz4s_s2_corrupt
|
|
CMP R1, R0
|
|
BHS lz4s_s2_dstfull
|
|
MOVBU (R2), R7
|
|
MOVD R7, R8
|
|
MOVD R7, R9
|
|
LSR $0x04, R8, R8
|
|
AND $0x0f, R9, R9
|
|
CMP $0xf0, R7
|
|
BLO lz4s_s2_ll_end
|
|
|
|
lz4s_s2_ll_loop:
|
|
ADD $1, R2, R2
|
|
CMP R3, R2
|
|
BHS lz4s_s2_corrupt
|
|
MOVBU (R2), R7
|
|
ADD R7, R8, R8
|
|
CMP $0xff, R7
|
|
BEQ lz4s_s2_ll_loop
|
|
|
|
lz4s_s2_ll_end:
|
|
ADD R8, R2, R7
|
|
ADD $0x03, R9, R9
|
|
CMP R3, R7
|
|
BHS lz4s_s2_corrupt
|
|
ADD $1, R2, R2
|
|
ADD $1, R7, R7
|
|
TST R8, R8
|
|
BEQ lz4s_s2_lits_done
|
|
ADD R8, R0, R10
|
|
CMP R1, R10
|
|
BHS lz4s_s2_dstfull
|
|
ADD R8, R5, R5
|
|
SUB $1, R8, R10
|
|
MOVWU R10, R10
|
|
CMPW $0x3c, R10
|
|
BLO one_byte_lz4s_s2
|
|
CMPW $0x00000100, R10
|
|
BLO two_bytes_lz4s_s2
|
|
CMPW $0x00010000, R10
|
|
BLO three_bytes_lz4s_s2
|
|
CMPW $0x01000000, R10
|
|
BLO four_bytes_lz4s_s2
|
|
MOVD $0xfc, R16
|
|
MOVB R16, (R0)
|
|
MOVW R10, 1(R0)
|
|
ADD $0x05, R0, R0
|
|
JMP memmove_long_lz4s_s2
|
|
|
|
four_bytes_lz4s_s2:
|
|
MOVWU R10, R11
|
|
LSRW $0x10, R11, R11
|
|
MOVD $0xf8, R16
|
|
MOVB R16, (R0)
|
|
MOVH R10, 1(R0)
|
|
MOVB R11, 3(R0)
|
|
ADD $0x04, R0, R0
|
|
JMP memmove_long_lz4s_s2
|
|
|
|
three_bytes_lz4s_s2:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R0)
|
|
MOVH R10, 1(R0)
|
|
ADD $0x03, R0, R0
|
|
JMP memmove_long_lz4s_s2
|
|
|
|
two_bytes_lz4s_s2:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R0)
|
|
MOVB R10, 1(R0)
|
|
ADD $0x02, R0, R0
|
|
CMPW $0x40, R10
|
|
BLO memmove_lz4s_s2
|
|
JMP memmove_long_lz4s_s2
|
|
|
|
one_byte_lz4s_s2:
|
|
LSLW $0x02, R10, R16
|
|
BFI $0, R16, $8, R10
|
|
MOVB R10, (R0)
|
|
ADD $0x01, R0, R0
|
|
|
|
memmove_lz4s_s2:
|
|
ADD R8, R0, R10
|
|
|
|
// genMemMoveShort
|
|
CMP $0x08, R8
|
|
BLS emit_lit_memmove_lz4s_s2_memmove_move_8
|
|
CMP $0x10, R8
|
|
BLS emit_lit_memmove_lz4s_s2_memmove_move_8through16
|
|
CMP $0x20, R8
|
|
BLS emit_lit_memmove_lz4s_s2_memmove_move_17through32
|
|
JMP emit_lit_memmove_lz4s_s2_memmove_move_33through64
|
|
|
|
emit_lit_memmove_lz4s_s2_memmove_move_8:
|
|
MOVD (R2), R11
|
|
MOVD R11, (R0)
|
|
JMP memmove_end_copy_lz4s_s2
|
|
|
|
emit_lit_memmove_lz4s_s2_memmove_move_8through16:
|
|
MOVD (R2), R11
|
|
ADD R8, R2, R15
|
|
MOVD -8(R15), R2
|
|
MOVD R11, (R0)
|
|
ADD R8, R0, R15
|
|
MOVD R2, -8(R15)
|
|
JMP memmove_end_copy_lz4s_s2
|
|
|
|
emit_lit_memmove_lz4s_s2_memmove_move_17through32:
|
|
FMOVQ (R2), F0
|
|
ADD R8, R2, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R0)
|
|
ADD R8, R0, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_lz4s_s2
|
|
|
|
emit_lit_memmove_lz4s_s2_memmove_move_33through64:
|
|
FMOVQ (R2), F0
|
|
FMOVQ 16(R2), F1
|
|
ADD R8, R2, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R8, R2, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R0)
|
|
FMOVQ F1, 16(R0)
|
|
ADD R8, R0, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R8, R0, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_lz4s_s2:
|
|
MOVD R10, R0
|
|
JMP lz4s_s2_lits_emit_done
|
|
|
|
memmove_long_lz4s_s2:
|
|
ADD R8, R0, R10
|
|
|
|
// genMemMoveLong
|
|
MOVD R2, R11
|
|
MOVD R0, R12
|
|
MOVD R8, R13
|
|
|
|
emit_lit_memmove_long_lz4s_s2large_big_loop_back:
|
|
FMOVQ (R11), F0
|
|
FMOVQ 16(R11), F1
|
|
FMOVQ F0, (R12)
|
|
FMOVQ F1, 16(R12)
|
|
ADD $0x20, R11, R11
|
|
ADD $0x20, R12, R12
|
|
SUB $0x20, R13, R13
|
|
CMP $0x20, R13
|
|
BHS emit_lit_memmove_long_lz4s_s2large_big_loop_back
|
|
ADD R8, R2, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R8, R2, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R8, R0, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R8, R0, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R10, R0
|
|
|
|
lz4s_s2_lits_emit_done:
|
|
MOVD R7, R2
|
|
|
|
lz4s_s2_lits_done:
|
|
CMP R3, R2
|
|
BNE lz4s_s2_match
|
|
CMP $0x03, R9
|
|
BEQ lz4s_s2_done
|
|
JMP lz4s_s2_corrupt
|
|
|
|
lz4s_s2_match:
|
|
CMP $0x03, R9
|
|
BEQ lz4s_s2_loop
|
|
ADD $2, R2, R7
|
|
CMP R3, R7
|
|
BHS lz4s_s2_corrupt
|
|
MOVHU (R2), R8
|
|
MOVD R7, R2
|
|
TST R8, R8
|
|
BEQ lz4s_s2_corrupt
|
|
CMP R5, R8
|
|
BHI lz4s_s2_corrupt
|
|
CMP $0x12, R9
|
|
BNE lz4s_s2_ml_done
|
|
|
|
lz4s_s2_ml_loop:
|
|
MOVBU (R2), R7
|
|
ADD $1, R2, R2
|
|
ADD R7, R9, R9
|
|
CMP R3, R2
|
|
BHS lz4s_s2_corrupt
|
|
CMP $0xff, R7
|
|
BEQ lz4s_s2_ml_loop
|
|
|
|
lz4s_s2_ml_done:
|
|
ADD R9, R5, R5
|
|
CMP R6, R8
|
|
BNE lz4s_s2_docopy
|
|
|
|
// emitRepeat
|
|
emit_repeat_again_lz4_s2:
|
|
MOVWU R9, R7
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
CMPW $0x08, R7
|
|
BLS repeat_two_lz4_s2
|
|
CMPW $0x0c, R7
|
|
BHS cant_repeat_two_offset_lz4_s2
|
|
CMPW $0x00000800, R8
|
|
BLO repeat_two_offset_lz4_s2
|
|
|
|
cant_repeat_two_offset_lz4_s2:
|
|
CMPW $0x00000104, R9
|
|
BLO repeat_three_lz4_s2
|
|
CMPW $0x00010100, R9
|
|
BLO repeat_four_lz4_s2
|
|
CMPW $0x0100ffff, R9
|
|
BLO repeat_five_lz4_s2
|
|
SUB $16842747, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0xfffb001d, R16
|
|
MOVW R16, (R0)
|
|
MOVD $0xff, R16
|
|
MOVB R16, 4(R0)
|
|
ADD $0x05, R0, R0
|
|
JMP emit_repeat_again_lz4_s2
|
|
|
|
repeat_five_lz4_s2:
|
|
SUB $65536, R9, R9
|
|
MOVWU R9, R9
|
|
MOVWU R9, R8
|
|
MOVD $0x001d, R16
|
|
MOVH R16, (R0)
|
|
MOVH R9, 2(R0)
|
|
ASRW $0x10, R8, R8
|
|
MOVB R8, 4(R0)
|
|
ADD $0x05, R0, R0
|
|
JMP lz4s_s2_loop
|
|
|
|
repeat_four_lz4_s2:
|
|
SUB $256, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R0)
|
|
MOVH R9, 2(R0)
|
|
ADD $0x04, R0, R0
|
|
JMP lz4s_s2_loop
|
|
|
|
repeat_three_lz4_s2:
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R0)
|
|
MOVB R9, 2(R0)
|
|
ADD $0x03, R0, R0
|
|
JMP lz4s_s2_loop
|
|
|
|
repeat_two_lz4_s2:
|
|
LSLW $0x02, R9, R9
|
|
ORRW $0x01, R9, R9
|
|
MOVH R9, (R0)
|
|
ADD $0x02, R0, R0
|
|
JMP lz4s_s2_loop
|
|
|
|
repeat_two_offset_lz4_s2:
|
|
MOVD $0, R7
|
|
ADD R9<<2, R7, R9
|
|
ADD $1, R9, R9
|
|
MOVWU R9, R9
|
|
MOVB R8, 1(R0)
|
|
ASRW $0x08, R8, R8
|
|
LSLW $0x05, R8, R8
|
|
ORRW R8, R9, R9
|
|
MOVB R9, (R0)
|
|
ADD $0x02, R0, R0
|
|
JMP lz4s_s2_loop
|
|
|
|
lz4s_s2_docopy:
|
|
MOVD R8, R6
|
|
|
|
// emitCopy
|
|
CMPW $0x40, R9
|
|
BLS two_byte_offset_short_lz4_s2
|
|
CMPW $0x00000800, R8
|
|
BHS long_offset_short_lz4_s2
|
|
MOVD $0x00000001, R7
|
|
ADD $16, R7, R7
|
|
MOVWU R7, R7
|
|
MOVB R8, 1(R0)
|
|
MOVWU R8, R10
|
|
LSRW $0x08, R10, R10
|
|
LSLW $0x05, R10, R10
|
|
ORRW R10, R7, R7
|
|
MOVB R7, (R0)
|
|
ADD $0x02, R0, R0
|
|
SUBW $0x08, R9, R9
|
|
|
|
// emitRepeat
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
JMP cant_repeat_two_offset_lz4_s2_emit_copy_short_2b
|
|
|
|
emit_repeat_again_lz4_s2_emit_copy_short_2b:
|
|
MOVWU R9, R7
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
CMPW $0x08, R7
|
|
BLS repeat_two_lz4_s2_emit_copy_short_2b
|
|
CMPW $0x0c, R7
|
|
BHS cant_repeat_two_offset_lz4_s2_emit_copy_short_2b
|
|
CMPW $0x00000800, R8
|
|
BLO repeat_two_offset_lz4_s2_emit_copy_short_2b
|
|
|
|
cant_repeat_two_offset_lz4_s2_emit_copy_short_2b:
|
|
CMPW $0x00000104, R9
|
|
BLO repeat_three_lz4_s2_emit_copy_short_2b
|
|
CMPW $0x00010100, R9
|
|
BLO repeat_four_lz4_s2_emit_copy_short_2b
|
|
CMPW $0x0100ffff, R9
|
|
BLO repeat_five_lz4_s2_emit_copy_short_2b
|
|
SUB $16842747, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0xfffb001d, R16
|
|
MOVW R16, (R0)
|
|
MOVD $0xff, R16
|
|
MOVB R16, 4(R0)
|
|
ADD $0x05, R0, R0
|
|
JMP emit_repeat_again_lz4_s2_emit_copy_short_2b
|
|
|
|
repeat_five_lz4_s2_emit_copy_short_2b:
|
|
SUB $65536, R9, R9
|
|
MOVWU R9, R9
|
|
MOVWU R9, R8
|
|
MOVD $0x001d, R16
|
|
MOVH R16, (R0)
|
|
MOVH R9, 2(R0)
|
|
ASRW $0x10, R8, R8
|
|
MOVB R8, 4(R0)
|
|
ADD $0x05, R0, R0
|
|
JMP lz4s_s2_loop
|
|
|
|
repeat_four_lz4_s2_emit_copy_short_2b:
|
|
SUB $256, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R0)
|
|
MOVH R9, 2(R0)
|
|
ADD $0x04, R0, R0
|
|
JMP lz4s_s2_loop
|
|
|
|
repeat_three_lz4_s2_emit_copy_short_2b:
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R0)
|
|
MOVB R9, 2(R0)
|
|
ADD $0x03, R0, R0
|
|
JMP lz4s_s2_loop
|
|
|
|
repeat_two_lz4_s2_emit_copy_short_2b:
|
|
LSLW $0x02, R9, R9
|
|
ORRW $0x01, R9, R9
|
|
MOVH R9, (R0)
|
|
ADD $0x02, R0, R0
|
|
JMP lz4s_s2_loop
|
|
|
|
repeat_two_offset_lz4_s2_emit_copy_short_2b:
|
|
MOVD $0, R7
|
|
ADD R9<<2, R7, R9
|
|
ADD $1, R9, R9
|
|
MOVWU R9, R9
|
|
MOVB R8, 1(R0)
|
|
ASRW $0x08, R8, R8
|
|
LSLW $0x05, R8, R8
|
|
ORRW R8, R9, R9
|
|
MOVB R9, (R0)
|
|
ADD $0x02, R0, R0
|
|
JMP lz4s_s2_loop
|
|
|
|
long_offset_short_lz4_s2:
|
|
MOVD $0xee, R16
|
|
MOVB R16, (R0)
|
|
MOVH R8, 1(R0)
|
|
SUB $60, R9, R9
|
|
MOVWU R9, R9
|
|
ADD $0x03, R0, R0
|
|
|
|
// emitRepeat
|
|
emit_repeat_again_lz4_s2_emit_copy_short:
|
|
MOVWU R9, R7
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
CMPW $0x08, R7
|
|
BLS repeat_two_lz4_s2_emit_copy_short
|
|
CMPW $0x0c, R7
|
|
BHS cant_repeat_two_offset_lz4_s2_emit_copy_short
|
|
CMPW $0x00000800, R8
|
|
BLO repeat_two_offset_lz4_s2_emit_copy_short
|
|
|
|
cant_repeat_two_offset_lz4_s2_emit_copy_short:
|
|
CMPW $0x00000104, R9
|
|
BLO repeat_three_lz4_s2_emit_copy_short
|
|
CMPW $0x00010100, R9
|
|
BLO repeat_four_lz4_s2_emit_copy_short
|
|
CMPW $0x0100ffff, R9
|
|
BLO repeat_five_lz4_s2_emit_copy_short
|
|
SUB $16842747, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0xfffb001d, R16
|
|
MOVW R16, (R0)
|
|
MOVD $0xff, R16
|
|
MOVB R16, 4(R0)
|
|
ADD $0x05, R0, R0
|
|
JMP emit_repeat_again_lz4_s2_emit_copy_short
|
|
|
|
repeat_five_lz4_s2_emit_copy_short:
|
|
SUB $65536, R9, R9
|
|
MOVWU R9, R9
|
|
MOVWU R9, R8
|
|
MOVD $0x001d, R16
|
|
MOVH R16, (R0)
|
|
MOVH R9, 2(R0)
|
|
ASRW $0x10, R8, R8
|
|
MOVB R8, 4(R0)
|
|
ADD $0x05, R0, R0
|
|
JMP lz4s_s2_loop
|
|
|
|
repeat_four_lz4_s2_emit_copy_short:
|
|
SUB $256, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0x0019, R16
|
|
MOVH R16, (R0)
|
|
MOVH R9, 2(R0)
|
|
ADD $0x04, R0, R0
|
|
JMP lz4s_s2_loop
|
|
|
|
repeat_three_lz4_s2_emit_copy_short:
|
|
SUB $4, R9, R9
|
|
MOVWU R9, R9
|
|
MOVD $0x0015, R16
|
|
MOVH R16, (R0)
|
|
MOVB R9, 2(R0)
|
|
ADD $0x03, R0, R0
|
|
JMP lz4s_s2_loop
|
|
|
|
repeat_two_lz4_s2_emit_copy_short:
|
|
LSLW $0x02, R9, R9
|
|
ORRW $0x01, R9, R9
|
|
MOVH R9, (R0)
|
|
ADD $0x02, R0, R0
|
|
JMP lz4s_s2_loop
|
|
|
|
repeat_two_offset_lz4_s2_emit_copy_short:
|
|
MOVD $0, R7
|
|
ADD R9<<2, R7, R9
|
|
ADD $1, R9, R9
|
|
MOVWU R9, R9
|
|
MOVB R8, 1(R0)
|
|
ASRW $0x08, R8, R8
|
|
LSLW $0x05, R8, R8
|
|
ORRW R8, R9, R9
|
|
MOVB R9, (R0)
|
|
ADD $0x02, R0, R0
|
|
JMP lz4s_s2_loop
|
|
|
|
two_byte_offset_short_lz4_s2:
|
|
MOVWU R9, R7
|
|
LSLW $0x02, R7, R7
|
|
CMPW $0x0c, R9
|
|
BHS emit_copy_three_lz4_s2
|
|
CMPW $0x00000800, R8
|
|
BHS emit_copy_three_lz4_s2
|
|
SUB $15, R7, R7
|
|
MOVWU R7, R7
|
|
MOVB R8, 1(R0)
|
|
LSRW $0x08, R8, R8
|
|
LSLW $0x05, R8, R8
|
|
ORRW R8, R7, R7
|
|
MOVB R7, (R0)
|
|
ADD $0x02, R0, R0
|
|
JMP lz4s_s2_loop
|
|
|
|
emit_copy_three_lz4_s2:
|
|
SUB $2, R7, R7
|
|
MOVWU R7, R7
|
|
MOVB R7, (R0)
|
|
MOVH R8, 1(R0)
|
|
ADD $0x03, R0, R0
|
|
JMP lz4s_s2_loop
|
|
|
|
lz4s_s2_done:
|
|
MOVD dst_base+0(FP), R1
|
|
SUB R1, R0, R0
|
|
MOVD R5, uncompressed+48(FP)
|
|
MOVD R0, dstUsed+56(FP)
|
|
RET
|
|
|
|
lz4s_s2_corrupt:
|
|
MOVD $0, R0
|
|
SUB $1, R0, R5
|
|
MOVD R5, uncompressed+48(FP)
|
|
RET
|
|
|
|
lz4s_s2_dstfull:
|
|
MOVD $0, R0
|
|
SUB $2, R0, R5
|
|
MOVD R5, uncompressed+48(FP)
|
|
RET
|
|
|
|
// func cvtLZ4BlockSnappyAsm(dst []byte, src []byte) (uncompressed int, dstUsed int)
|
|
// Requires: SSE2
|
|
TEXT ·cvtLZ4BlockSnappyAsm(SB), NOSPLIT, $0-64
|
|
MOVD $0, R5
|
|
MOVD dst_base+0(FP), R0
|
|
MOVD dst_len+8(FP), R1
|
|
MOVD src_base+24(FP), R2
|
|
MOVD src_len+32(FP), R3
|
|
ADD R3, R2, R3
|
|
ADD R1, R0, R1
|
|
SUB $8, R1, R1
|
|
|
|
lz4_snappy_loop:
|
|
CMP R3, R2
|
|
BHS lz4_snappy_corrupt
|
|
CMP R1, R0
|
|
BHS lz4_snappy_dstfull
|
|
MOVBU (R2), R6
|
|
MOVD R6, R7
|
|
MOVD R6, R8
|
|
LSR $0x04, R7, R7
|
|
AND $0x0f, R8, R8
|
|
CMP $0xf0, R6
|
|
BLO lz4_snappy_ll_end
|
|
|
|
lz4_snappy_ll_loop:
|
|
ADD $1, R2, R2
|
|
CMP R3, R2
|
|
BHS lz4_snappy_corrupt
|
|
MOVBU (R2), R6
|
|
ADD R6, R7, R7
|
|
CMP $0xff, R6
|
|
BEQ lz4_snappy_ll_loop
|
|
|
|
lz4_snappy_ll_end:
|
|
ADD R7, R2, R6
|
|
ADD $0x04, R8, R8
|
|
CMP R3, R6
|
|
BHS lz4_snappy_corrupt
|
|
ADD $1, R2, R2
|
|
ADD $1, R6, R6
|
|
TST R7, R7
|
|
BEQ lz4_snappy_lits_done
|
|
ADD R7, R0, R9
|
|
CMP R1, R9
|
|
BHS lz4_snappy_dstfull
|
|
ADD R7, R5, R5
|
|
SUB $1, R7, R9
|
|
MOVWU R9, R9
|
|
CMPW $0x3c, R9
|
|
BLO one_byte_lz4_snappy
|
|
CMPW $0x00000100, R9
|
|
BLO two_bytes_lz4_snappy
|
|
CMPW $0x00010000, R9
|
|
BLO three_bytes_lz4_snappy
|
|
CMPW $0x01000000, R9
|
|
BLO four_bytes_lz4_snappy
|
|
MOVD $0xfc, R16
|
|
MOVB R16, (R0)
|
|
MOVW R9, 1(R0)
|
|
ADD $0x05, R0, R0
|
|
JMP memmove_long_lz4_snappy
|
|
|
|
four_bytes_lz4_snappy:
|
|
MOVWU R9, R10
|
|
LSRW $0x10, R10, R10
|
|
MOVD $0xf8, R16
|
|
MOVB R16, (R0)
|
|
MOVH R9, 1(R0)
|
|
MOVB R10, 3(R0)
|
|
ADD $0x04, R0, R0
|
|
JMP memmove_long_lz4_snappy
|
|
|
|
three_bytes_lz4_snappy:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R0)
|
|
MOVH R9, 1(R0)
|
|
ADD $0x03, R0, R0
|
|
JMP memmove_long_lz4_snappy
|
|
|
|
two_bytes_lz4_snappy:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R0)
|
|
MOVB R9, 1(R0)
|
|
ADD $0x02, R0, R0
|
|
CMPW $0x40, R9
|
|
BLO memmove_lz4_snappy
|
|
JMP memmove_long_lz4_snappy
|
|
|
|
one_byte_lz4_snappy:
|
|
LSLW $0x02, R9, R16
|
|
BFI $0, R16, $8, R9
|
|
MOVB R9, (R0)
|
|
ADD $0x01, R0, R0
|
|
|
|
memmove_lz4_snappy:
|
|
ADD R7, R0, R9
|
|
|
|
// genMemMoveShort
|
|
CMP $0x08, R7
|
|
BLS emit_lit_memmove_lz4_snappy_memmove_move_8
|
|
CMP $0x10, R7
|
|
BLS emit_lit_memmove_lz4_snappy_memmove_move_8through16
|
|
CMP $0x20, R7
|
|
BLS emit_lit_memmove_lz4_snappy_memmove_move_17through32
|
|
JMP emit_lit_memmove_lz4_snappy_memmove_move_33through64
|
|
|
|
emit_lit_memmove_lz4_snappy_memmove_move_8:
|
|
MOVD (R2), R10
|
|
MOVD R10, (R0)
|
|
JMP memmove_end_copy_lz4_snappy
|
|
|
|
emit_lit_memmove_lz4_snappy_memmove_move_8through16:
|
|
MOVD (R2), R10
|
|
ADD R7, R2, R15
|
|
MOVD -8(R15), R2
|
|
MOVD R10, (R0)
|
|
ADD R7, R0, R15
|
|
MOVD R2, -8(R15)
|
|
JMP memmove_end_copy_lz4_snappy
|
|
|
|
emit_lit_memmove_lz4_snappy_memmove_move_17through32:
|
|
FMOVQ (R2), F0
|
|
ADD R7, R2, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R0)
|
|
ADD R7, R0, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_lz4_snappy
|
|
|
|
emit_lit_memmove_lz4_snappy_memmove_move_33through64:
|
|
FMOVQ (R2), F0
|
|
FMOVQ 16(R2), F1
|
|
ADD R7, R2, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R7, R2, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R0)
|
|
FMOVQ F1, 16(R0)
|
|
ADD R7, R0, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R7, R0, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_lz4_snappy:
|
|
MOVD R9, R0
|
|
JMP lz4_snappy_lits_emit_done
|
|
|
|
memmove_long_lz4_snappy:
|
|
ADD R7, R0, R9
|
|
|
|
// genMemMoveLong
|
|
MOVD R2, R10
|
|
MOVD R0, R11
|
|
MOVD R7, R12
|
|
|
|
emit_lit_memmove_long_lz4_snappylarge_big_loop_back:
|
|
FMOVQ (R10), F0
|
|
FMOVQ 16(R10), F1
|
|
FMOVQ F0, (R11)
|
|
FMOVQ F1, 16(R11)
|
|
ADD $0x20, R10, R10
|
|
ADD $0x20, R11, R11
|
|
SUB $0x20, R12, R12
|
|
CMP $0x20, R12
|
|
BHS emit_lit_memmove_long_lz4_snappylarge_big_loop_back
|
|
ADD R7, R2, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R7, R2, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R7, R0, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R7, R0, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R9, R0
|
|
|
|
lz4_snappy_lits_emit_done:
|
|
MOVD R6, R2
|
|
|
|
lz4_snappy_lits_done:
|
|
CMP R3, R2
|
|
BNE lz4_snappy_match
|
|
CMP $0x04, R8
|
|
BEQ lz4_snappy_done
|
|
JMP lz4_snappy_corrupt
|
|
|
|
lz4_snappy_match:
|
|
ADD $2, R2, R6
|
|
CMP R3, R6
|
|
BHS lz4_snappy_corrupt
|
|
MOVHU (R2), R7
|
|
MOVD R6, R2
|
|
TST R7, R7
|
|
BEQ lz4_snappy_corrupt
|
|
CMP R5, R7
|
|
BHI lz4_snappy_corrupt
|
|
CMP $0x13, R8
|
|
BNE lz4_snappy_ml_done
|
|
|
|
lz4_snappy_ml_loop:
|
|
MOVBU (R2), R6
|
|
ADD $1, R2, R2
|
|
ADD R6, R8, R8
|
|
CMP R3, R2
|
|
BHS lz4_snappy_corrupt
|
|
CMP $0xff, R6
|
|
BEQ lz4_snappy_ml_loop
|
|
|
|
lz4_snappy_ml_done:
|
|
ADD R8, R5, R5
|
|
|
|
// emitCopy
|
|
two_byte_offset_lz4_s2:
|
|
CMPW $0x40, R8
|
|
BLS two_byte_offset_short_lz4_s2
|
|
MOVD $0xee, R16
|
|
MOVB R16, (R0)
|
|
MOVH R7, 1(R0)
|
|
SUB $60, R8, R8
|
|
MOVWU R8, R8
|
|
ADD $0x03, R0, R0
|
|
CMP R1, R0
|
|
BHS lz4_snappy_loop
|
|
JMP two_byte_offset_lz4_s2
|
|
|
|
two_byte_offset_short_lz4_s2:
|
|
MOVWU R8, R6
|
|
LSLW $0x02, R6, R6
|
|
CMPW $0x0c, R8
|
|
BHS emit_copy_three_lz4_s2
|
|
CMPW $0x00000800, R7
|
|
BHS emit_copy_three_lz4_s2
|
|
SUB $15, R6, R6
|
|
MOVWU R6, R6
|
|
MOVB R7, 1(R0)
|
|
LSRW $0x08, R7, R7
|
|
LSLW $0x05, R7, R7
|
|
ORRW R7, R6, R6
|
|
MOVB R6, (R0)
|
|
ADD $0x02, R0, R0
|
|
JMP lz4_snappy_loop
|
|
|
|
emit_copy_three_lz4_s2:
|
|
SUB $2, R6, R6
|
|
MOVWU R6, R6
|
|
MOVB R6, (R0)
|
|
MOVH R7, 1(R0)
|
|
ADD $0x03, R0, R0
|
|
JMP lz4_snappy_loop
|
|
|
|
lz4_snappy_done:
|
|
MOVD dst_base+0(FP), R1
|
|
SUB R1, R0, R0
|
|
MOVD R5, uncompressed+48(FP)
|
|
MOVD R0, dstUsed+56(FP)
|
|
RET
|
|
|
|
lz4_snappy_corrupt:
|
|
MOVD $0, R0
|
|
SUB $1, R0, R5
|
|
MOVD R5, uncompressed+48(FP)
|
|
RET
|
|
|
|
lz4_snappy_dstfull:
|
|
MOVD $0, R0
|
|
SUB $2, R0, R5
|
|
MOVD R5, uncompressed+48(FP)
|
|
RET
|
|
|
|
// func cvtLZ4sBlockSnappyAsm(dst []byte, src []byte) (uncompressed int, dstUsed int)
|
|
// Requires: SSE2
|
|
TEXT ·cvtLZ4sBlockSnappyAsm(SB), NOSPLIT, $0-64
|
|
MOVD $0, R5
|
|
MOVD dst_base+0(FP), R0
|
|
MOVD dst_len+8(FP), R1
|
|
MOVD src_base+24(FP), R2
|
|
MOVD src_len+32(FP), R3
|
|
ADD R3, R2, R3
|
|
ADD R1, R0, R1
|
|
SUB $8, R1, R1
|
|
|
|
lz4s_snappy_loop:
|
|
CMP R3, R2
|
|
BHS lz4s_snappy_corrupt
|
|
CMP R1, R0
|
|
BHS lz4s_snappy_dstfull
|
|
MOVBU (R2), R6
|
|
MOVD R6, R7
|
|
MOVD R6, R8
|
|
LSR $0x04, R7, R7
|
|
AND $0x0f, R8, R8
|
|
CMP $0xf0, R6
|
|
BLO lz4s_snappy_ll_end
|
|
|
|
lz4s_snappy_ll_loop:
|
|
ADD $1, R2, R2
|
|
CMP R3, R2
|
|
BHS lz4s_snappy_corrupt
|
|
MOVBU (R2), R6
|
|
ADD R6, R7, R7
|
|
CMP $0xff, R6
|
|
BEQ lz4s_snappy_ll_loop
|
|
|
|
lz4s_snappy_ll_end:
|
|
ADD R7, R2, R6
|
|
ADD $0x03, R8, R8
|
|
CMP R3, R6
|
|
BHS lz4s_snappy_corrupt
|
|
ADD $1, R2, R2
|
|
ADD $1, R6, R6
|
|
TST R7, R7
|
|
BEQ lz4s_snappy_lits_done
|
|
ADD R7, R0, R9
|
|
CMP R1, R9
|
|
BHS lz4s_snappy_dstfull
|
|
ADD R7, R5, R5
|
|
SUB $1, R7, R9
|
|
MOVWU R9, R9
|
|
CMPW $0x3c, R9
|
|
BLO one_byte_lz4s_snappy
|
|
CMPW $0x00000100, R9
|
|
BLO two_bytes_lz4s_snappy
|
|
CMPW $0x00010000, R9
|
|
BLO three_bytes_lz4s_snappy
|
|
CMPW $0x01000000, R9
|
|
BLO four_bytes_lz4s_snappy
|
|
MOVD $0xfc, R16
|
|
MOVB R16, (R0)
|
|
MOVW R9, 1(R0)
|
|
ADD $0x05, R0, R0
|
|
JMP memmove_long_lz4s_snappy
|
|
|
|
four_bytes_lz4s_snappy:
|
|
MOVWU R9, R10
|
|
LSRW $0x10, R10, R10
|
|
MOVD $0xf8, R16
|
|
MOVB R16, (R0)
|
|
MOVH R9, 1(R0)
|
|
MOVB R10, 3(R0)
|
|
ADD $0x04, R0, R0
|
|
JMP memmove_long_lz4s_snappy
|
|
|
|
three_bytes_lz4s_snappy:
|
|
MOVD $0xf4, R16
|
|
MOVB R16, (R0)
|
|
MOVH R9, 1(R0)
|
|
ADD $0x03, R0, R0
|
|
JMP memmove_long_lz4s_snappy
|
|
|
|
two_bytes_lz4s_snappy:
|
|
MOVD $0xf0, R16
|
|
MOVB R16, (R0)
|
|
MOVB R9, 1(R0)
|
|
ADD $0x02, R0, R0
|
|
CMPW $0x40, R9
|
|
BLO memmove_lz4s_snappy
|
|
JMP memmove_long_lz4s_snappy
|
|
|
|
one_byte_lz4s_snappy:
|
|
LSLW $0x02, R9, R16
|
|
BFI $0, R16, $8, R9
|
|
MOVB R9, (R0)
|
|
ADD $0x01, R0, R0
|
|
|
|
memmove_lz4s_snappy:
|
|
ADD R7, R0, R9
|
|
|
|
// genMemMoveShort
|
|
CMP $0x08, R7
|
|
BLS emit_lit_memmove_lz4s_snappy_memmove_move_8
|
|
CMP $0x10, R7
|
|
BLS emit_lit_memmove_lz4s_snappy_memmove_move_8through16
|
|
CMP $0x20, R7
|
|
BLS emit_lit_memmove_lz4s_snappy_memmove_move_17through32
|
|
JMP emit_lit_memmove_lz4s_snappy_memmove_move_33through64
|
|
|
|
emit_lit_memmove_lz4s_snappy_memmove_move_8:
|
|
MOVD (R2), R10
|
|
MOVD R10, (R0)
|
|
JMP memmove_end_copy_lz4s_snappy
|
|
|
|
emit_lit_memmove_lz4s_snappy_memmove_move_8through16:
|
|
MOVD (R2), R10
|
|
ADD R7, R2, R15
|
|
MOVD -8(R15), R2
|
|
MOVD R10, (R0)
|
|
ADD R7, R0, R15
|
|
MOVD R2, -8(R15)
|
|
JMP memmove_end_copy_lz4s_snappy
|
|
|
|
emit_lit_memmove_lz4s_snappy_memmove_move_17through32:
|
|
FMOVQ (R2), F0
|
|
ADD R7, R2, R15
|
|
FMOVQ -16(R15), F1
|
|
FMOVQ F0, (R0)
|
|
ADD R7, R0, R15
|
|
FMOVQ F1, -16(R15)
|
|
JMP memmove_end_copy_lz4s_snappy
|
|
|
|
emit_lit_memmove_lz4s_snappy_memmove_move_33through64:
|
|
FMOVQ (R2), F0
|
|
FMOVQ 16(R2), F1
|
|
ADD R7, R2, R15
|
|
FMOVQ -32(R15), F2
|
|
ADD R7, R2, R15
|
|
FMOVQ -16(R15), F3
|
|
FMOVQ F0, (R0)
|
|
FMOVQ F1, 16(R0)
|
|
ADD R7, R0, R15
|
|
FMOVQ F2, -32(R15)
|
|
ADD R7, R0, R15
|
|
FMOVQ F3, -16(R15)
|
|
|
|
memmove_end_copy_lz4s_snappy:
|
|
MOVD R9, R0
|
|
JMP lz4s_snappy_lits_emit_done
|
|
|
|
memmove_long_lz4s_snappy:
|
|
ADD R7, R0, R9
|
|
|
|
// genMemMoveLong
|
|
MOVD R2, R10
|
|
MOVD R0, R11
|
|
MOVD R7, R12
|
|
|
|
emit_lit_memmove_long_lz4s_snappylarge_big_loop_back:
|
|
FMOVQ (R10), F0
|
|
FMOVQ 16(R10), F1
|
|
FMOVQ F0, (R11)
|
|
FMOVQ F1, 16(R11)
|
|
ADD $0x20, R10, R10
|
|
ADD $0x20, R11, R11
|
|
SUB $0x20, R12, R12
|
|
CMP $0x20, R12
|
|
BHS emit_lit_memmove_long_lz4s_snappylarge_big_loop_back
|
|
ADD R7, R2, R15
|
|
FMOVQ -32(R15), F0
|
|
ADD R7, R2, R15
|
|
FMOVQ -16(R15), F1
|
|
ADD R7, R0, R15
|
|
FMOVQ F0, -32(R15)
|
|
ADD R7, R0, R15
|
|
FMOVQ F1, -16(R15)
|
|
MOVD R9, R0
|
|
|
|
lz4s_snappy_lits_emit_done:
|
|
MOVD R6, R2
|
|
|
|
lz4s_snappy_lits_done:
|
|
CMP R3, R2
|
|
BNE lz4s_snappy_match
|
|
CMP $0x03, R8
|
|
BEQ lz4s_snappy_done
|
|
JMP lz4s_snappy_corrupt
|
|
|
|
lz4s_snappy_match:
|
|
CMP $0x03, R8
|
|
BEQ lz4s_snappy_loop
|
|
ADD $2, R2, R6
|
|
CMP R3, R6
|
|
BHS lz4s_snappy_corrupt
|
|
MOVHU (R2), R7
|
|
MOVD R6, R2
|
|
TST R7, R7
|
|
BEQ lz4s_snappy_corrupt
|
|
CMP R5, R7
|
|
BHI lz4s_snappy_corrupt
|
|
CMP $0x12, R8
|
|
BNE lz4s_snappy_ml_done
|
|
|
|
lz4s_snappy_ml_loop:
|
|
MOVBU (R2), R6
|
|
ADD $1, R2, R2
|
|
ADD R6, R8, R8
|
|
CMP R3, R2
|
|
BHS lz4s_snappy_corrupt
|
|
CMP $0xff, R6
|
|
BEQ lz4s_snappy_ml_loop
|
|
|
|
lz4s_snappy_ml_done:
|
|
ADD R8, R5, R5
|
|
|
|
// emitCopy
|
|
two_byte_offset_lz4_s2:
|
|
CMPW $0x40, R8
|
|
BLS two_byte_offset_short_lz4_s2
|
|
MOVD $0xee, R16
|
|
MOVB R16, (R0)
|
|
MOVH R7, 1(R0)
|
|
SUB $60, R8, R8
|
|
MOVWU R8, R8
|
|
ADD $0x03, R0, R0
|
|
CMP R1, R0
|
|
BHS lz4s_snappy_loop
|
|
JMP two_byte_offset_lz4_s2
|
|
|
|
two_byte_offset_short_lz4_s2:
|
|
MOVWU R8, R6
|
|
LSLW $0x02, R6, R6
|
|
CMPW $0x0c, R8
|
|
BHS emit_copy_three_lz4_s2
|
|
CMPW $0x00000800, R7
|
|
BHS emit_copy_three_lz4_s2
|
|
SUB $15, R6, R6
|
|
MOVWU R6, R6
|
|
MOVB R7, 1(R0)
|
|
LSRW $0x08, R7, R7
|
|
LSLW $0x05, R7, R7
|
|
ORRW R7, R6, R6
|
|
MOVB R6, (R0)
|
|
ADD $0x02, R0, R0
|
|
JMP lz4s_snappy_loop
|
|
|
|
emit_copy_three_lz4_s2:
|
|
SUB $2, R6, R6
|
|
MOVWU R6, R6
|
|
MOVB R6, (R0)
|
|
MOVH R7, 1(R0)
|
|
ADD $0x03, R0, R0
|
|
JMP lz4s_snappy_loop
|
|
|
|
lz4s_snappy_done:
|
|
MOVD dst_base+0(FP), R1
|
|
SUB R1, R0, R0
|
|
MOVD R5, uncompressed+48(FP)
|
|
MOVD R0, dstUsed+56(FP)
|
|
RET
|
|
|
|
lz4s_snappy_corrupt:
|
|
MOVD $0, R0
|
|
SUB $1, R0, R5
|
|
MOVD R5, uncompressed+48(FP)
|
|
RET
|
|
|
|
lz4s_snappy_dstfull:
|
|
MOVD $0, R0
|
|
SUB $2, R0, R5
|
|
MOVD R5, uncompressed+48(FP)
|
|
RET
|