crypto/fipsmodule/modes/asm/ghash-armv4.pl - boringssl - Git at Google

 #! /usr/bin/env perl
 # Copyright 2010-2018 The OpenSSL Project Authors. All Rights Reserved.
 #
 # Licensed under the OpenSSL license (the "License").  You may not use
 # this file except in compliance with the License.  You can obtain a copy
 # in the file LICENSE in the source distribution or at
 # https://www.openssl.org/source/license.html

 #
 # ====================================================================
 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
 # project. The module is, however, dual licensed under OpenSSL and
 # CRYPTOGAMS licenses depending on where you obtain it. For further
 # details see http://www.openssl.org/~appro/cryptogams/.
 # ====================================================================
 #
 # April 2010
 #
 # The module implements "4-bit" GCM GHASH function and underlying
 # single multiplication operation in GF(2^128). "4-bit" means that it
 # uses 256 bytes per-key table [+32 bytes shared table]. There is no
 # experimental performance data available yet. The only approximation
 # that can be made at this point is based on code size. Inner loop is
 # 32 instructions long and on single-issue core should execute in <40
 # cycles. Having verified that gcc 3.4 didn't unroll corresponding
 # loop, this assembler loop body was found to be ~3x smaller than
 # compiler-generated one...
 #
 # July 2010
 #
 # Rescheduling for dual-issue pipeline resulted in 8.5% improvement on
 # Cortex A8 core and ~25 cycles per processed byte (which was observed
 # to be ~3 times faster than gcc-generated code:-)
 #
 # February 2011
 #
 # Profiler-assisted and platform-specific optimization resulted in 7%
 # improvement on Cortex A8 core and ~23.5 cycles per byte.
 #
 # March 2011
 #
 # Add NEON implementation featuring polynomial multiplication, i.e. no
 # lookup tables involved. On Cortex A8 it was measured to process one
 # byte in 15 cycles or 55% faster than integer-only code.
 #
 # April 2014
 #
 # Switch to multiplication algorithm suggested in paper referred
 # below and combine it with reduction algorithm from x86 module.
 # Performance improvement over previous version varies from 65% on
 # Snapdragon S4 to 110% on Cortex A9. In absolute terms Cortex A8
 # processes one byte in 8.45 cycles, A9 - in 10.2, A15 - in 7.63,
 # Snapdragon S4 - in 9.33.
 #
 # Câmara, D.; Gouvêa, C. P. L.; López, J. & Dahab, R.: Fast Software
 # Polynomial Multiplication on ARM Processors using the NEON Engine.
 #
 # http://conradoplg.cryptoland.net/files/2010/12/mocrysen13.pdf

 # ====================================================================
 # Note about "528B" variant. In ARM case it makes lesser sense to
 # implement it for following reasons:
 #
 # - performance improvement won't be anywhere near 50%, because 128-
 #   bit shift operation is neatly fused with 128-bit xor here, and
 #   "538B" variant would eliminate only 4-5 instructions out of 32
 #   in the inner loop (meaning that estimated improvement is ~15%);
 # - ARM-based systems are often embedded ones and extra memory
 #   consumption might be unappreciated (for so little improvement);
 #
 # Byte order [in]dependence. =========================================
 #
 # Caller is expected to maintain specific *dword* order in Htable,
 # namely with *least* significant dword of 128-bit value at *lower*
 # address. This differs completely from C code and has everything to
 # do with ldm instruction and order in which dwords are "consumed" by
 # algorithm. *Byte* order within these dwords in turn is whatever
 # *native* byte order on current platform. See gcm128.c for working
 # example...

 # This file was patched in BoringSSL to remove the variable-time 4-bit
 # implementation.

 $flavour = shift;
 if ($flavour=~/\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; }
 else { while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} }

 if ($flavour && $flavour ne "void") {
     $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
     ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
     ( $xlate="${dir}../../../perlasm/arm-xlate.pl" and -f $xlate) or
     die "can't locate arm-xlate.pl";

     open OUT,"| \"$^X\" $xlate $flavour $output";
     *STDOUT=*OUT;
 } else {
     open OUT,">$output";
     *STDOUT=*OUT;
 }

 $Xi="r0";	# argument block
 $Htbl="r1";
 $inp="r2";
 $len="r3";

 $code=<<___;
 #include <openssl/arm_arch.h>

 @ Silence ARMv8 deprecated IT instruction warnings. This file is used by both
 @ ARMv7 and ARMv8 processors and does not use ARMv8 instructions. (ARMv8 PMULL
 @ instructions are in aesv8-armx.pl.)
 .arch  armv7-a

 .text
 #if defined(__thumb2__) || defined(__clang__)
 .syntax	unified
 #define ldrplb  ldrbpl
 #define ldrneb  ldrbne
 #endif
 #if defined(__thumb2__)
 .thumb
 #else
 .code	32
 #endif
 ___
 {
 my ($Xl,$Xm,$Xh,$IN)=map("q$_",(0..3));
 my ($t0,$t1,$t2,$t3)=map("q$_",(8..12));
 my ($Hlo,$Hhi,$Hhl,$k48,$k32,$k16)=map("d$_",(26..31));

 sub clmul64x64 {
 my ($r,$a,$b)=@_;
 $code.=<<___;
 	vext.8		$t0#lo, $a, $a, #1	@ A1
 	vmull.p8	$t0, $t0#lo, $b		@ F = A1*B
 	vext.8		$r#lo, $b, $b, #1	@ B1
 	vmull.p8	$r, $a, $r#lo		@ E = A*B1
 	vext.8		$t1#lo, $a, $a, #2	@ A2
 	vmull.p8	$t1, $t1#lo, $b		@ H = A2*B
 	vext.8		$t3#lo, $b, $b, #2	@ B2
 	vmull.p8	$t3, $a, $t3#lo		@ G = A*B2
 	vext.8		$t2#lo, $a, $a, #3	@ A3
 	veor		$t0, $t0, $r		@ L = E + F
 	vmull.p8	$t2, $t2#lo, $b		@ J = A3*B
 	vext.8		$r#lo, $b, $b, #3	@ B3
 	veor		$t1, $t1, $t3		@ M = G + H
 	vmull.p8	$r, $a, $r#lo		@ I = A*B3
 	veor		$t0#lo, $t0#lo, $t0#hi	@ t0 = (L) (P0 + P1) << 8
 	vand		$t0#hi, $t0#hi, $k48
 	vext.8		$t3#lo, $b, $b, #4	@ B4
 	veor		$t1#lo, $t1#lo, $t1#hi	@ t1 = (M) (P2 + P3) << 16
 	vand		$t1#hi, $t1#hi, $k32
 	vmull.p8	$t3, $a, $t3#lo		@ K = A*B4
 	veor		$t2, $t2, $r		@ N = I + J
 	veor		$t0#lo, $t0#lo, $t0#hi
 	veor		$t1#lo, $t1#lo, $t1#hi
 	veor		$t2#lo, $t2#lo, $t2#hi	@ t2 = (N) (P4 + P5) << 24
 	vand		$t2#hi, $t2#hi, $k16
 	vext.8		$t0, $t0, $t0, #15
 	veor		$t3#lo, $t3#lo, $t3#hi	@ t3 = (K) (P6 + P7) << 32
 	vmov.i64	$t3#hi, #0
 	vext.8		$t1, $t1, $t1, #14
 	veor		$t2#lo, $t2#lo, $t2#hi
 	vmull.p8	$r, $a, $b		@ D = A*B
 	vext.8		$t3, $t3, $t3, #12
 	vext.8		$t2, $t2, $t2, #13
 	veor		$t0, $t0, $t1
 	veor		$t2, $t2, $t3
 	veor		$r, $r, $t0
 	veor		$r, $r, $t2
 ___
 }

 $code.=<<___;
 #if __ARM_MAX_ARCH__>=7
 .arch	armv7-a
 .fpu	neon

 .global	gcm_init_neon
 .type	gcm_init_neon,%function
 .align	4
 gcm_init_neon:
 	vld1.64		$IN#hi,[r1]!		@ load H
 	vmov.i8		$t0,#0xe1
 	vld1.64		$IN#lo,[r1]
 	vshl.i64	$t0#hi,#57
 	vshr.u64	$t0#lo,#63		@ t0=0xc2....01
 	vdup.8		$t1,$IN#hi[7]
 	vshr.u64	$Hlo,$IN#lo,#63
 	vshr.s8		$t1,#7			@ broadcast carry bit
 	vshl.i64	$IN,$IN,#1
 	vand		$t0,$t0,$t1
 	vorr		$IN#hi,$Hlo		@ H<<<=1
 	veor		$IN,$IN,$t0		@ twisted H
 	vstmia		r0,{$IN}

 	ret					@ bx lr
 .size	gcm_init_neon,.-gcm_init_neon

 .global	gcm_gmult_neon
 .type	gcm_gmult_neon,%function
 .align	4
 gcm_gmult_neon:
 	vld1.64		$IN#hi,[$Xi]!		@ load Xi
 	vld1.64		$IN#lo,[$Xi]!
 	vmov.i64	$k48,#0x0000ffffffffffff
 	vldmia		$Htbl,{$Hlo-$Hhi}	@ load twisted H
 	vmov.i64	$k32,#0x00000000ffffffff
 #ifdef __ARMEL__
 	vrev64.8	$IN,$IN
 #endif
 	vmov.i64	$k16,#0x000000000000ffff
 	veor		$Hhl,$Hlo,$Hhi		@ Karatsuba pre-processing
 	mov		$len,#16
 	b		.Lgmult_neon
 .size	gcm_gmult_neon,.-gcm_gmult_neon

 .global	gcm_ghash_neon
 .type	gcm_ghash_neon,%function
 .align	4
 gcm_ghash_neon:
 	vld1.64		$Xl#hi,[$Xi]!		@ load Xi
 	vld1.64		$Xl#lo,[$Xi]!
 	vmov.i64	$k48,#0x0000ffffffffffff
 	vldmia		$Htbl,{$Hlo-$Hhi}	@ load twisted H
 	vmov.i64	$k32,#0x00000000ffffffff
 #ifdef __ARMEL__
 	vrev64.8	$Xl,$Xl
 #endif
 	vmov.i64	$k16,#0x000000000000ffff
 	veor		$Hhl,$Hlo,$Hhi		@ Karatsuba pre-processing

 .Loop_neon:
 	vld1.64		$IN#hi,[$inp]!		@ load inp
 	vld1.64		$IN#lo,[$inp]!
 #ifdef __ARMEL__
 	vrev64.8	$IN,$IN
 #endif
 	veor		$IN,$Xl			@ inp^=Xi
 .Lgmult_neon:
 ___
 	&clmul64x64	($Xl,$Hlo,"$IN#lo");	# H.lo·Xi.lo
 $code.=<<___;
 	veor		$IN#lo,$IN#lo,$IN#hi	@ Karatsuba pre-processing
 ___
 	&clmul64x64	($Xm,$Hhl,"$IN#lo");	# (H.lo+H.hi)·(Xi.lo+Xi.hi)
 	&clmul64x64	($Xh,$Hhi,"$IN#hi");	# H.hi·Xi.hi
 $code.=<<___;
 	veor		$Xm,$Xm,$Xl		@ Karatsuba post-processing
 	veor		$Xm,$Xm,$Xh
 	veor		$Xl#hi,$Xl#hi,$Xm#lo
 	veor		$Xh#lo,$Xh#lo,$Xm#hi	@ Xh|Xl - 256-bit result

 	@ equivalent of reduction_avx from ghash-x86_64.pl
 	vshl.i64	$t1,$Xl,#57		@ 1st phase
 	vshl.i64	$t2,$Xl,#62
 	veor		$t2,$t2,$t1		@
 	vshl.i64	$t1,$Xl,#63
 	veor		$t2, $t2, $t1		@
  	veor		$Xl#hi,$Xl#hi,$t2#lo	@
 	veor		$Xh#lo,$Xh#lo,$t2#hi

 	vshr.u64	$t2,$Xl,#1		@ 2nd phase
 	veor		$Xh,$Xh,$Xl
 	veor		$Xl,$Xl,$t2		@
 	vshr.u64	$t2,$t2,#6
 	vshr.u64	$Xl,$Xl,#1		@
 	veor		$Xl,$Xl,$Xh		@
 	veor		$Xl,$Xl,$t2		@

 	subs		$len,#16
 	bne		.Loop_neon

 #ifdef __ARMEL__
 	vrev64.8	$Xl,$Xl
 #endif
 	sub		$Xi,#16
 	vst1.64		$Xl#hi,[$Xi]!		@ write out Xi
 	vst1.64		$Xl#lo,[$Xi]

 	ret					@ bx lr
 .size	gcm_ghash_neon,.-gcm_ghash_neon
 #endif
 ___
 }
 $code.=<<___;
 .asciz  "GHASH for ARMv4/NEON, CRYPTOGAMS by <appro\@openssl.org>"
 .align  2
 ___

 foreach (split("\n",$code)) {
 	s/\`([^\`]*)\`/eval $1/geo;

 	s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo	or
 	s/\bret\b/bx	lr/go		or
 	s/\bbx\s+lr\b/.word\t0xe12fff1e/go;    # make it possible to compile with -march=armv4

 	print $_,"\n";
 }
 close STDOUT or die "error closing STDOUT"; # enforce flush
	#! /usr/bin/env perl
	# Copyright 2010-2018 The OpenSSL Project Authors. All Rights Reserved.
	#
	# Licensed under the OpenSSL license (the "License"). You may not use
	# this file except in compliance with the License. You can obtain a copy
	# in the file LICENSE in the source distribution or at
	# https://www.openssl.org/source/license.html

	#
	# ====================================================================
	# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
	# project. The module is, however, dual licensed under OpenSSL and
	# CRYPTOGAMS licenses depending on where you obtain it. For further
	# details see http://www.openssl.org/~appro/cryptogams/.
	# ====================================================================
	#
	# April 2010
	#
	# The module implements "4-bit" GCM GHASH function and underlying
	# single multiplication operation in GF(2^128). "4-bit" means that it
	# uses 256 bytes per-key table [+32 bytes shared table]. There is no
	# experimental performance data available yet. The only approximation
	# that can be made at this point is based on code size. Inner loop is
	# 32 instructions long and on single-issue core should execute in <40
	# cycles. Having verified that gcc 3.4 didn't unroll corresponding
	# loop, this assembler loop body was found to be ~3x smaller than
	# compiler-generated one...
	#
	# July 2010
	#
	# Rescheduling for dual-issue pipeline resulted in 8.5% improvement on
	# Cortex A8 core and ~25 cycles per processed byte (which was observed
	# to be ~3 times faster than gcc-generated code:-)
	#
	# February 2011
	#
	# Profiler-assisted and platform-specific optimization resulted in 7%
	# improvement on Cortex A8 core and ~23.5 cycles per byte.
	#
	# March 2011
	#
	# Add NEON implementation featuring polynomial multiplication, i.e. no
	# lookup tables involved. On Cortex A8 it was measured to process one
	# byte in 15 cycles or 55% faster than integer-only code.
	#
	# April 2014
	#
	# Switch to multiplication algorithm suggested in paper referred
	# below and combine it with reduction algorithm from x86 module.
	# Performance improvement over previous version varies from 65% on
	# Snapdragon S4 to 110% on Cortex A9. In absolute terms Cortex A8
	# processes one byte in 8.45 cycles, A9 - in 10.2, A15 - in 7.63,
	# Snapdragon S4 - in 9.33.
	#
	# Câmara, D.; Gouvêa, C. P. L.; López, J. & Dahab, R.: Fast Software
	# Polynomial Multiplication on ARM Processors using the NEON Engine.
	#
	# http://conradoplg.cryptoland.net/files/2010/12/mocrysen13.pdf

	# ====================================================================
	# Note about "528B" variant. In ARM case it makes lesser sense to
	# implement it for following reasons:
	#
	# - performance improvement won't be anywhere near 50%, because 128-
	# bit shift operation is neatly fused with 128-bit xor here, and
	# "538B" variant would eliminate only 4-5 instructions out of 32
	# in the inner loop (meaning that estimated improvement is ~15%);
	# - ARM-based systems are often embedded ones and extra memory
	# consumption might be unappreciated (for so little improvement);
	#
	# Byte order [in]dependence. =========================================
	#
	# Caller is expected to maintain specific dword order in Htable,
	# namely with least significant dword of 128-bit value at lower
	# address. This differs completely from C code and has everything to
	# do with ldm instruction and order in which dwords are "consumed" by
	# algorithm. Byte order within these dwords in turn is whatever
	# native byte order on current platform. See gcm128.c for working
	# example...

	# This file was patched in BoringSSL to remove the variable-time 4-bit
	# implementation.

	$flavour = shift;
	if ($flavour=~/\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; }
	else { while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} }

	if ($flavour && $flavour ne "void") {
	$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
	( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
	( $xlate="${dir}../../../perlasm/arm-xlate.pl" and -f $xlate) or
	die "can't locate arm-xlate.pl";

	open OUT,"\| \"$^X\" $xlate $flavour $output";
	STDOUT=OUT;
	} else {
	open OUT,">$output";
	STDOUT=OUT;
	}

	$Xi="r0"; # argument block
	$Htbl="r1";
	$inp="r2";
	$len="r3";

	$code=<<___;
	#include <openssl/arm_arch.h>

	@ Silence ARMv8 deprecated IT instruction warnings. This file is used by both
	@ ARMv7 and ARMv8 processors and does not use ARMv8 instructions. (ARMv8 PMULL
	@ instructions are in aesv8-armx.pl.)
	.arch armv7-a

	.text
	#if defined(__thumb2__) \|\| defined(__clang__)
	.syntax unified
	#define ldrplb ldrbpl
	#define ldrneb ldrbne
	#endif
	#if defined(__thumb2__)
	.thumb
	#else
	.code 32
	#endif
	___
	{
	my ($Xl,$Xm,$Xh,$IN)=map("q$_",(0..3));
	my ($t0,$t1,$t2,$t3)=map("q$_",(8..12));
	my ($Hlo,$Hhi,$Hhl,$k48,$k32,$k16)=map("d$_",(26..31));

	sub clmul64x64 {
	my ($r,$a,$b)=@_;
	$code.=<<___;
	vext.8 $t0#lo, $a, $a, #1 @ A1
	vmull.p8 $t0, $t0#lo, $b @ F = A1*B
	vext.8 $r#lo, $b, $b, #1 @ B1
	vmull.p8 $r, $a, $r#lo @ E = A*B1
	vext.8 $t1#lo, $a, $a, #2 @ A2
	vmull.p8 $t1, $t1#lo, $b @ H = A2*B
	vext.8 $t3#lo, $b, $b, #2 @ B2
	vmull.p8 $t3, $a, $t3#lo @ G = A*B2
	vext.8 $t2#lo, $a, $a, #3 @ A3
	veor $t0, $t0, $r @ L = E + F
	vmull.p8 $t2, $t2#lo, $b @ J = A3*B
	vext.8 $r#lo, $b, $b, #3 @ B3
	veor $t1, $t1, $t3 @ M = G + H
	vmull.p8 $r, $a, $r#lo @ I = A*B3
	veor $t0#lo, $t0#lo, $t0#hi @ t0 = (L) (P0 + P1) << 8
	vand $t0#hi, $t0#hi, $k48
	vext.8 $t3#lo, $b, $b, #4 @ B4
	veor $t1#lo, $t1#lo, $t1#hi @ t1 = (M) (P2 + P3) << 16
	vand $t1#hi, $t1#hi, $k32
	vmull.p8 $t3, $a, $t3#lo @ K = A*B4
	veor $t2, $t2, $r @ N = I + J
	veor $t0#lo, $t0#lo, $t0#hi
	veor $t1#lo, $t1#lo, $t1#hi
	veor $t2#lo, $t2#lo, $t2#hi @ t2 = (N) (P4 + P5) << 24
	vand $t2#hi, $t2#hi, $k16
	vext.8 $t0, $t0, $t0, #15
	veor $t3#lo, $t3#lo, $t3#hi @ t3 = (K) (P6 + P7) << 32
	vmov.i64 $t3#hi, #0
	vext.8 $t1, $t1, $t1, #14
	veor $t2#lo, $t2#lo, $t2#hi
	vmull.p8 $r, $a, $b @ D = A*B
	vext.8 $t3, $t3, $t3, #12
	vext.8 $t2, $t2, $t2, #13
	veor $t0, $t0, $t1
	veor $t2, $t2, $t3
	veor $r, $r, $t0
	veor $r, $r, $t2
	___
	}

	$code.=<<___;
	#if __ARM_MAX_ARCH__>=7
	.arch armv7-a
	.fpu neon

	.global gcm_init_neon
	.type gcm_init_neon,%function
	.align 4
	gcm_init_neon:
	vld1.64 $IN#hi,[r1]! @ load H
	vmov.i8 $t0,#0xe1
	vld1.64 $IN#lo,[r1]
	vshl.i64 $t0#hi,#57
	vshr.u64 $t0#lo,#63 @ t0=0xc2....01
	vdup.8 $t1,$IN#hi[7]
	vshr.u64 $Hlo,$IN#lo,#63
	vshr.s8 $t1,#7 @ broadcast carry bit
	vshl.i64 $IN,$IN,#1
	vand $t0,$t0,$t1
	vorr $IN#hi,$Hlo @ H<<<=1
	veor $IN,$IN,$t0 @ twisted H
	vstmia r0,{$IN}

	ret @ bx lr
	.size gcm_init_neon,.-gcm_init_neon

	.global gcm_gmult_neon
	.type gcm_gmult_neon,%function
	.align 4
	gcm_gmult_neon:
	vld1.64 $IN#hi,[$Xi]! @ load Xi
	vld1.64 $IN#lo,[$Xi]!
	vmov.i64 $k48,#0x0000ffffffffffff
	vldmia $Htbl,{$Hlo-$Hhi} @ load twisted H
	vmov.i64 $k32,#0x00000000ffffffff
	#ifdef __ARMEL__
	vrev64.8 $IN,$IN
	#endif
	vmov.i64 $k16,#0x000000000000ffff
	veor $Hhl,$Hlo,$Hhi @ Karatsuba pre-processing
	mov $len,#16
	b .Lgmult_neon
	.size gcm_gmult_neon,.-gcm_gmult_neon

	.global gcm_ghash_neon
	.type gcm_ghash_neon,%function
	.align 4
	gcm_ghash_neon:
	vld1.64 $Xl#hi,[$Xi]! @ load Xi
	vld1.64 $Xl#lo,[$Xi]!
	vmov.i64 $k48,#0x0000ffffffffffff
	vldmia $Htbl,{$Hlo-$Hhi} @ load twisted H
	vmov.i64 $k32,#0x00000000ffffffff
	#ifdef __ARMEL__
	vrev64.8 $Xl,$Xl
	#endif
	vmov.i64 $k16,#0x000000000000ffff
	veor $Hhl,$Hlo,$Hhi @ Karatsuba pre-processing

	.Loop_neon:
	vld1.64 $IN#hi,[$inp]! @ load inp
	vld1.64 $IN#lo,[$inp]!
	#ifdef __ARMEL__
	vrev64.8 $IN,$IN
	#endif
	veor $IN,$Xl @ inp^=Xi
	.Lgmult_neon:
	___
	&clmul64x64 ($Xl,$Hlo,"$IN#lo"); # H.lo·Xi.lo
	$code.=<<___;
	veor $IN#lo,$IN#lo,$IN#hi @ Karatsuba pre-processing
	___
	&clmul64x64 ($Xm,$Hhl,"$IN#lo"); # (H.lo+H.hi)·(Xi.lo+Xi.hi)
	&clmul64x64 ($Xh,$Hhi,"$IN#hi"); # H.hi·Xi.hi
	$code.=<<___;
	veor $Xm,$Xm,$Xl @ Karatsuba post-processing
	veor $Xm,$Xm,$Xh
	veor $Xl#hi,$Xl#hi,$Xm#lo
	veor $Xh#lo,$Xh#lo,$Xm#hi @ Xh\|Xl - 256-bit result

	@ equivalent of reduction_avx from ghash-x86_64.pl
	vshl.i64 $t1,$Xl,#57 @ 1st phase
	vshl.i64 $t2,$Xl,#62
	veor $t2,$t2,$t1 @
	vshl.i64 $t1,$Xl,#63
	veor $t2, $t2, $t1 @
	veor $Xl#hi,$Xl#hi,$t2#lo @
	veor $Xh#lo,$Xh#lo,$t2#hi

	vshr.u64 $t2,$Xl,#1 @ 2nd phase
	veor $Xh,$Xh,$Xl
	veor $Xl,$Xl,$t2 @
	vshr.u64 $t2,$t2,#6
	vshr.u64 $Xl,$Xl,#1 @
	veor $Xl,$Xl,$Xh @
	veor $Xl,$Xl,$t2 @

	subs $len,#16
	bne .Loop_neon

	#ifdef __ARMEL__
	vrev64.8 $Xl,$Xl
	#endif
	sub $Xi,#16
	vst1.64 $Xl#hi,[$Xi]! @ write out Xi
	vst1.64 $Xl#lo,[$Xi]

	ret @ bx lr
	.size gcm_ghash_neon,.-gcm_ghash_neon
	#endif
	___
	}
	$code.=<<___;
	.asciz "GHASH for ARMv4/NEON, CRYPTOGAMS by <appro\@openssl.org>"
	.align 2
	___

	foreach (split("\n",$code)) {
	s/\`([^\`]*)\`/eval $1/geo;

	s/\bq([0-9]+)#(lo\|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo or
	s/\bret\b/bx lr/go or
	s/\bbx\s+lr\b/.word\t0xe12fff1e/go; # make it possible to compile with -march=armv4

	print $_,"\n";
	}
	close STDOUT or die "error closing STDOUT"; # enforce flush