WireGuard Archive on lore.kernel.org
 help / color / Atom feed
* [PATCH] [Zinc] Add PowerPC chacha20 implementation from openssl/cryptograms
@ 2019-05-11 13:10 Shawn Landden
  2019-05-11 18:03 ` [PATCH 1/2] " Shawn Landden
  0 siblings, 1 reply; 5+ messages in thread
From: Shawn Landden @ 2019-05-11 13:10 UTC (permalink / raw)
  To: wireguard

There is a bug where may_use_simd()
returns false in kworker, and prevents most of this code from running.

Otherwise this is tested (with the VSX code enabled) on Power 9 (ppc64le).
It is quite a bit faster (look at the benchmarks in the assembly file)
but it is difficult to give numbers because the interface line
speed is the limiting factor.

Signed-off-by: Shawn Landden <shawn@git.icu>
---
 src/compat/simd/include/linux/simd.h         |   18 +-
 src/crypto/Kbuild.include                    |   26 +-
 src/crypto/zinc/chacha20/chacha20-ppc-glue.c |   67 +
 src/crypto/zinc/chacha20/chacha20-ppc.pl     | 1355 ++++++++++++++++++
 src/crypto/zinc/chacha20/chacha20.c          |    2 +
 src/crypto/zinc/chacha20/ppc-xlate.pl        |  353 +++++
 6 files changed, 1818 insertions(+), 3 deletions(-)
 create mode 100644 src/crypto/zinc/chacha20/chacha20-ppc-glue.c
 create mode 100644 src/crypto/zinc/chacha20/chacha20-ppc.pl
 create mode 100644 src/crypto/zinc/chacha20/ppc-xlate.pl

diff --git a/src/compat/simd/include/linux/simd.h b/src/compat/simd/include/linux/simd.h
index c75c724..23af478 100644
--- a/src/compat/simd/include/linux/simd.h
+++ b/src/compat/simd/include/linux/simd.h
@@ -11,10 +11,12 @@
 #if defined(CONFIG_X86_64)
 #include <linux/version.h>
 #include <asm/fpu/api.h>
 #elif defined(CONFIG_KERNEL_MODE_NEON)
 #include <asm/neon.h>
+#elif defined(CONFIG_ALTIVEC) || defined(CONFIG_VSX)
+#include <asm/switch_to.h>
 #endif
 
 typedef enum {
 	HAVE_NO_SIMD = 1 << 0,
 	HAVE_FULL_SIMD = 1 << 1,
@@ -28,17 +30,23 @@ static inline void simd_get(simd_context_t *ctx)
 	*ctx = !IS_ENABLED(CONFIG_PREEMPT_RT_BASE) && may_use_simd() ? HAVE_FULL_SIMD : HAVE_NO_SIMD;
 }
 
 static inline void simd_put(simd_context_t *ctx)
 {
+        if (*ctx & HAVE_SIMD_IN_USE) {
 #if defined(CONFIG_X86_64)
-	if (*ctx & HAVE_SIMD_IN_USE)
 		kernel_fpu_end();
 #elif defined(CONFIG_KERNEL_MODE_NEON)
-	if (*ctx & HAVE_SIMD_IN_USE)
 		kernel_neon_end();
+#elif defined(CONFIG_VSX)
+		disable_kernel_vsx();
+		preempt_enable();
+#elif defined(CONFIG_ALTIVEC)
+		disable_kernel_altivec();
+		preempt_enable();
 #endif
+	}
 	*ctx = HAVE_NO_SIMD;
 }
 
 static inline bool simd_relax(simd_context_t *ctx)
 {
@@ -60,10 +68,16 @@ static __must_check inline bool simd_use(simd_context_t *ctx)
 		return true;
 #if defined(CONFIG_X86_64)
 	kernel_fpu_begin();
 #elif defined(CONFIG_KERNEL_MODE_NEON)
 	kernel_neon_begin();
+#elif defined(CONFIG_VSX)
+	preempt_disable();
+	enable_kernel_vsx();
+#elif defined(CONFIG_ALTIVEC)
+	preempt_disable();
+	enable_kernel_altivec();
 #endif
 	*ctx |= HAVE_SIMD_IN_USE;
 	return true;
 }
 
diff --git a/src/crypto/Kbuild.include b/src/crypto/Kbuild.include
index 460684d..64894a5 100644
--- a/src/crypto/Kbuild.include
+++ b/src/crypto/Kbuild.include
@@ -11,17 +11,25 @@ ifeq ($(CONFIG_MIPS)$(CONFIG_CPU_MIPS32_R2),yy)
 CONFIG_ZINC_ARCH_MIPS := y
 endif
 ifeq ($(CONFIG_MIPS)$(CONFIG_64BIT),yy)
 CONFIG_ZINC_ARCH_MIPS64 := y
 endif
+ifeq ($(CONFIG_PPC32),y)
+CONFIG_ZINC_ARCH_PPC32 := y
+endif
+ifeq ($(CONFIG_PPC64),y)
+CONFIG_ZINC_ARCH_PPC64 := y
+endif
 
 zinc-y += chacha20/chacha20.o
 zinc-$(CONFIG_ZINC_ARCH_X86_64) += chacha20/chacha20-x86_64.o
 zinc-$(CONFIG_ZINC_ARCH_ARM) += chacha20/chacha20-arm.o chacha20/chacha20-unrolled-arm.o
 zinc-$(CONFIG_ZINC_ARCH_ARM64) += chacha20/chacha20-arm64.o
 zinc-$(CONFIG_ZINC_ARCH_MIPS) += chacha20/chacha20-mips.o
 AFLAGS_chacha20-mips.o += -O2 # This is required to fill the branch delay slots
+zinc-$(CONFIG_ZINC_ARCH_PPC32) += chacha20/chacha20-ppc.o
+zinc-$(CONFIG_ZINC_ARCH_PPC64) += chacha20/chacha20-ppc.o
 
 zinc-y += poly1305/poly1305.o
 zinc-$(CONFIG_ZINC_ARCH_X86_64) += poly1305/poly1305-x86_64.o
 zinc-$(CONFIG_ZINC_ARCH_ARM) += poly1305/poly1305-arm.o
 zinc-$(CONFIG_ZINC_ARCH_ARM64) += poly1305/poly1305-arm64.o
@@ -36,22 +44,38 @@ zinc-$(CONFIG_ZINC_ARCH_X86_64) += blake2s/blake2s-x86_64.o
 
 zinc-y += curve25519/curve25519.o
 zinc-$(CONFIG_ZINC_ARCH_ARM) += curve25519/curve25519-arm.o
 
 quiet_cmd_perlasm = PERLASM $@
-      cmd_perlasm = $(PERL) $< > $@
+      cmd_perlasm = $(PERL) $< $(perlflags-y) > $@
 $(obj)/%.S: $(src)/%.pl FORCE
 	$(call if_changed,perlasm)
 kbuild-dir := $(if $(filter /%,$(src)),$(src),$(srctree)/$(src))
 targets := $(patsubst $(kbuild-dir)/%.pl,%.S,$(wildcard $(patsubst %.o,$(kbuild-dir)/crypto/zinc/%.pl,$(zinc-y) $(zinc-m) $(zinc-))))
 
+perlflags-$(CONFIG_ZINC_ARCH_PPC32) += linux32
+ifeq ($(CONFIG_ZINC_ARCH_PPC64),y)
+perlflags-$(CONFIG_CPU_BIG_ENDIAN) += linux64
+perlflags-$(CONFIG_CPU_LITTLE_ENDIAN) += linux64le
+endif
+
 # Old kernels don't set this, which causes trouble.
 .SECONDARY:
 
 wireguard-y += $(addprefix crypto/zinc/,$(zinc-y))
 ccflags-y += -I$(src)/crypto/include
 ccflags-$(CONFIG_ZINC_ARCH_X86_64) += -DCONFIG_ZINC_ARCH_X86_64
 ccflags-$(CONFIG_ZINC_ARCH_ARM) += -DCONFIG_ZINC_ARCH_ARM
 ccflags-$(CONFIG_ZINC_ARCH_ARM64) += -DCONFIG_ZINC_ARCH_ARM64
 ccflags-$(CONFIG_ZINC_ARCH_MIPS) += -DCONFIG_ZINC_ARCH_MIPS
 ccflags-$(CONFIG_ZINC_ARCH_MIPS64) += -DCONFIG_ZINC_ARCH_MIPS64
+ccflags-$(CONFIG_ZINC_ARCH_PPC32) += -DCONFIG_ZINC_ARCH_PPC32
+ccflags-$(CONFIG_ZINC_ARCH_PPC64) += -DCONFIG_ZINC_ARCH_PPC64
+ifdef ($(CONFIG_VSX),y)
+ccflags-$(CONFIG_ZINC_ARCH_PPC32) += -DCONFIG_VSX
+ccflags-$(CONFIG_ZINC_ARCH_PPC64) += -DCONFIG_VSX
+endif
+ifdef ($(CONFIG_ALTIVEC),y)
+ccflags-$(CONFIG_ZINC_ARCH_PPC32) += -DCONFIG_ALTIVEC
+ccflags-$(CONFIG_ZINC_ARCH_PPC64) += -DCONFIG_ALTIVEC
+endif
 ccflags-$(CONFIG_WIREGUARD_DEBUG) += -DCONFIG_ZINC_SELFTEST
diff --git a/src/crypto/zinc/chacha20/chacha20-ppc-glue.c b/src/crypto/zinc/chacha20/chacha20-ppc-glue.c
new file mode 100644
index 0000000..d13098b
--- /dev/null
+++ b/src/crypto/zinc/chacha20/chacha20-ppc-glue.c
@@ -0,0 +1,67 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (C) 2019 Shawn Landden <shawn@git.icu>. All Rights Reserved.
+ */
+
+asmlinkage void ChaCha20_ctr32_int(u8 *out, const u8 *inp,
+                        size_t len, const u32 key[8],
+                        const u32 counter[4]);
+asmlinkage void ChaCha20_ctr32_vmx(u8 *out, const u8 *inp,
+                        size_t len, const u32 key[8],
+                        const u32 counter[4]);
+asmlinkage void ChaCha20_ctr32_vsx(u8 *out, const u8 *inp,
+                        size_t len, const u32 key[8],
+                        const u32 counter[4]);
+static bool *const chacha20_nobs[] __initconst = { };
+static void __init chacha20_fpu_init(void)
+{
+}
+
+static inline bool chacha20_arch(struct chacha20_ctx *ctx, u8 *dst,
+				 const u8 *src, size_t len,
+				 simd_context_t *simd_context)
+{
+	void (*ChaCha20SIMD)(u8 *out, const u8 *inp,
+                        size_t len, const u32 key[8],
+                        const u32 counter[4]);
+
+	/* SIMD disables preemption, so relax after processing each page. */
+	BUILD_BUG_ON(PAGE_SIZE < CHACHA20_BLOCK_SIZE ||
+		     PAGE_SIZE % CHACHA20_BLOCK_SIZE);
+
+	if (cpu_has_feature(CPU_FTR_VSX_COMP))
+		ChaCha20SIMD = &ChaCha20_ctr32_vsx;
+	else if (cpu_has_feature(CPU_FTR_ALTIVEC))
+                ChaCha20SIMD = &ChaCha20_ctr32_vmx;
+	else {
+		ChaCha20_ctr32_int(dst, src, len, ctx->key, ctx->counter);
+		return true;
+	}
+	for (;;) {
+		if (len >= CHACHA20_BLOCK_SIZE * 3 && simd_use(simd_context)) {
+			const size_t bytes = min_t(size_t, len, PAGE_SIZE);
+
+			ChaCha20SIMD(dst, src, bytes, ctx->key, ctx->counter);
+			ctx->counter[0] += (bytes + 63) / 64;
+			len -= bytes;
+			if (!len)
+				break;
+			dst += bytes;
+			src += bytes;
+			simd_relax(simd_context);
+		} else {
+			ChaCha20_ctr32_int(dst, src, len, ctx->key, ctx->counter);
+                        ctx->counter[0] += (len + 63) / 64;
+			return true;
+		}
+	}
+	return true;
+}
+
+static inline bool hchacha20_arch(u32 derived_key[CHACHA20_KEY_WORDS],
+				  const u8 nonce[HCHACHA20_NONCE_SIZE],
+				  const u8 key[HCHACHA20_KEY_SIZE],
+				  simd_context_t *simd_context)
+{
+	return false;
+}
diff --git a/src/crypto/zinc/chacha20/chacha20-ppc.pl b/src/crypto/zinc/chacha20/chacha20-ppc.pl
new file mode 100644
index 0000000..07468c8
--- /dev/null
+++ b/src/crypto/zinc/chacha20/chacha20-ppc.pl
@@ -0,0 +1,1355 @@
+#! /usr/bin/env perl
+# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+#
+# This code is taken from the OpenSSL project but the author, Andy Polyakov,
+# has relicensed it under the licenses specified in the SPDX header above.
+# The original headers, including the original license headers, are
+# included below for completeness.
+#
+# Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the Apache License 2.0 (the "License").  You may not use
+# this file except in compliance with the License.  You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
+#
+# ====================================================================
+# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
+# project. The module is, however, dual licensed under OpenSSL and
+# CRYPTOGAMS licenses depending on where you obtain it. For further
+# details see http://www.openssl.org/~appro/cryptogams/.
+# ====================================================================
+#
+# October 2015
+#
+# ChaCha20 for PowerPC/AltiVec.
+#
+# June 2018
+#
+# Add VSX 2.07 code path. Original 3xAltiVec+1xIALU is well-suited for
+# processors that can't issue more than one vector instruction per
+# cycle. But POWER8 (and POWER9) can issue a pair, and vector-only 4x
+# interleave would perform better. Incidentally PowerISA 2.07 (first
+# implemented by POWER8) defined new usable instructions, hence 4xVSX
+# code path...
+#
+# Performance in cycles per byte out of large buffer.
+#
+#			IALU/gcc-4.x    3xAltiVec+1xIALU	4xVSX
+#
+# Freescale e300	13.6/+115%	-			-
+# PPC74x0/G4e		6.81/+310%	3.81			-
+# PPC970/G5		9.29/+160%	?			-
+# POWER7		8.62/+61%	3.35			-
+# POWER8		8.70/+51%	2.91			2.09
+# POWER9		8.80/+29%	4.44(*)			2.45(**)
+#
+# (*)	this is trade-off result, it's possible to improve it, but
+#	then it would negatively affect all others;
+# (**)	POWER9 seems to be "allergic" to mixing vector and integer
+#	instructions, which is why switch to vector-only code pays
+#	off that much;
+
+$flavour = shift;
+
+if ($flavour =~ /64/) {
+	$SIZE_T	=8;
+	$LRSAVE	=2*$SIZE_T;
+	$STU	="stdu";
+	$POP	="ld";
+	$PUSH	="std";
+	$UCMP	="cmpld";
+} elsif ($flavour =~ /32/) {
+	$SIZE_T	=4;
+	$LRSAVE	=$SIZE_T;
+	$STU	="stwu";
+	$POP	="lwz";
+	$PUSH	="stw";
+	$UCMP	="cmplw";
+} else { die "nonsense $flavour"; }
+
+$LITTLE_ENDIAN = ($flavour=~/le$/) ? 1 : 0;
+
+$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
+( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
+( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
+die "can't locate ppc-xlate.pl";
+
+open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";
+
+$LOCALS=6*$SIZE_T;
+$FRAME=$LOCALS+64+18*$SIZE_T;	# 64 is for local variables
+
+sub AUTOLOAD()		# thunk [simplified] x86-style perlasm
+{ my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./;
+    $code .= "\t$opcode\t".join(',',@_)."\n";
+}
+
+my $sp = "r1";
+
+my ($out,$inp,$len,$key,$ctr) = map("r$_",(3..7));
+
+my @x=map("r$_",(16..31));
+my @d=map("r$_",(11,12,14,15));
+my @t=map("r$_",(7..10));
+
+sub ROUND {
+my ($a0,$b0,$c0,$d0)=@_;
+my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
+my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
+my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
+
+    (
+	"&add		(@x[$a0],@x[$a0],@x[$b0])",
+	 "&add		(@x[$a1],@x[$a1],@x[$b1])",
+	  "&add		(@x[$a2],@x[$a2],@x[$b2])",
+	   "&add	(@x[$a3],@x[$a3],@x[$b3])",
+	"&xor		(@x[$d0],@x[$d0],@x[$a0])",
+	 "&xor		(@x[$d1],@x[$d1],@x[$a1])",
+	  "&xor		(@x[$d2],@x[$d2],@x[$a2])",
+	   "&xor	(@x[$d3],@x[$d3],@x[$a3])",
+	"&rotlwi	(@x[$d0],@x[$d0],16)",
+	 "&rotlwi	(@x[$d1],@x[$d1],16)",
+	  "&rotlwi	(@x[$d2],@x[$d2],16)",
+	   "&rotlwi	(@x[$d3],@x[$d3],16)",
+
+	"&add		(@x[$c0],@x[$c0],@x[$d0])",
+	 "&add		(@x[$c1],@x[$c1],@x[$d1])",
+	  "&add		(@x[$c2],@x[$c2],@x[$d2])",
+	   "&add	(@x[$c3],@x[$c3],@x[$d3])",
+	"&xor		(@x[$b0],@x[$b0],@x[$c0])",
+	 "&xor		(@x[$b1],@x[$b1],@x[$c1])",
+	  "&xor		(@x[$b2],@x[$b2],@x[$c2])",
+	   "&xor	(@x[$b3],@x[$b3],@x[$c3])",
+	"&rotlwi	(@x[$b0],@x[$b0],12)",
+	 "&rotlwi	(@x[$b1],@x[$b1],12)",
+	  "&rotlwi	(@x[$b2],@x[$b2],12)",
+	   "&rotlwi	(@x[$b3],@x[$b3],12)",
+
+	"&add		(@x[$a0],@x[$a0],@x[$b0])",
+	 "&add		(@x[$a1],@x[$a1],@x[$b1])",
+	  "&add		(@x[$a2],@x[$a2],@x[$b2])",
+	   "&add	(@x[$a3],@x[$a3],@x[$b3])",
+	"&xor		(@x[$d0],@x[$d0],@x[$a0])",
+	 "&xor		(@x[$d1],@x[$d1],@x[$a1])",
+	  "&xor		(@x[$d2],@x[$d2],@x[$a2])",
+	   "&xor	(@x[$d3],@x[$d3],@x[$a3])",
+	"&rotlwi	(@x[$d0],@x[$d0],8)",
+	 "&rotlwi	(@x[$d1],@x[$d1],8)",
+	  "&rotlwi	(@x[$d2],@x[$d2],8)",
+	   "&rotlwi	(@x[$d3],@x[$d3],8)",
+
+	"&add		(@x[$c0],@x[$c0],@x[$d0])",
+	 "&add		(@x[$c1],@x[$c1],@x[$d1])",
+	  "&add		(@x[$c2],@x[$c2],@x[$d2])",
+	   "&add	(@x[$c3],@x[$c3],@x[$d3])",
+	"&xor		(@x[$b0],@x[$b0],@x[$c0])",
+	 "&xor		(@x[$b1],@x[$b1],@x[$c1])",
+	  "&xor		(@x[$b2],@x[$b2],@x[$c2])",
+	   "&xor	(@x[$b3],@x[$b3],@x[$c3])",
+	"&rotlwi	(@x[$b0],@x[$b0],7)",
+	 "&rotlwi	(@x[$b1],@x[$b1],7)",
+	  "&rotlwi	(@x[$b2],@x[$b2],7)",
+	   "&rotlwi	(@x[$b3],@x[$b3],7)"
+    );
+}
+
+$code.=<<___;
+.machine	"any"
+.text
+
+.globl	.ChaCha20_ctr32_int
+.align	5
+.ChaCha20_ctr32_int:
+__ChaCha20_ctr32_int:
+	${UCMP}i $len,0
+	beqlr-
+
+	$STU	$sp,-$FRAME($sp)
+	mflr	r0
+
+	$PUSH	r14,`$FRAME-$SIZE_T*18`($sp)
+	$PUSH	r15,`$FRAME-$SIZE_T*17`($sp)
+	$PUSH	r16,`$FRAME-$SIZE_T*16`($sp)
+	$PUSH	r17,`$FRAME-$SIZE_T*15`($sp)
+	$PUSH	r18,`$FRAME-$SIZE_T*14`($sp)
+	$PUSH	r19,`$FRAME-$SIZE_T*13`($sp)
+	$PUSH	r20,`$FRAME-$SIZE_T*12`($sp)
+	$PUSH	r21,`$FRAME-$SIZE_T*11`($sp)
+	$PUSH	r22,`$FRAME-$SIZE_T*10`($sp)
+	$PUSH	r23,`$FRAME-$SIZE_T*9`($sp)
+	$PUSH	r24,`$FRAME-$SIZE_T*8`($sp)
+	$PUSH	r25,`$FRAME-$SIZE_T*7`($sp)
+	$PUSH	r26,`$FRAME-$SIZE_T*6`($sp)
+	$PUSH	r27,`$FRAME-$SIZE_T*5`($sp)
+	$PUSH	r28,`$FRAME-$SIZE_T*4`($sp)
+	$PUSH	r29,`$FRAME-$SIZE_T*3`($sp)
+	$PUSH	r30,`$FRAME-$SIZE_T*2`($sp)
+	$PUSH	r31,`$FRAME-$SIZE_T*1`($sp)
+	$PUSH	r0,`$FRAME+$LRSAVE`($sp)
+
+	lwz	@d[0],0($ctr)			# load counter
+	lwz	@d[1],4($ctr)
+	lwz	@d[2],8($ctr)
+	lwz	@d[3],12($ctr)
+
+	bl	__ChaCha20_1x
+
+	$POP	r0,`$FRAME+$LRSAVE`($sp)
+	$POP	r14,`$FRAME-$SIZE_T*18`($sp)
+	$POP	r15,`$FRAME-$SIZE_T*17`($sp)
+	$POP	r16,`$FRAME-$SIZE_T*16`($sp)
+	$POP	r17,`$FRAME-$SIZE_T*15`($sp)
+	$POP	r18,`$FRAME-$SIZE_T*14`($sp)
+	$POP	r19,`$FRAME-$SIZE_T*13`($sp)
+	$POP	r20,`$FRAME-$SIZE_T*12`($sp)
+	$POP	r21,`$FRAME-$SIZE_T*11`($sp)
+	$POP	r22,`$FRAME-$SIZE_T*10`($sp)
+	$POP	r23,`$FRAME-$SIZE_T*9`($sp)
+	$POP	r24,`$FRAME-$SIZE_T*8`($sp)
+	$POP	r25,`$FRAME-$SIZE_T*7`($sp)
+	$POP	r26,`$FRAME-$SIZE_T*6`($sp)
+	$POP	r27,`$FRAME-$SIZE_T*5`($sp)
+	$POP	r28,`$FRAME-$SIZE_T*4`($sp)
+	$POP	r29,`$FRAME-$SIZE_T*3`($sp)
+	$POP	r30,`$FRAME-$SIZE_T*2`($sp)
+	$POP	r31,`$FRAME-$SIZE_T*1`($sp)
+	mtlr	r0
+	addi	$sp,$sp,$FRAME
+	blr
+	.long	0
+	.byte	0,12,4,1,0x80,18,5,0
+	.long	0
+.size	.ChaCha20_ctr32_int,.-.ChaCha20_ctr32_int
+
+.align	5
+__ChaCha20_1x:
+Loop_outer:
+	lis	@x[0],0x6170			# synthesize sigma
+	lis	@x[1],0x3320
+	lis	@x[2],0x7962
+	lis	@x[3],0x6b20
+	ori	@x[0],@x[0],0x7865
+	ori	@x[1],@x[1],0x646e
+	ori	@x[2],@x[2],0x2d32
+	ori	@x[3],@x[3],0x6574
+
+	li	r0,10				# inner loop counter
+	lwz	@x[4],0($key)			# load key
+	lwz	@x[5],4($key)
+	lwz	@x[6],8($key)
+	lwz	@x[7],12($key)
+	lwz	@x[8],16($key)
+	mr	@x[12],@d[0]			# copy counter
+	lwz	@x[9],20($key)
+	mr	@x[13],@d[1]
+	lwz	@x[10],24($key)
+	mr	@x[14],@d[2]
+	lwz	@x[11],28($key)
+	mr	@x[15],@d[3]
+
+	mr	@t[0],@x[4]
+	mr	@t[1],@x[5]
+	mr	@t[2],@x[6]
+	mr	@t[3],@x[7]
+
+	mtctr	r0
+Loop:
+___
+	foreach (&ROUND(0, 4, 8,12)) { eval; }
+	foreach (&ROUND(0, 5,10,15)) { eval; }
+$code.=<<___;
+	bdnz	Loop
+
+	subic	$len,$len,64			# $len-=64
+	addi	@x[0],@x[0],0x7865		# accumulate key block
+	addi	@x[1],@x[1],0x646e
+	addi	@x[2],@x[2],0x2d32
+	addi	@x[3],@x[3],0x6574
+	addis	@x[0],@x[0],0x6170
+	addis	@x[1],@x[1],0x3320
+	addis	@x[2],@x[2],0x7962
+	addis	@x[3],@x[3],0x6b20
+
+	subfe.	r0,r0,r0			# borrow?-1:0
+	add	@x[4],@x[4],@t[0]
+	lwz	@t[0],16($key)
+	add	@x[5],@x[5],@t[1]
+	lwz	@t[1],20($key)
+	add	@x[6],@x[6],@t[2]
+	lwz	@t[2],24($key)
+	add	@x[7],@x[7],@t[3]
+	lwz	@t[3],28($key)
+	add	@x[8],@x[8],@t[0]
+	add	@x[9],@x[9],@t[1]
+	add	@x[10],@x[10],@t[2]
+	add	@x[11],@x[11],@t[3]
+
+	add	@x[12],@x[12],@d[0]
+	add	@x[13],@x[13],@d[1]
+	add	@x[14],@x[14],@d[2]
+	add	@x[15],@x[15],@d[3]
+	addi	@d[0],@d[0],1			# increment counter
+___
+if (!$LITTLE_ENDIAN) { for($i=0;$i<16;$i++) {	# flip byte order
+$code.=<<___;
+	mr	@t[$i&3],@x[$i]
+	rotlwi	@x[$i],@x[$i],8
+	rlwimi	@x[$i],@t[$i&3],24,0,7
+	rlwimi	@x[$i],@t[$i&3],24,16,23
+___
+} }
+$code.=<<___;
+	bne	Ltail				# $len-=64 borrowed
+
+	lwz	@t[0],0($inp)			# load input, aligned or not
+	lwz	@t[1],4($inp)
+	${UCMP}i $len,0				# done already?
+	lwz	@t[2],8($inp)
+	lwz	@t[3],12($inp)
+	xor	@x[0],@x[0],@t[0]		# xor with input
+	lwz	@t[0],16($inp)
+	xor	@x[1],@x[1],@t[1]
+	lwz	@t[1],20($inp)
+	xor	@x[2],@x[2],@t[2]
+	lwz	@t[2],24($inp)
+	xor	@x[3],@x[3],@t[3]
+	lwz	@t[3],28($inp)
+	xor	@x[4],@x[4],@t[0]
+	lwz	@t[0],32($inp)
+	xor	@x[5],@x[5],@t[1]
+	lwz	@t[1],36($inp)
+	xor	@x[6],@x[6],@t[2]
+	lwz	@t[2],40($inp)
+	xor	@x[7],@x[7],@t[3]
+	lwz	@t[3],44($inp)
+	xor	@x[8],@x[8],@t[0]
+	lwz	@t[0],48($inp)
+	xor	@x[9],@x[9],@t[1]
+	lwz	@t[1],52($inp)
+	xor	@x[10],@x[10],@t[2]
+	lwz	@t[2],56($inp)
+	xor	@x[11],@x[11],@t[3]
+	lwz	@t[3],60($inp)
+	xor	@x[12],@x[12],@t[0]
+	stw	@x[0],0($out)			# store output, aligned or not
+	xor	@x[13],@x[13],@t[1]
+	stw	@x[1],4($out)
+	xor	@x[14],@x[14],@t[2]
+	stw	@x[2],8($out)
+	xor	@x[15],@x[15],@t[3]
+	stw	@x[3],12($out)
+	stw	@x[4],16($out)
+	stw	@x[5],20($out)
+	stw	@x[6],24($out)
+	stw	@x[7],28($out)
+	stw	@x[8],32($out)
+	stw	@x[9],36($out)
+	stw	@x[10],40($out)
+	stw	@x[11],44($out)
+	stw	@x[12],48($out)
+	stw	@x[13],52($out)
+	stw	@x[14],56($out)
+	addi	$inp,$inp,64
+	stw	@x[15],60($out)
+	addi	$out,$out,64
+
+	bne	Loop_outer
+
+	blr
+
+.align	4
+Ltail:
+	addi	$len,$len,64			# restore tail length
+	subi	$inp,$inp,1			# prepare for *++ptr
+	subi	$out,$out,1
+	addi	@t[0],$sp,$LOCALS-1
+	mtctr	$len
+
+	stw	@x[0],`$LOCALS+0`($sp)		# save whole block to stack
+	stw	@x[1],`$LOCALS+4`($sp)
+	stw	@x[2],`$LOCALS+8`($sp)
+	stw	@x[3],`$LOCALS+12`($sp)
+	stw	@x[4],`$LOCALS+16`($sp)
+	stw	@x[5],`$LOCALS+20`($sp)
+	stw	@x[6],`$LOCALS+24`($sp)
+	stw	@x[7],`$LOCALS+28`($sp)
+	stw	@x[8],`$LOCALS+32`($sp)
+	stw	@x[9],`$LOCALS+36`($sp)
+	stw	@x[10],`$LOCALS+40`($sp)
+	stw	@x[11],`$LOCALS+44`($sp)
+	stw	@x[12],`$LOCALS+48`($sp)
+	stw	@x[13],`$LOCALS+52`($sp)
+	stw	@x[14],`$LOCALS+56`($sp)
+	stw	@x[15],`$LOCALS+60`($sp)
+
+Loop_tail:					# byte-by-byte loop
+	lbzu	@d[0],1($inp)
+	lbzu	@x[0],1(@t[0])
+	xor	@d[1],@d[0],@x[0]
+	stbu	@d[1],1($out)
+	bdnz	Loop_tail
+
+	stw	$sp,`$LOCALS+0`($sp)		# wipe block on stack
+	stw	$sp,`$LOCALS+4`($sp)
+	stw	$sp,`$LOCALS+8`($sp)
+	stw	$sp,`$LOCALS+12`($sp)
+	stw	$sp,`$LOCALS+16`($sp)
+	stw	$sp,`$LOCALS+20`($sp)
+	stw	$sp,`$LOCALS+24`($sp)
+	stw	$sp,`$LOCALS+28`($sp)
+	stw	$sp,`$LOCALS+32`($sp)
+	stw	$sp,`$LOCALS+36`($sp)
+	stw	$sp,`$LOCALS+40`($sp)
+	stw	$sp,`$LOCALS+44`($sp)
+	stw	$sp,`$LOCALS+48`($sp)
+	stw	$sp,`$LOCALS+52`($sp)
+	stw	$sp,`$LOCALS+56`($sp)
+	stw	$sp,`$LOCALS+60`($sp)
+
+	blr
+	.long	0
+	.byte	0,12,0x14,0,0,0,0,0
+___
+
+{{{
+my ($A0,$B0,$C0,$D0,$A1,$B1,$C1,$D1,$A2,$B2,$C2,$D2)
+				= map("v$_",(0..11));
+my @K				= map("v$_",(12..17));
+my ($FOUR,$sixteen,$twenty4)	= map("v$_",(18..19,23));
+my ($inpperm,$outperm,$outmask)	= map("v$_",(24..26));
+my @D				= map("v$_",(27..31));
+my ($twelve,$seven,$T0,$T1) = @D;
+
+my $FRAME=$LOCALS+64+10*16+18*$SIZE_T;	# 10*16 is for v23-v31 offload
+
+sub VMXROUND {
+my $odd = pop;
+my ($a,$b,$c,$d)=@_;
+
+	(
+	"&vadduwm	('$a','$a','$b')",
+	"&vxor		('$d','$d','$a')",
+	"&vperm		('$d','$d','$d','$sixteen')",
+
+	"&vadduwm	('$c','$c','$d')",
+	"&vxor		('$b','$b','$c')",
+	"&vrlw		('$b','$b','$twelve')",
+
+	"&vadduwm	('$a','$a','$b')",
+	"&vxor		('$d','$d','$a')",
+	"&vperm		('$d','$d','$d','$twenty4')",
+
+	"&vadduwm	('$c','$c','$d')",
+	"&vxor		('$b','$b','$c')",
+	"&vrlw		('$b','$b','$seven')",
+
+	"&vrldoi	('$c','$c',8)",
+	"&vrldoi	('$b','$b',$odd?4:12)",
+	"&vrldoi	('$d','$d',$odd?12:4)"
+	);
+}
+
+$code.=<<___;
+
+.globl	.ChaCha20_ctr32_vmx
+.align	5
+.ChaCha20_ctr32_vmx:
+	${UCMP}i $len,256
+	blt	__ChaCha20_ctr32_int
+
+	$STU	$sp,-$FRAME($sp)
+	mflr	r0
+	li	r10,`15+$LOCALS+64`
+	li	r11,`31+$LOCALS+64`
+	mfspr	r12,256
+	stvx	v23,r10,$sp
+	addi	r10,r10,32
+	stvx	v24,r11,$sp
+	addi	r11,r11,32
+	stvx	v25,r10,$sp
+	addi	r10,r10,32
+	stvx	v26,r11,$sp
+	addi	r11,r11,32
+	stvx	v27,r10,$sp
+	addi	r10,r10,32
+	stvx	v28,r11,$sp
+	addi	r11,r11,32
+	stvx	v29,r10,$sp
+	addi	r10,r10,32
+	stvx	v30,r11,$sp
+	stvx	v31,r10,$sp
+	stw	r12,`$FRAME-$SIZE_T*18-4`($sp)	# save vrsave
+	$PUSH	r14,`$FRAME-$SIZE_T*18`($sp)
+	$PUSH	r15,`$FRAME-$SIZE_T*17`($sp)
+	$PUSH	r16,`$FRAME-$SIZE_T*16`($sp)
+	$PUSH	r17,`$FRAME-$SIZE_T*15`($sp)
+	$PUSH	r18,`$FRAME-$SIZE_T*14`($sp)
+	$PUSH	r19,`$FRAME-$SIZE_T*13`($sp)
+	$PUSH	r20,`$FRAME-$SIZE_T*12`($sp)
+	$PUSH	r21,`$FRAME-$SIZE_T*11`($sp)
+	$PUSH	r22,`$FRAME-$SIZE_T*10`($sp)
+	$PUSH	r23,`$FRAME-$SIZE_T*9`($sp)
+	$PUSH	r24,`$FRAME-$SIZE_T*8`($sp)
+	$PUSH	r25,`$FRAME-$SIZE_T*7`($sp)
+	$PUSH	r26,`$FRAME-$SIZE_T*6`($sp)
+	$PUSH	r27,`$FRAME-$SIZE_T*5`($sp)
+	$PUSH	r28,`$FRAME-$SIZE_T*4`($sp)
+	$PUSH	r29,`$FRAME-$SIZE_T*3`($sp)
+	$PUSH	r30,`$FRAME-$SIZE_T*2`($sp)
+	$PUSH	r31,`$FRAME-$SIZE_T*1`($sp)
+	li	r12,-4096+511
+	$PUSH	r0, `$FRAME+$LRSAVE`($sp)
+	mtspr	256,r12				# preserve 29 AltiVec registers
+
+	bl	Lconsts				# returns pointer Lsigma in r12
+	li	@x[0],16
+	li	@x[1],32
+	li	@x[2],48
+	li	@x[3],64
+	li	@x[4],31			# 31 is not a typo
+	li	@x[5],15			# nor is 15
+
+	lvx	@K[1],0,$key			# load key
+	?lvsr	$T0,0,$key			# prepare unaligned load
+	lvx	@K[2],@x[0],$key
+	lvx	@D[0],@x[4],$key
+
+	lvx	@K[3],0,$ctr			# load counter
+	?lvsr	$T1,0,$ctr			# prepare unaligned load
+	lvx	@D[1],@x[5],$ctr
+
+	lvx	@K[0],0,r12			# load constants
+	lvx	@K[5],@x[0],r12			# one
+	lvx	$FOUR,@x[1],r12
+	lvx	$sixteen,@x[2],r12
+	lvx	$twenty4,@x[3],r12
+
+	?vperm	@K[1],@K[2],@K[1],$T0		# align key
+	?vperm	@K[2],@D[0],@K[2],$T0
+	?vperm	@K[3],@D[1],@K[3],$T1		# align counter
+
+	lwz	@d[0],0($ctr)			# load counter to GPR
+	lwz	@d[1],4($ctr)
+	vadduwm	@K[3],@K[3],@K[5]		# adjust AltiVec counter
+	lwz	@d[2],8($ctr)
+	vadduwm	@K[4],@K[3],@K[5]
+	lwz	@d[3],12($ctr)
+	vadduwm	@K[5],@K[4],@K[5]
+
+	vxor	$T0,$T0,$T0			# 0x00..00
+	vspltisw $outmask,-1			# 0xff..ff
+	?lvsr	$inpperm,0,$inp			# prepare for unaligned load
+	?lvsl	$outperm,0,$out			# prepare for unaligned store
+	?vperm	$outmask,$outmask,$T0,$outperm
+
+	be?lvsl	$T0,0,@x[0]			# 0x00..0f
+	be?vspltisb $T1,3			# 0x03..03
+	be?vxor	$T0,$T0,$T1			# swap bytes within words
+	be?vxor	$outperm,$outperm,$T1
+	be?vperm $inpperm,$inpperm,$inpperm,$T0
+
+	li	r0,10				# inner loop counter
+	b	Loop_outer_vmx
+
+.align	4
+Loop_outer_vmx:
+	lis	@x[0],0x6170			# synthesize sigma
+	lis	@x[1],0x3320
+	 vmr	$A0,@K[0]
+	lis	@x[2],0x7962
+	lis	@x[3],0x6b20
+	 vmr	$A1,@K[0]
+	ori	@x[0],@x[0],0x7865
+	ori	@x[1],@x[1],0x646e
+	 vmr	$A2,@K[0]
+	ori	@x[2],@x[2],0x2d32
+	ori	@x[3],@x[3],0x6574
+	 vmr	$B0,@K[1]
+
+	lwz	@x[4],0($key)			# load key to GPR
+	 vmr	$B1,@K[1]
+	lwz	@x[5],4($key)
+	 vmr	$B2,@K[1]
+	lwz	@x[6],8($key)
+	 vmr	$C0,@K[2]
+	lwz	@x[7],12($key)
+	 vmr	$C1,@K[2]
+	lwz	@x[8],16($key)
+	 vmr	$C2,@K[2]
+	mr	@x[12],@d[0]			# copy GPR counter
+	lwz	@x[9],20($key)
+	 vmr	$D0,@K[3]
+	mr	@x[13],@d[1]
+	lwz	@x[10],24($key)
+	 vmr	$D1,@K[4]
+	mr	@x[14],@d[2]
+	lwz	@x[11],28($key)
+	 vmr	$D2,@K[5]
+	mr	@x[15],@d[3]
+
+	mr	@t[0],@x[4]
+	mr	@t[1],@x[5]
+	mr	@t[2],@x[6]
+	mr	@t[3],@x[7]
+
+	vspltisw $twelve,12			# synthesize constants
+	vspltisw $seven,7
+
+	mtctr	r0
+	nop
+Loop_vmx:
+___
+	my @thread0=&VMXROUND($A0,$B0,$C0,$D0,0);
+	my @thread1=&VMXROUND($A1,$B1,$C1,$D1,0);
+	my @thread2=&VMXROUND($A2,$B2,$C2,$D2,0);
+	my @thread3=&ROUND(0,4,8,12);
+
+	foreach (@thread0) {
+		eval;
+		eval(shift(@thread1));
+		eval(shift(@thread2));
+
+		eval(shift(@thread3));
+		eval(shift(@thread3));
+		eval(shift(@thread3));
+	}
+	foreach (@thread3) { eval; }
+
+	@thread0=&VMXROUND($A0,$B0,$C0,$D0,1);
+	@thread1=&VMXROUND($A1,$B1,$C1,$D1,1);
+	@thread2=&VMXROUND($A2,$B2,$C2,$D2,1);
+	@thread3=&ROUND(0,5,10,15);
+
+	foreach (@thread0) {
+		eval;
+		eval(shift(@thread1));
+		eval(shift(@thread2));
+
+		eval(shift(@thread3));
+		eval(shift(@thread3));
+		eval(shift(@thread3));
+	}
+	foreach (@thread3) { eval; }
+$code.=<<___;
+	bdnz	Loop_vmx
+
+	subi	$len,$len,256			# $len-=256
+	addi	@x[0],@x[0],0x7865		# accumulate key block
+	addi	@x[1],@x[1],0x646e
+	addi	@x[2],@x[2],0x2d32
+	addi	@x[3],@x[3],0x6574
+	addis	@x[0],@x[0],0x6170
+	addis	@x[1],@x[1],0x3320
+	addis	@x[2],@x[2],0x7962
+	addis	@x[3],@x[3],0x6b20
+	add	@x[4],@x[4],@t[0]
+	lwz	@t[0],16($key)
+	add	@x[5],@x[5],@t[1]
+	lwz	@t[1],20($key)
+	add	@x[6],@x[6],@t[2]
+	lwz	@t[2],24($key)
+	add	@x[7],@x[7],@t[3]
+	lwz	@t[3],28($key)
+	add	@x[8],@x[8],@t[0]
+	add	@x[9],@x[9],@t[1]
+	add	@x[10],@x[10],@t[2]
+	add	@x[11],@x[11],@t[3]
+	add	@x[12],@x[12],@d[0]
+	add	@x[13],@x[13],@d[1]
+	add	@x[14],@x[14],@d[2]
+	add	@x[15],@x[15],@d[3]
+
+	vadduwm	$A0,$A0,@K[0]			# accumulate key block
+	vadduwm	$A1,$A1,@K[0]
+	vadduwm	$A2,$A2,@K[0]
+	vadduwm	$B0,$B0,@K[1]
+	vadduwm	$B1,$B1,@K[1]
+	vadduwm	$B2,$B2,@K[1]
+	vadduwm	$C0,$C0,@K[2]
+	vadduwm	$C1,$C1,@K[2]
+	vadduwm	$C2,$C2,@K[2]
+	vadduwm	$D0,$D0,@K[3]
+	vadduwm	$D1,$D1,@K[4]
+	vadduwm	$D2,$D2,@K[5]
+
+	addi	@d[0],@d[0],4			# increment counter
+	vadduwm	@K[3],@K[3],$FOUR
+	vadduwm	@K[4],@K[4],$FOUR
+	vadduwm	@K[5],@K[5],$FOUR
+
+___
+if (!$LITTLE_ENDIAN) { for($i=0;$i<16;$i++) {	# flip byte order
+$code.=<<___;
+	mr	@t[$i&3],@x[$i]
+	rotlwi	@x[$i],@x[$i],8
+	rlwimi	@x[$i],@t[$i&3],24,0,7
+	rlwimi	@x[$i],@t[$i&3],24,16,23
+___
+} }
+$code.=<<___;
+	lwz	@t[0],0($inp)			# load input, aligned or not
+	lwz	@t[1],4($inp)
+	lwz	@t[2],8($inp)
+	lwz	@t[3],12($inp)
+	xor	@x[0],@x[0],@t[0]		# xor with input
+	lwz	@t[0],16($inp)
+	xor	@x[1],@x[1],@t[1]
+	lwz	@t[1],20($inp)
+	xor	@x[2],@x[2],@t[2]
+	lwz	@t[2],24($inp)
+	xor	@x[3],@x[3],@t[3]
+	lwz	@t[3],28($inp)
+	xor	@x[4],@x[4],@t[0]
+	lwz	@t[0],32($inp)
+	xor	@x[5],@x[5],@t[1]
+	lwz	@t[1],36($inp)
+	xor	@x[6],@x[6],@t[2]
+	lwz	@t[2],40($inp)
+	xor	@x[7],@x[7],@t[3]
+	lwz	@t[3],44($inp)
+	xor	@x[8],@x[8],@t[0]
+	lwz	@t[0],48($inp)
+	xor	@x[9],@x[9],@t[1]
+	lwz	@t[1],52($inp)
+	xor	@x[10],@x[10],@t[2]
+	lwz	@t[2],56($inp)
+	xor	@x[11],@x[11],@t[3]
+	lwz	@t[3],60($inp)
+	xor	@x[12],@x[12],@t[0]
+	stw	@x[0],0($out)			# store output, aligned or not
+	xor	@x[13],@x[13],@t[1]
+	stw	@x[1],4($out)
+	xor	@x[14],@x[14],@t[2]
+	stw	@x[2],8($out)
+	xor	@x[15],@x[15],@t[3]
+	stw	@x[3],12($out)
+	addi	$inp,$inp,64
+	stw	@x[4],16($out)
+	li	@t[0],16
+	stw	@x[5],20($out)
+	li	@t[1],32
+	stw	@x[6],24($out)
+	li	@t[2],48
+	stw	@x[7],28($out)
+	li	@t[3],64
+	stw	@x[8],32($out)
+	stw	@x[9],36($out)
+	stw	@x[10],40($out)
+	stw	@x[11],44($out)
+	stw	@x[12],48($out)
+	stw	@x[13],52($out)
+	stw	@x[14],56($out)
+	stw	@x[15],60($out)
+	addi	$out,$out,64
+
+	lvx	@D[0],0,$inp			# load input
+	lvx	@D[1],@t[0],$inp
+	lvx	@D[2],@t[1],$inp
+	lvx	@D[3],@t[2],$inp
+	lvx	@D[4],@t[3],$inp
+	addi	$inp,$inp,64
+
+	?vperm	@D[0],@D[1],@D[0],$inpperm	# align input
+	?vperm	@D[1],@D[2],@D[1],$inpperm
+	?vperm	@D[2],@D[3],@D[2],$inpperm
+	?vperm	@D[3],@D[4],@D[3],$inpperm
+	vxor	$A0,$A0,@D[0]			# xor with input
+	vxor	$B0,$B0,@D[1]
+	lvx	@D[1],@t[0],$inp		# keep loading input
+	vxor	$C0,$C0,@D[2]
+	lvx	@D[2],@t[1],$inp
+	vxor	$D0,$D0,@D[3]
+	lvx	@D[3],@t[2],$inp
+	lvx	@D[0],@t[3],$inp
+	addi	$inp,$inp,64
+	li	@t[3],63			# 63 is not a typo
+	vperm	$A0,$A0,$A0,$outperm		# pre-misalign output
+	vperm	$B0,$B0,$B0,$outperm
+	vperm	$C0,$C0,$C0,$outperm
+	vperm	$D0,$D0,$D0,$outperm
+
+	?vperm	@D[4],@D[1],@D[4],$inpperm	# align input
+	?vperm	@D[1],@D[2],@D[1],$inpperm
+	?vperm	@D[2],@D[3],@D[2],$inpperm
+	?vperm	@D[3],@D[0],@D[3],$inpperm
+	vxor	$A1,$A1,@D[4]
+	vxor	$B1,$B1,@D[1]
+	lvx	@D[1],@t[0],$inp		# keep loading input
+	vxor	$C1,$C1,@D[2]
+	lvx	@D[2],@t[1],$inp
+	vxor	$D1,$D1,@D[3]
+	lvx	@D[3],@t[2],$inp
+	lvx	@D[4],@t[3],$inp		# redundant in aligned case
+	addi	$inp,$inp,64
+	vperm	$A1,$A1,$A1,$outperm		# pre-misalign output
+	vperm	$B1,$B1,$B1,$outperm
+	vperm	$C1,$C1,$C1,$outperm
+	vperm	$D1,$D1,$D1,$outperm
+
+	?vperm	@D[0],@D[1],@D[0],$inpperm	# align input
+	?vperm	@D[1],@D[2],@D[1],$inpperm
+	?vperm	@D[2],@D[3],@D[2],$inpperm
+	?vperm	@D[3],@D[4],@D[3],$inpperm
+	vxor	$A2,$A2,@D[0]
+	vxor	$B2,$B2,@D[1]
+	vxor	$C2,$C2,@D[2]
+	vxor	$D2,$D2,@D[3]
+	vperm	$A2,$A2,$A2,$outperm		# pre-misalign output
+	vperm	$B2,$B2,$B2,$outperm
+	vperm	$C2,$C2,$C2,$outperm
+	vperm	$D2,$D2,$D2,$outperm
+
+	andi.	@x[1],$out,15			# is $out aligned?
+	mr	@x[0],$out
+
+	vsel	@D[0],$A0,$B0,$outmask		# collect pre-misaligned output
+	vsel	@D[1],$B0,$C0,$outmask
+	vsel	@D[2],$C0,$D0,$outmask
+	vsel	@D[3],$D0,$A1,$outmask
+	vsel	$B0,$A1,$B1,$outmask
+	vsel	$C0,$B1,$C1,$outmask
+	vsel	$D0,$C1,$D1,$outmask
+	vsel	$A1,$D1,$A2,$outmask
+	vsel	$B1,$A2,$B2,$outmask
+	vsel	$C1,$B2,$C2,$outmask
+	vsel	$D1,$C2,$D2,$outmask
+
+	#stvx	$A0,0,$out			# take it easy on the edges
+	stvx	@D[0],@t[0],$out		# store output
+	stvx	@D[1],@t[1],$out
+	stvx	@D[2],@t[2],$out
+	addi	$out,$out,64
+	stvx	@D[3],0,$out
+	stvx	$B0,@t[0],$out
+	stvx	$C0,@t[1],$out
+	stvx	$D0,@t[2],$out
+	addi	$out,$out,64
+	stvx	$A1,0,$out
+	stvx	$B1,@t[0],$out
+	stvx	$C1,@t[1],$out
+	stvx	$D1,@t[2],$out
+	addi	$out,$out,64
+
+	beq	Laligned_vmx
+
+	sub	@x[2],$out,@x[1]		# in misaligned case edges
+	li	@x[3],0				# are written byte-by-byte
+Lunaligned_tail_vmx:
+	stvebx	$D2,@x[3],@x[2]
+	addi	@x[3],@x[3],1
+	cmpw	@x[3],@x[1]
+	bne	Lunaligned_tail_vmx
+
+	sub	@x[2],@x[0],@x[1]
+Lunaligned_head_vmx:
+	stvebx	$A0,@x[1],@x[2]
+	cmpwi	@x[1],15
+	addi	@x[1],@x[1],1
+	bne	Lunaligned_head_vmx
+
+	${UCMP}i $len,255			# done with 256-byte blocks yet?
+	bgt	Loop_outer_vmx
+
+	b	Ldone_vmx
+
+.align	4
+Laligned_vmx:
+	stvx	$A0,0,@x[0]			# head hexaword was not stored
+
+	${UCMP}i $len,255			# done with 256-byte blocks yet?
+	bgt	Loop_outer_vmx
+	nop
+
+Ldone_vmx:
+	${UCMP}i $len,0				# done yet?
+	bnel	__ChaCha20_1x
+
+	lwz	r12,`$FRAME-$SIZE_T*18-4`($sp)	# pull vrsave
+	li	r10,`15+$LOCALS+64`
+	li	r11,`31+$LOCALS+64`
+	mtspr	256,r12				# restore vrsave
+	lvx	v23,r10,$sp
+	addi	r10,r10,32
+	lvx	v24,r11,$sp
+	addi	r11,r11,32
+	lvx	v25,r10,$sp
+	addi	r10,r10,32
+	lvx	v26,r11,$sp
+	addi	r11,r11,32
+	lvx	v27,r10,$sp
+	addi	r10,r10,32
+	lvx	v28,r11,$sp
+	addi	r11,r11,32
+	lvx	v29,r10,$sp
+	addi	r10,r10,32
+	lvx	v30,r11,$sp
+	lvx	v31,r10,$sp
+	$POP	r0, `$FRAME+$LRSAVE`($sp)
+	$POP	r14,`$FRAME-$SIZE_T*18`($sp)
+	$POP	r15,`$FRAME-$SIZE_T*17`($sp)
+	$POP	r16,`$FRAME-$SIZE_T*16`($sp)
+	$POP	r17,`$FRAME-$SIZE_T*15`($sp)
+	$POP	r18,`$FRAME-$SIZE_T*14`($sp)
+	$POP	r19,`$FRAME-$SIZE_T*13`($sp)
+	$POP	r20,`$FRAME-$SIZE_T*12`($sp)
+	$POP	r21,`$FRAME-$SIZE_T*11`($sp)
+	$POP	r22,`$FRAME-$SIZE_T*10`($sp)
+	$POP	r23,`$FRAME-$SIZE_T*9`($sp)
+	$POP	r24,`$FRAME-$SIZE_T*8`($sp)
+	$POP	r25,`$FRAME-$SIZE_T*7`($sp)
+	$POP	r26,`$FRAME-$SIZE_T*6`($sp)
+	$POP	r27,`$FRAME-$SIZE_T*5`($sp)
+	$POP	r28,`$FRAME-$SIZE_T*4`($sp)
+	$POP	r29,`$FRAME-$SIZE_T*3`($sp)
+	$POP	r30,`$FRAME-$SIZE_T*2`($sp)
+	$POP	r31,`$FRAME-$SIZE_T*1`($sp)
+	mtlr	r0
+	addi	$sp,$sp,$FRAME
+	blr
+	.long	0
+	.byte	0,12,0x04,1,0x80,18,5,0
+	.long	0
+.size	.ChaCha20_ctr32_vmx,.-.ChaCha20_ctr32_vmx
+___
+}}}
+{{{
+my ($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3,
+    $xc0,$xc1,$xc2,$xc3, $xd0,$xd1,$xd2,$xd3) = map("v$_",(0..15));
+my @K = map("v$_",(16..19));
+my $CTR = "v26";
+my ($xt0,$xt1,$xt2,$xt3) = map("v$_",(27..30));
+my ($sixteen,$twelve,$eight,$seven) = ($xt0,$xt1,$xt2,$xt3);
+my $beperm = "v31";
+
+my ($x00,$x10,$x20,$x30) = (0, map("r$_",(8..10)));
+
+my $FRAME=$LOCALS+64+7*16;	# 7*16 is for v26-v31 offload
+
+sub VSX_lane_ROUND {
+my ($a0,$b0,$c0,$d0)=@_;
+my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
+my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
+my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
+my @x=map("\"v$_\"",(0..15));
+
+	(
+	"&vadduwm	(@x[$a0],@x[$a0],@x[$b0])",	# Q1
+	 "&vadduwm	(@x[$a1],@x[$a1],@x[$b1])",	# Q2
+	  "&vadduwm	(@x[$a2],@x[$a2],@x[$b2])",	# Q3
+	   "&vadduwm	(@x[$a3],@x[$a3],@x[$b3])",	# Q4
+	"&vxor		(@x[$d0],@x[$d0],@x[$a0])",
+	 "&vxor		(@x[$d1],@x[$d1],@x[$a1])",
+	  "&vxor	(@x[$d2],@x[$d2],@x[$a2])",
+	   "&vxor	(@x[$d3],@x[$d3],@x[$a3])",
+	"&vrlw		(@x[$d0],@x[$d0],'$sixteen')",
+	 "&vrlw		(@x[$d1],@x[$d1],'$sixteen')",
+	  "&vrlw	(@x[$d2],@x[$d2],'$sixteen')",
+	   "&vrlw	(@x[$d3],@x[$d3],'$sixteen')",
+
+	"&vadduwm	(@x[$c0],@x[$c0],@x[$d0])",
+	 "&vadduwm	(@x[$c1],@x[$c1],@x[$d1])",
+	  "&vadduwm	(@x[$c2],@x[$c2],@x[$d2])",
+	   "&vadduwm	(@x[$c3],@x[$c3],@x[$d3])",
+	"&vxor		(@x[$b0],@x[$b0],@x[$c0])",
+	 "&vxor		(@x[$b1],@x[$b1],@x[$c1])",
+	  "&vxor	(@x[$b2],@x[$b2],@x[$c2])",
+	   "&vxor	(@x[$b3],@x[$b3],@x[$c3])",
+	"&vrlw		(@x[$b0],@x[$b0],'$twelve')",
+	 "&vrlw		(@x[$b1],@x[$b1],'$twelve')",
+	  "&vrlw	(@x[$b2],@x[$b2],'$twelve')",
+	   "&vrlw	(@x[$b3],@x[$b3],'$twelve')",
+
+	"&vadduwm	(@x[$a0],@x[$a0],@x[$b0])",
+	 "&vadduwm	(@x[$a1],@x[$a1],@x[$b1])",
+	  "&vadduwm	(@x[$a2],@x[$a2],@x[$b2])",
+	   "&vadduwm	(@x[$a3],@x[$a3],@x[$b3])",
+	"&vxor		(@x[$d0],@x[$d0],@x[$a0])",
+	 "&vxor		(@x[$d1],@x[$d1],@x[$a1])",
+	  "&vxor	(@x[$d2],@x[$d2],@x[$a2])",
+	   "&vxor	(@x[$d3],@x[$d3],@x[$a3])",
+	"&vrlw		(@x[$d0],@x[$d0],'$eight')",
+	 "&vrlw		(@x[$d1],@x[$d1],'$eight')",
+	  "&vrlw	(@x[$d2],@x[$d2],'$eight')",
+	   "&vrlw	(@x[$d3],@x[$d3],'$eight')",
+
+	"&vadduwm	(@x[$c0],@x[$c0],@x[$d0])",
+	 "&vadduwm	(@x[$c1],@x[$c1],@x[$d1])",
+	  "&vadduwm	(@x[$c2],@x[$c2],@x[$d2])",
+	   "&vadduwm	(@x[$c3],@x[$c3],@x[$d3])",
+	"&vxor		(@x[$b0],@x[$b0],@x[$c0])",
+	 "&vxor		(@x[$b1],@x[$b1],@x[$c1])",
+	  "&vxor	(@x[$b2],@x[$b2],@x[$c2])",
+	   "&vxor	(@x[$b3],@x[$b3],@x[$c3])",
+	"&vrlw		(@x[$b0],@x[$b0],'$seven')",
+	 "&vrlw		(@x[$b1],@x[$b1],'$seven')",
+	  "&vrlw	(@x[$b2],@x[$b2],'$seven')",
+	   "&vrlw	(@x[$b3],@x[$b3],'$seven')"
+	);
+}
+
+$code.=<<___;
+
+.globl	.ChaCha20_ctr32_vsx
+.align	5
+.ChaCha20_ctr32_vsx:
+	$STU	$sp,-$FRAME($sp)
+	mflr	r0
+	li	r10,`15+$LOCALS+64`
+	li	r11,`31+$LOCALS+64`
+	mfspr	r12,256
+	stvx	v26,r10,$sp
+	addi	r10,r10,32
+	stvx	v27,r11,$sp
+	addi	r11,r11,32
+	stvx	v28,r10,$sp
+	addi	r10,r10,32
+	stvx	v29,r11,$sp
+	addi	r11,r11,32
+	stvx	v30,r10,$sp
+	stvx	v31,r11,$sp
+	stw	r12,`$FRAME-4`($sp)		# save vrsave
+	li	r12,-4096+63
+	$PUSH	r0, `$FRAME+$LRSAVE`($sp)
+	mtspr	256,r12				# preserve 29 AltiVec registers
+
+	bl	Lconsts				# returns pointer Lsigma in r12
+	lvx_4w	@K[0],0,r12			# load sigma
+	addi	r12,r12,0x50
+	li	$x10,16
+	li	$x20,32
+	li	$x30,48
+	li	r11,64
+
+	lvx_4w	@K[1],0,$key			# load key
+	lvx_4w	@K[2],$x10,$key
+	lvx_4w	@K[3],0,$ctr			# load counter
+
+	vxor	$xt0,$xt0,$xt0
+	lvx_4w	$xt1,r11,r12
+	vspltw	$CTR,@K[3],0
+	vsldoi	@K[3],@K[3],$xt0,4
+	vsldoi	@K[3],$xt0,@K[3],12		# clear @K[3].word[0]
+	vadduwm	$CTR,$CTR,$xt1
+
+	be?lvsl	$beperm,0,$x10			# 0x00..0f
+	be?vspltisb $xt0,3			# 0x03..03
+	be?vxor	$beperm,$beperm,$xt0		# swap bytes within words
+
+	li	r0,10				# inner loop counter
+	mtctr	r0
+	b	Loop_outer_vsx
+
+.align	5
+Loop_outer_vsx:
+	lvx	$xa0,$x00,r12			# load [smashed] sigma
+	lvx	$xa1,$x10,r12
+	lvx	$xa2,$x20,r12
+	lvx	$xa3,$x30,r12
+
+	vspltw	$xb0,@K[1],0			# smash the key
+	vspltw	$xb1,@K[1],1
+	vspltw	$xb2,@K[1],2
+	vspltw	$xb3,@K[1],3
+
+	vspltw	$xc0,@K[2],0
+	vspltw	$xc1,@K[2],1
+	vspltw	$xc2,@K[2],2
+	vspltw	$xc3,@K[2],3
+
+	vmr	$xd0,$CTR			# smash the counter
+	vspltw	$xd1,@K[3],1
+	vspltw	$xd2,@K[3],2
+	vspltw	$xd3,@K[3],3
+
+	vspltisw $sixteen,-16			# synthesize constants
+	vspltisw $twelve,12
+	vspltisw $eight,8
+	vspltisw $seven,7
+
+Loop_vsx:
+___
+	foreach (&VSX_lane_ROUND(0, 4, 8,12)) { eval; }
+	foreach (&VSX_lane_ROUND(0, 5,10,15)) { eval; }
+$code.=<<___;
+	bdnz	Loop_vsx
+
+	vadduwm	$xd0,$xd0,$CTR
+
+	vmrgew	$xt0,$xa0,$xa1			# transpose data
+	vmrgew	$xt1,$xa2,$xa3
+	vmrgow	$xa0,$xa0,$xa1
+	vmrgow	$xa2,$xa2,$xa3
+	 vmrgew	$xt2,$xb0,$xb1
+	 vmrgew	$xt3,$xb2,$xb3
+	vpermdi	$xa1,$xa0,$xa2,0b00
+	vpermdi	$xa3,$xa0,$xa2,0b11
+	vpermdi	$xa0,$xt0,$xt1,0b00
+	vpermdi	$xa2,$xt0,$xt1,0b11
+
+	vmrgow	$xb0,$xb0,$xb1
+	vmrgow	$xb2,$xb2,$xb3
+	 vmrgew	$xt0,$xc0,$xc1
+	 vmrgew	$xt1,$xc2,$xc3
+	vpermdi	$xb1,$xb0,$xb2,0b00
+	vpermdi	$xb3,$xb0,$xb2,0b11
+	vpermdi	$xb0,$xt2,$xt3,0b00
+	vpermdi	$xb2,$xt2,$xt3,0b11
+
+	vmrgow	$xc0,$xc0,$xc1
+	vmrgow	$xc2,$xc2,$xc3
+	 vmrgew	$xt2,$xd0,$xd1
+	 vmrgew	$xt3,$xd2,$xd3
+	vpermdi	$xc1,$xc0,$xc2,0b00
+	vpermdi	$xc3,$xc0,$xc2,0b11
+	vpermdi	$xc0,$xt0,$xt1,0b00
+	vpermdi	$xc2,$xt0,$xt1,0b11
+
+	vmrgow	$xd0,$xd0,$xd1
+	vmrgow	$xd2,$xd2,$xd3
+	 vspltisw $xt0,4
+	 vadduwm  $CTR,$CTR,$xt0		# next counter value
+	vpermdi	$xd1,$xd0,$xd2,0b00
+	vpermdi	$xd3,$xd0,$xd2,0b11
+	vpermdi	$xd0,$xt2,$xt3,0b00
+	vpermdi	$xd2,$xt2,$xt3,0b11
+
+	vadduwm	$xa0,$xa0,@K[0]
+	vadduwm	$xb0,$xb0,@K[1]
+	vadduwm	$xc0,$xc0,@K[2]
+	vadduwm	$xd0,$xd0,@K[3]
+
+	be?vperm $xa0,$xa0,$xa0,$beperm
+	be?vperm $xb0,$xb0,$xb0,$beperm
+	be?vperm $xc0,$xc0,$xc0,$beperm
+	be?vperm $xd0,$xd0,$xd0,$beperm
+
+	${UCMP}i $len,0x40
+	blt	Ltail_vsx
+
+	lvx_4w	$xt0,$x00,$inp
+	lvx_4w	$xt1,$x10,$inp
+	lvx_4w	$xt2,$x20,$inp
+	lvx_4w	$xt3,$x30,$inp
+
+	vxor	$xt0,$xt0,$xa0
+	vxor	$xt1,$xt1,$xb0
+	vxor	$xt2,$xt2,$xc0
+	vxor	$xt3,$xt3,$xd0
+
+	stvx_4w	$xt0,$x00,$out
+	stvx_4w	$xt1,$x10,$out
+	addi	$inp,$inp,0x40
+	stvx_4w	$xt2,$x20,$out
+	subi	$len,$len,0x40
+	stvx_4w	$xt3,$x30,$out
+	addi	$out,$out,0x40
+	beq	Ldone_vsx
+
+	vadduwm	$xa0,$xa1,@K[0]
+	vadduwm	$xb0,$xb1,@K[1]
+	vadduwm	$xc0,$xc1,@K[2]
+	vadduwm	$xd0,$xd1,@K[3]
+
+	be?vperm $xa0,$xa0,$xa0,$beperm
+	be?vperm $xb0,$xb0,$xb0,$beperm
+	be?vperm $xc0,$xc0,$xc0,$beperm
+	be?vperm $xd0,$xd0,$xd0,$beperm
+
+	${UCMP}i $len,0x40
+	blt	Ltail_vsx
+
+	lvx_4w	$xt0,$x00,$inp
+	lvx_4w	$xt1,$x10,$inp
+	lvx_4w	$xt2,$x20,$inp
+	lvx_4w	$xt3,$x30,$inp
+
+	vxor	$xt0,$xt0,$xa0
+	vxor	$xt1,$xt1,$xb0
+	vxor	$xt2,$xt2,$xc0
+	vxor	$xt3,$xt3,$xd0
+
+	stvx_4w	$xt0,$x00,$out
+	stvx_4w	$xt1,$x10,$out
+	addi	$inp,$inp,0x40
+	stvx_4w	$xt2,$x20,$out
+	subi	$len,$len,0x40
+	stvx_4w	$xt3,$x30,$out
+	addi	$out,$out,0x40
+	beq	Ldone_vsx
+
+	vadduwm	$xa0,$xa2,@K[0]
+	vadduwm	$xb0,$xb2,@K[1]
+	vadduwm	$xc0,$xc2,@K[2]
+	vadduwm	$xd0,$xd2,@K[3]
+
+	be?vperm $xa0,$xa0,$xa0,$beperm
+	be?vperm $xb0,$xb0,$xb0,$beperm
+	be?vperm $xc0,$xc0,$xc0,$beperm
+	be?vperm $xd0,$xd0,$xd0,$beperm
+
+	${UCMP}i $len,0x40
+	blt	Ltail_vsx
+
+	lvx_4w	$xt0,$x00,$inp
+	lvx_4w	$xt1,$x10,$inp
+	lvx_4w	$xt2,$x20,$inp
+	lvx_4w	$xt3,$x30,$inp
+
+	vxor	$xt0,$xt0,$xa0
+	vxor	$xt1,$xt1,$xb0
+	vxor	$xt2,$xt2,$xc0
+	vxor	$xt3,$xt3,$xd0
+
+	stvx_4w	$xt0,$x00,$out
+	stvx_4w	$xt1,$x10,$out
+	addi	$inp,$inp,0x40
+	stvx_4w	$xt2,$x20,$out
+	subi	$len,$len,0x40
+	stvx_4w	$xt3,$x30,$out
+	addi	$out,$out,0x40
+	beq	Ldone_vsx
+
+	vadduwm	$xa0,$xa3,@K[0]
+	vadduwm	$xb0,$xb3,@K[1]
+	vadduwm	$xc0,$xc3,@K[2]
+	vadduwm	$xd0,$xd3,@K[3]
+
+	be?vperm $xa0,$xa0,$xa0,$beperm
+	be?vperm $xb0,$xb0,$xb0,$beperm
+	be?vperm $xc0,$xc0,$xc0,$beperm
+	be?vperm $xd0,$xd0,$xd0,$beperm
+
+	${UCMP}i $len,0x40
+	blt	Ltail_vsx
+
+	lvx_4w	$xt0,$x00,$inp
+	lvx_4w	$xt1,$x10,$inp
+	lvx_4w	$xt2,$x20,$inp
+	lvx_4w	$xt3,$x30,$inp
+
+	vxor	$xt0,$xt0,$xa0
+	vxor	$xt1,$xt1,$xb0
+	vxor	$xt2,$xt2,$xc0
+	vxor	$xt3,$xt3,$xd0
+
+	stvx_4w	$xt0,$x00,$out
+	stvx_4w	$xt1,$x10,$out
+	addi	$inp,$inp,0x40
+	stvx_4w	$xt2,$x20,$out
+	subi	$len,$len,0x40
+	stvx_4w	$xt3,$x30,$out
+	addi	$out,$out,0x40
+	mtctr	r0
+	bne	Loop_outer_vsx
+
+Ldone_vsx:
+	lwz	r12,`$FRAME-4`($sp)		# pull vrsave
+	li	r10,`15+$LOCALS+64`
+	li	r11,`31+$LOCALS+64`
+	$POP	r0, `$FRAME+$LRSAVE`($sp)
+	mtspr	256,r12				# restore vrsave
+	lvx	v26,r10,$sp
+	addi	r10,r10,32
+	lvx	v27,r11,$sp
+	addi	r11,r11,32
+	lvx	v28,r10,$sp
+	addi	r10,r10,32
+	lvx	v29,r11,$sp
+	addi	r11,r11,32
+	lvx	v30,r10,$sp
+	lvx	v31,r11,$sp
+	mtlr	r0
+	addi	$sp,$sp,$FRAME
+	blr
+
+.align	4
+Ltail_vsx:
+	addi	r11,$sp,$LOCALS
+	mtctr	$len
+	stvx_4w	$xa0,$x00,r11			# offload block to stack
+	stvx_4w	$xb0,$x10,r11
+	stvx_4w	$xc0,$x20,r11
+	stvx_4w	$xd0,$x30,r11
+	subi	r12,r11,1			# prepare for *++ptr
+	subi	$inp,$inp,1
+	subi	$out,$out,1
+
+Loop_tail_vsx:
+	lbzu	r6,1(r12)
+	lbzu	r7,1($inp)
+	xor	r6,r6,r7
+	stbu	r6,1($out)
+	bdnz	Loop_tail_vsx
+
+	stvx_4w	$K[0],$x00,r11			# wipe copy of the block
+	stvx_4w	$K[0],$x10,r11
+	stvx_4w	$K[0],$x20,r11
+	stvx_4w	$K[0],$x30,r11
+
+	b	Ldone_vsx
+	.long	0
+	.byte	0,12,0x04,1,0x80,0,5,0
+	.long	0
+.size	.ChaCha20_ctr32_vsx,.-.ChaCha20_ctr32_vsx
+___
+}}}
+$code.=<<___;
+.align	5
+Lconsts:
+	mflr	r0
+	bcl	20,31,\$+4
+	mflr	r12	#vvvvv "distance between . and Lsigma
+	addi	r12,r12,`64-8`
+	mtlr	r0
+	blr
+	.long	0
+	.byte	0,12,0x14,0,0,0,0,0
+	.space	`64-9*4`
+Lsigma:
+	.long   0x61707865,0x3320646e,0x79622d32,0x6b206574
+	.long	1,0,0,0
+	.long	4,0,0,0
+___
+$code.=<<___ 	if ($LITTLE_ENDIAN);
+	.long	0x0e0f0c0d,0x0a0b0809,0x06070405,0x02030001
+	.long	0x0d0e0f0c,0x090a0b08,0x05060704,0x01020300
+___
+$code.=<<___ 	if (!$LITTLE_ENDIAN);	# flipped words
+	.long	0x02030001,0x06070405,0x0a0b0809,0x0e0f0c0d
+	.long	0x01020300,0x05060704,0x090a0b08,0x0d0e0f0c
+___
+$code.=<<___;
+	.long	0x61707865,0x61707865,0x61707865,0x61707865
+	.long	0x3320646e,0x3320646e,0x3320646e,0x3320646e
+	.long	0x79622d32,0x79622d32,0x79622d32,0x79622d32
+	.long	0x6b206574,0x6b206574,0x6b206574,0x6b206574
+	.long	0,1,2,3
+.asciz  "ChaCha20 for PowerPC/AltiVec, CRYPTOGAMS by <appro\@openssl.org>"
+.align	2
+___
+
+foreach (split("\n",$code)) {
+	s/\`([^\`]*)\`/eval $1/ge;
+
+	# instructions prefixed with '?' are endian-specific and need
+	# to be adjusted accordingly...
+	if ($flavour !~ /le$/) {	# big-endian
+	    s/be\?//		or
+	    s/le\?/#le#/	or
+	    s/\?lvsr/lvsl/	or
+	    s/\?lvsl/lvsr/	or
+	    s/\?(vperm\s+v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+)/$1$3$2$4/ or
+	    s/vrldoi(\s+v[0-9]+,\s*)(v[0-9]+,)\s*([0-9]+)/vsldoi$1$2$2 16-$3/;
+	} else {			# little-endian
+	    s/le\?//		or
+	    s/be\?/#be#/	or
+	    s/\?([a-z]+)/$1/	or
+	    s/vrldoi(\s+v[0-9]+,\s*)(v[0-9]+,)\s*([0-9]+)/vsldoi$1$2$2 $3/;
+	}
+
+	print $_,"\n";
+}
+
+close STDOUT;
diff --git a/src/crypto/zinc/chacha20/chacha20.c b/src/crypto/zinc/chacha20/chacha20.c
index b4763c8..42e5360 100644
--- a/src/crypto/zinc/chacha20/chacha20.c
+++ b/src/crypto/zinc/chacha20/chacha20.c
@@ -20,10 +20,12 @@
 #include "chacha20-x86_64-glue.c"
 #elif defined(CONFIG_ZINC_ARCH_ARM) || defined(CONFIG_ZINC_ARCH_ARM64)
 #include "chacha20-arm-glue.c"
 #elif defined(CONFIG_ZINC_ARCH_MIPS)
 #include "chacha20-mips-glue.c"
+#elif defined(CONFIG_ZINC_ARCH_PPC32) || defined(CONFIG_ZINC_ARCH_PPC64)
+#include "chacha20-ppc-glue.c"
 #else
 static bool *const chacha20_nobs[] __initconst = { };
 static void __init chacha20_fpu_init(void)
 {
 }
diff --git a/src/crypto/zinc/chacha20/ppc-xlate.pl b/src/crypto/zinc/chacha20/ppc-xlate.pl
new file mode 100644
index 0000000..2362071
--- /dev/null
+++ b/src/crypto/zinc/chacha20/ppc-xlate.pl
@@ -0,0 +1,353 @@
+#! /usr/bin/env perl
+# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+#
+# This code is taken from the OpenSSL project but the author, Andy Polyakov,
+# has relicensed it under the licenses specified in the SPDX header above.
+# The original headers, including the original license headers, are
+# included below for completeness.
+#
+# Copyright 2006-2018 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the Apache License 2.0 (the "License").  You may not use
+# this file except in compliance with the License.  You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
+my $flavour = shift;
+my $output = shift;
+open STDOUT,">$output" || die "can't open $output: $!";
+
+my %GLOBALS;
+my %TYPES;
+my $dotinlocallabels=($flavour=~/linux/)?1:0;
+
+################################################################
+# directives which need special treatment on different platforms
+################################################################
+my $type = sub {
+    my ($dir,$name,$type) = @_;
+
+    $TYPES{$name} = $type;
+    if ($flavour =~ /linux/) {
+	$name =~ s|^\.||;
+	".type	$name,$type";
+    } else {
+	"";
+    }
+};
+my $globl = sub {
+    my $junk = shift;
+    my $name = shift;
+    my $global = \$GLOBALS{$name};
+    my $type = \$TYPES{$name};
+    my $ret;
+
+    $name =~ s|^\.||;
+
+    SWITCH: for ($flavour) {
+	/aix/		&& do { if (!$$type) {
+				    $$type = "\@function";
+				}
+				if ($$type =~ /function/) {
+				    $name = ".$name";
+				}
+				last;
+			      };
+	/osx/		&& do { $name = "_$name";
+				last;
+			      };
+	/linux.*(32|64le)/
+			&& do {	$ret .= ".globl	$name";
+				if (!$$type) {
+				    $ret .= "\n.type	$name,\@function";
+				    $$type = "\@function";
+				}
+				last;
+			      };
+	/linux.*64/	&& do {	$ret .= ".globl	$name";
+				if (!$$type) {
+				    $ret .= "\n.type	$name,\@function";
+				    $$type = "\@function";
+				}
+				if ($$type =~ /function/) {
+				    $ret .= "\n.section	\".opd\",\"aw\"";
+				    $ret .= "\n.align	3";
+				    $ret .= "\n$name:";
+				    $ret .= "\n.quad	.$name,.TOC.\@tocbase,0";
+				    $ret .= "\n.previous";
+				    $name = ".$name";
+				}
+				last;
+			      };
+    }
+
+    $ret = ".globl	$name" if (!$ret);
+    $$global = $name;
+    $ret;
+};
+my $text = sub {
+    my $ret = ($flavour =~ /aix/) ? ".csect\t.text[PR],7" : ".text";
+    $ret = ".abiversion	2\n".$ret	if ($flavour =~ /linux.*64le/);
+    $ret;
+};
+my $machine = sub {
+    my $junk = shift;
+    my $arch = shift;
+    if ($flavour =~ /osx/)
+    {	$arch =~ s/\"//g;
+	$arch = ($flavour=~/64/) ? "ppc970-64" : "ppc970" if ($arch eq "any");
+    }
+    ".machine	$arch";
+};
+my $size = sub {
+    if ($flavour =~ /linux/)
+    {	shift;
+	my $name = shift;
+	my $real = $GLOBALS{$name} ? \$GLOBALS{$name} : \$name;
+	my $ret  = ".size	$$real,.-$$real";
+	$name =~ s|^\.||;
+	if ($$real ne $name) {
+	    $ret .= "\n.size	$name,.-$$real";
+	}
+	$ret;
+    }
+    else
+    {	"";	}
+};
+my $asciz = sub {
+    shift;
+    my $line = join(",",@_);
+    if ($line =~ /^"(.*)"$/)
+    {	".byte	" . join(",",unpack("C*",$1),0) . "\n.align	2";	}
+    else
+    {	"";	}
+};
+my $quad = sub {
+    shift;
+    my @ret;
+    my ($hi,$lo);
+    for (@_) {
+	if (/^0x([0-9a-f]*?)([0-9a-f]{1,8})$/io)
+	{  $hi=$1?"0x$1":"0"; $lo="0x$2";  }
+	elsif (/^([0-9]+)$/o)
+	{  $hi=$1>>32; $lo=$1&0xffffffff;  } # error-prone with 32-bit perl
+	else
+	{  $hi=undef; $lo=$_; }
+
+	if (defined($hi))
+	{  push(@ret,$flavour=~/le$/o?".long\t$lo,$hi":".long\t$hi,$lo");  }
+	else
+	{  push(@ret,".quad	$lo");  }
+    }
+    join("\n",@ret);
+};
+
+################################################################
+# simplified mnemonics not handled by at least one assembler
+################################################################
+my $cmplw = sub {
+    my $f = shift;
+    my $cr = 0; $cr = shift if ($#_>1);
+    # Some out-of-date 32-bit GNU assembler just can't handle cmplw...
+    ($flavour =~ /linux.*32/) ?
+	"	.long	".sprintf "0x%x",31<<26|$cr<<23|$_[0]<<16|$_[1]<<11|64 :
+	"	cmplw	".join(',',$cr,@_);
+};
+my $bdnz = sub {
+    my $f = shift;
+    my $bo = $f=~/[\+\-]/ ? 16+9 : 16;	# optional "to be taken" hint
+    "	bc	$bo,0,".shift;
+} if ($flavour!~/linux/);
+my $bltlr = sub {
+    my $f = shift;
+    my $bo = $f=~/\-/ ? 12+2 : 12;	# optional "not to be taken" hint
+    ($flavour =~ /linux/) ?		# GNU as doesn't allow most recent hints
+	"	.long	".sprintf "0x%x",19<<26|$bo<<21|16<<1 :
+	"	bclr	$bo,0";
+};
+my $bnelr = sub {
+    my $f = shift;
+    my $bo = $f=~/\-/ ? 4+2 : 4;	# optional "not to be taken" hint
+    ($flavour =~ /linux/) ?		# GNU as doesn't allow most recent hints
+	"	.long	".sprintf "0x%x",19<<26|$bo<<21|2<<16|16<<1 :
+	"	bclr	$bo,2";
+};
+my $beqlr = sub {
+    my $f = shift;
+    my $bo = $f=~/-/ ? 12+2 : 12;	# optional "not to be taken" hint
+    ($flavour =~ /linux/) ?		# GNU as doesn't allow most recent hints
+	"	.long	".sprintf "0x%X",19<<26|$bo<<21|2<<16|16<<1 :
+	"	bclr	$bo,2";
+};
+# GNU assembler can't handle extrdi rA,rS,16,48, or when sum of last two
+# arguments is 64, with "operand out of range" error.
+my $extrdi = sub {
+    my ($f,$ra,$rs,$n,$b) = @_;
+    $b = ($b+$n)&63; $n = 64-$n;
+    "	rldicl	$ra,$rs,$b,$n";
+};
+my $vmr = sub {
+    my ($f,$vx,$vy) = @_;
+    "	vor	$vx,$vy,$vy";
+};
+
+# Some ABIs specify vrsave, special-purpose register #256, as reserved
+# for system use.
+my $no_vrsave = ($flavour =~ /aix|linux64le/);
+my $mtspr = sub {
+    my ($f,$idx,$ra) = @_;
+    if ($idx == 256 && $no_vrsave) {
+	"	or	$ra,$ra,$ra";
+    } else {
+	"	mtspr	$idx,$ra";
+    }
+};
+my $mfspr = sub {
+    my ($f,$rd,$idx) = @_;
+    if ($idx == 256 && $no_vrsave) {
+	"	li	$rd,-1";
+    } else {
+	"	mfspr	$rd,$idx";
+    }
+};
+
+# PowerISA 2.06 stuff
+sub vsxmem_op {
+    my ($f, $vrt, $ra, $rb, $op) = @_;
+    "	.long	".sprintf "0x%X",(31<<26)|($vrt<<21)|($ra<<16)|($rb<<11)|($op*2+1);
+}
+# made-up unaligned memory reference AltiVec/VMX instructions
+my $lvx_u	= sub {	vsxmem_op(@_, 844); };	# lxvd2x
+my $stvx_u	= sub {	vsxmem_op(@_, 972); };	# stxvd2x
+my $lvdx_u	= sub {	vsxmem_op(@_, 588); };	# lxsdx
+my $stvdx_u	= sub {	vsxmem_op(@_, 716); };	# stxsdx
+my $lvx_4w	= sub { vsxmem_op(@_, 780); };	# lxvw4x
+my $stvx_4w	= sub { vsxmem_op(@_, 908); };	# stxvw4x
+my $lvx_splt	= sub { vsxmem_op(@_, 332); };	# lxvdsx
+# VSX instruction[s] masqueraded as made-up AltiVec/VMX
+my $vpermdi	= sub {				# xxpermdi
+    my ($f, $vrt, $vra, $vrb, $dm) = @_;
+    $dm = oct($dm) if ($dm =~ /^0/);
+    "	.long	".sprintf "0x%X",(60<<26)|($vrt<<21)|($vra<<16)|($vrb<<11)|($dm<<8)|(10<<3)|7;
+};
+
+# PowerISA 2.07 stuff
+sub vcrypto_op {
+    my ($f, $vrt, $vra, $vrb, $op) = @_;
+    "	.long	".sprintf "0x%X",(4<<26)|($vrt<<21)|($vra<<16)|($vrb<<11)|$op;
+}
+sub vfour {
+    my ($f, $vrt, $vra, $vrb, $vrc, $op) = @_;
+    "	.long	".sprintf "0x%X",(4<<26)|($vrt<<21)|($vra<<16)|($vrb<<11)|($vrc<<6)|$op;
+};
+my $vcipher	= sub { vcrypto_op(@_, 1288); };
+my $vcipherlast	= sub { vcrypto_op(@_, 1289); };
+my $vncipher	= sub { vcrypto_op(@_, 1352); };
+my $vncipherlast= sub { vcrypto_op(@_, 1353); };
+my $vsbox	= sub { vcrypto_op(@_, 0, 1480); };
+my $vshasigmad	= sub { my ($st,$six)=splice(@_,-2); vcrypto_op(@_, $st<<4|$six, 1730); };
+my $vshasigmaw	= sub { my ($st,$six)=splice(@_,-2); vcrypto_op(@_, $st<<4|$six, 1666); };
+my $vpmsumb	= sub { vcrypto_op(@_, 1032); };
+my $vpmsumd	= sub { vcrypto_op(@_, 1224); };
+my $vpmsubh	= sub { vcrypto_op(@_, 1096); };
+my $vpmsumw	= sub { vcrypto_op(@_, 1160); };
+# These are not really crypto, but vcrypto_op template works
+my $vaddudm	= sub { vcrypto_op(@_, 192);  };
+my $vadduqm	= sub { vcrypto_op(@_, 256);  };
+my $vmuleuw	= sub { vcrypto_op(@_, 648);  };
+my $vmulouw	= sub { vcrypto_op(@_, 136);  };
+my $vrld	= sub { vcrypto_op(@_, 196);  };
+my $vsld	= sub { vcrypto_op(@_, 1476); };
+my $vsrd	= sub { vcrypto_op(@_, 1732); };
+my $vsubudm	= sub { vcrypto_op(@_, 1216); };
+my $vaddcuq	= sub { vcrypto_op(@_, 320);  };
+my $vaddeuqm	= sub { vfour(@_,60); };
+my $vaddecuq	= sub { vfour(@_,61); };
+my $vmrgew	= sub { vfour(@_,0,1932); };
+my $vmrgow	= sub { vfour(@_,0,1676); };
+
+my $mtsle	= sub {
+    my ($f, $arg) = @_;
+    "	.long	".sprintf "0x%X",(31<<26)|($arg<<21)|(147*2);
+};
+
+# VSX instructions masqueraded as AltiVec/VMX
+my $mtvrd	= sub {
+    my ($f, $vrt, $ra) = @_;
+    "	.long	".sprintf "0x%X",(31<<26)|($vrt<<21)|($ra<<16)|(179<<1)|1;
+};
+my $mtvrwz	= sub {
+    my ($f, $vrt, $ra) = @_;
+    "	.long	".sprintf "0x%X",(31<<26)|($vrt<<21)|($ra<<16)|(243<<1)|1;
+};
+my $lvwzx_u	= sub { vsxmem_op(@_, 12); };	# lxsiwzx
+my $stvwx_u	= sub { vsxmem_op(@_, 140); };	# stxsiwx
+
+# PowerISA 3.0 stuff
+my $maddhdu	= sub { vfour(@_,49); };
+my $maddld	= sub { vfour(@_,51); };
+my $darn = sub {
+    my ($f, $rt, $l) = @_;
+    "	.long	".sprintf "0x%X",(31<<26)|($rt<<21)|($l<<16)|(755<<1);
+};
+my $iseleq = sub {
+    my ($f, $rt, $ra, $rb) = @_;
+    "	.long	".sprintf "0x%X",(31<<26)|($rt<<21)|($ra<<16)|($rb<<11)|(2<<6)|30;
+};
+# VSX instruction[s] masqueraded as made-up AltiVec/VMX
+my $vspltib	= sub {				# xxspltib
+    my ($f, $vrt, $imm8) = @_;
+    $imm8 = oct($imm8) if ($imm8 =~ /^0/);
+    $imm8 &= 0xff;
+    "	.long	".sprintf "0x%X",(60<<26)|($vrt<<21)|($imm8<<11)|(360<<1)|1;
+};
+
+# PowerISA 3.0B stuff
+my $addex = sub {
+    my ($f, $rt, $ra, $rb, $cy) = @_;	# only cy==0 is specified in 3.0B
+    "	.long	".sprintf "0x%X",(31<<26)|($rt<<21)|($ra<<16)|($rb<<11)|($cy<<9)|(170<<1);
+};
+my $vmsumudm	= sub { vfour(@_,35); };
+
+while($line=<>) {
+
+    $line =~ s|[#!;].*$||;	# get rid of asm-style comments...
+    $line =~ s|/\*.*\*/||;	# ... and C-style comments...
+    $line =~ s|^\s+||;		# ... and skip white spaces in beginning...
+    $line =~ s|\s+$||;		# ... and at the end
+
+    {
+	$line =~ s|\.L(\w+)|L$1|g;	# common denominator for Locallabel
+	$line =~ s|\bL(\w+)|\.L$1|g	if ($dotinlocallabels);
+    }
+
+    {
+	$line =~ s|(^[\.\w]+)\:\s*||;
+	my $label = $1;
+	if ($label) {
+	    my $xlated = ($GLOBALS{$label} or $label);
+	    print "$xlated:";
+	    if ($flavour =~ /linux.*64le/) {
+		if ($TYPES{$label} =~ /function/) {
+		    printf "\n.localentry	%s,0\n",$xlated;
+		}
+	    }
+	}
+    }
+
+    {
+	$line =~ s|^\s*(\.?)(\w+)([\.\+\-]?)\s*||;
+	my $c = $1; $c = "\t" if ($c eq "");
+	my $mnemonic = $2;
+	my $f = $3;
+	my $opcode = eval("\$$mnemonic");
+	$line =~ s/\b(c?[rf]|v|vs)([0-9]+)\b/$2/g if ($c ne "." and $flavour !~ /osx/);
+	if (ref($opcode) eq 'CODE') { $line = &$opcode($f,split(/,\s*/,$line)); }
+	elsif ($mnemonic)           { $line = $c.$mnemonic.$f."\t".$line; }
+    }
+
+    print $line if ($line);
+    print "\n";
+}
+
+close STDOUT;
-- 
2.20.1

_______________________________________________
WireGuard mailing list
WireGuard@lists.zx2c4.com
https://lists.zx2c4.com/mailman/listinfo/wireguard

^ permalink raw reply	[flat|nested] 5+ messages in thread

* [PATCH 1/2] [Zinc] Add PowerPC chacha20 implementation from openssl/cryptograms
  2019-05-11 13:10 [PATCH] [Zinc] Add PowerPC chacha20 implementation from openssl/cryptograms Shawn Landden
@ 2019-05-11 18:03 ` " Shawn Landden
  2019-05-11 18:03   ` [PATCH 2/2] [zinc] add accelerated poly1305 " Shawn Landden
  2019-05-13 21:31   ` [PATCH 1/2 v3] [Zinc] Add PowerPC chacha20 implementation " Shawn Landden
  0 siblings, 2 replies; 5+ messages in thread
From: Shawn Landden @ 2019-05-11 18:03 UTC (permalink / raw)
  To: wireguard

There is a bug where may_use_simd()
returns false in kworker, and prevents most of this code from running.
I reported this upstream: https://bugzilla.kernel.org/show_bug.cgi?id=203571

Otherwise this is tested (with the VSX code enabled) on Power 9 (ppc64le).
Without this I get 2GiB/s over the loopback (so 4 GiB/s), and with this
I get 2.8 GiB/s (so 5.6 GiB/s), and more time is spent in poly1305 than chacha20.
This is on a 4-thread VPS.

Signed-off-by: Shawn Landden <shawn@git.icu>

v2: more complete simd.h for PPC
    benchmarks
    whitespace issues
---
 src/compat/simd/include/linux/simd.h         |   29 +-
 src/crypto/Kbuild.include                    |   18 +-
 src/crypto/zinc/chacha20/chacha20-ppc-glue.c |   67 +
 src/crypto/zinc/chacha20/chacha20-ppc.pl     | 1355 ++++++++++++++++++
 src/crypto/zinc/chacha20/chacha20.c          |    2 +
 src/crypto/zinc/chacha20/ppc-xlate.pl        |  353 +++++
 6 files changed, 1821 insertions(+), 3 deletions(-)
 create mode 100644 src/crypto/zinc/chacha20/chacha20-ppc-glue.c
 create mode 100644 src/crypto/zinc/chacha20/chacha20-ppc.pl
 create mode 100644 src/crypto/zinc/chacha20/ppc-xlate.pl

diff --git a/src/compat/simd/include/linux/simd.h b/src/compat/simd/include/linux/simd.h
index c75c724..44060a9 100644
--- a/src/compat/simd/include/linux/simd.h
+++ b/src/compat/simd/include/linux/simd.h
@@ -11,10 +11,13 @@
 #if defined(CONFIG_X86_64)
 #include <linux/version.h>
 #include <asm/fpu/api.h>
 #elif defined(CONFIG_KERNEL_MODE_NEON)
 #include <asm/neon.h>
+#elif defined(CONFIG_ALTIVEC) || defined(CONFIG_VSX)
+#include <asm/switch_to.h>
+#include <asm/cputable.h>
 #endif
 
 typedef enum {
 	HAVE_NO_SIMD = 1 << 0,
 	HAVE_FULL_SIMD = 1 << 1,
@@ -28,17 +31,28 @@ static inline void simd_get(simd_context_t *ctx)
 	*ctx = !IS_ENABLED(CONFIG_PREEMPT_RT_BASE) && may_use_simd() ? HAVE_FULL_SIMD : HAVE_NO_SIMD;
 }
 
 static inline void simd_put(simd_context_t *ctx)
 {
+        if (*ctx & HAVE_SIMD_IN_USE) {
 #if defined(CONFIG_X86_64)
-	if (*ctx & HAVE_SIMD_IN_USE)
 		kernel_fpu_end();
 #elif defined(CONFIG_KERNEL_MODE_NEON)
-	if (*ctx & HAVE_SIMD_IN_USE)
 		kernel_neon_end();
+#elif defined(CONFIG_PPC32) || defined(CONFIG_PPC64)
+		if (cpu_has_feature(CPU_FTR_VSX_COMP)) {
+			disable_kernel_vsx();
+			preempt_enable();
+		} else if (cpu_has_feature(CPU_FTR_ALTIVEC_COMP)) {
+			disable_kernel_altivec();
+			preempt_enable();
+		} else if (!cpu_has_feature(CPU_FTR_FPU_UNAVAILABLE)) {
+			disable_kernel_fp();
+			preempt_enable();
+		}
 #endif
+	}
 	*ctx = HAVE_NO_SIMD;
 }
 
 static inline bool simd_relax(simd_context_t *ctx)
 {
@@ -60,10 +74,21 @@ static __must_check inline bool simd_use(simd_context_t *ctx)
 		return true;
 #if defined(CONFIG_X86_64)
 	kernel_fpu_begin();
 #elif defined(CONFIG_KERNEL_MODE_NEON)
 	kernel_neon_begin();
+#elif defined(CONFIG_PPC32) || defined(CONFIG_PPC64)
+		if (cpu_has_feature(CPU_FTR_VSX_COMP)) {
+			preempt_disable();
+			enable_kernel_vsx();
+		} else if (cpu_has_feature(CPU_FTR_ALTIVEC_COMP)) {
+			preempt_disable();
+			enable_kernel_altivec();
+		} else if (!cpu_has_feature(CPU_FTR_FPU_UNAVAILABLE)) {
+			preempt_disable();
+			enable_kernel_fp();
+		}
 #endif
 	*ctx |= HAVE_SIMD_IN_USE;
 	return true;
 }
 
diff --git a/src/crypto/Kbuild.include b/src/crypto/Kbuild.include
index 460684d..4e05181 100644
--- a/src/crypto/Kbuild.include
+++ b/src/crypto/Kbuild.include
@@ -11,17 +11,25 @@ ifeq ($(CONFIG_MIPS)$(CONFIG_CPU_MIPS32_R2),yy)
 CONFIG_ZINC_ARCH_MIPS := y
 endif
 ifeq ($(CONFIG_MIPS)$(CONFIG_64BIT),yy)
 CONFIG_ZINC_ARCH_MIPS64 := y
 endif
+ifeq ($(CONFIG_PPC32),y)
+CONFIG_ZINC_ARCH_PPC32 := y
+endif
+ifeq ($(CONFIG_PPC64),y)
+CONFIG_ZINC_ARCH_PPC64 := y
+endif
 
 zinc-y += chacha20/chacha20.o
 zinc-$(CONFIG_ZINC_ARCH_X86_64) += chacha20/chacha20-x86_64.o
 zinc-$(CONFIG_ZINC_ARCH_ARM) += chacha20/chacha20-arm.o chacha20/chacha20-unrolled-arm.o
 zinc-$(CONFIG_ZINC_ARCH_ARM64) += chacha20/chacha20-arm64.o
 zinc-$(CONFIG_ZINC_ARCH_MIPS) += chacha20/chacha20-mips.o
 AFLAGS_chacha20-mips.o += -O2 # This is required to fill the branch delay slots
+zinc-$(CONFIG_ZINC_ARCH_PPC32) += chacha20/chacha20-ppc.o
+zinc-$(CONFIG_ZINC_ARCH_PPC64) += chacha20/chacha20-ppc.o
 
 zinc-y += poly1305/poly1305.o
 zinc-$(CONFIG_ZINC_ARCH_X86_64) += poly1305/poly1305-x86_64.o
 zinc-$(CONFIG_ZINC_ARCH_ARM) += poly1305/poly1305-arm.o
 zinc-$(CONFIG_ZINC_ARCH_ARM64) += poly1305/poly1305-arm64.o
@@ -36,22 +44,30 @@ zinc-$(CONFIG_ZINC_ARCH_X86_64) += blake2s/blake2s-x86_64.o
 
 zinc-y += curve25519/curve25519.o
 zinc-$(CONFIG_ZINC_ARCH_ARM) += curve25519/curve25519-arm.o
 
 quiet_cmd_perlasm = PERLASM $@
-      cmd_perlasm = $(PERL) $< > $@
+      cmd_perlasm = $(PERL) $< $(perlflags-y) > $@
 $(obj)/%.S: $(src)/%.pl FORCE
 	$(call if_changed,perlasm)
 kbuild-dir := $(if $(filter /%,$(src)),$(src),$(srctree)/$(src))
 targets := $(patsubst $(kbuild-dir)/%.pl,%.S,$(wildcard $(patsubst %.o,$(kbuild-dir)/crypto/zinc/%.pl,$(zinc-y) $(zinc-m) $(zinc-))))
 
+perlflags-$(CONFIG_ZINC_ARCH_PPC32) += linux32
+ifeq ($(CONFIG_ZINC_ARCH_PPC64),y)
+perlflags-$(CONFIG_CPU_BIG_ENDIAN) += linux64
+perlflags-$(CONFIG_CPU_LITTLE_ENDIAN) += linux64le
+endif
+
 # Old kernels don't set this, which causes trouble.
 .SECONDARY:
 
 wireguard-y += $(addprefix crypto/zinc/,$(zinc-y))
 ccflags-y += -I$(src)/crypto/include
 ccflags-$(CONFIG_ZINC_ARCH_X86_64) += -DCONFIG_ZINC_ARCH_X86_64
 ccflags-$(CONFIG_ZINC_ARCH_ARM) += -DCONFIG_ZINC_ARCH_ARM
 ccflags-$(CONFIG_ZINC_ARCH_ARM64) += -DCONFIG_ZINC_ARCH_ARM64
 ccflags-$(CONFIG_ZINC_ARCH_MIPS) += -DCONFIG_ZINC_ARCH_MIPS
 ccflags-$(CONFIG_ZINC_ARCH_MIPS64) += -DCONFIG_ZINC_ARCH_MIPS64
+ccflags-$(CONFIG_ZINC_ARCH_PPC32) += -DCONFIG_ZINC_ARCH_PPC32
+ccflags-$(CONFIG_ZINC_ARCH_PPC64) += -DCONFIG_ZINC_ARCH_PPC64
 ccflags-$(CONFIG_WIREGUARD_DEBUG) += -DCONFIG_ZINC_SELFTEST
diff --git a/src/crypto/zinc/chacha20/chacha20-ppc-glue.c b/src/crypto/zinc/chacha20/chacha20-ppc-glue.c
new file mode 100644
index 0000000..addea57
--- /dev/null
+++ b/src/crypto/zinc/chacha20/chacha20-ppc-glue.c
@@ -0,0 +1,67 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (C) 2019 Shawn Landden <shawn@git.icu>. All Rights Reserved.
+ */
+
+asmlinkage void ChaCha20_ctr32_int(u8 *out, const u8 *inp,
+				   size_t len, const u32 key[8],
+				   const u32 counter[4]);
+asmlinkage void ChaCha20_ctr32_vmx(u8 *out, const u8 *inp,
+				   size_t len, const u32 key[8],
+				   const u32 counter[4]);
+asmlinkage void ChaCha20_ctr32_vsx(u8 *out, const u8 *inp,
+				   size_t len, const u32 key[8],
+				   const u32 counter[4]);
+static bool *const chacha20_nobs[] __initconst = { };
+static void __init chacha20_fpu_init(void)
+{
+}
+
+static inline bool chacha20_arch(struct chacha20_ctx *ctx, u8 *dst,
+				 const u8 *src, size_t len,
+				 simd_context_t *simd_context)
+{
+	void (*ChaCha20SIMD)(u8 *out, const u8 *inp,
+			     size_t len, const u32 key[8],
+			     const u32 counter[4]);
+
+	/* SIMD disables preemption, so relax after processing each page. */
+	BUILD_BUG_ON(PAGE_SIZE < CHACHA20_BLOCK_SIZE ||
+		     PAGE_SIZE % CHACHA20_BLOCK_SIZE);
+
+	if (cpu_has_feature(CPU_FTR_VSX_COMP))
+		ChaCha20SIMD = &ChaCha20_ctr32_vsx;
+	else if (cpu_has_feature(CPU_FTR_ALTIVEC))
+                ChaCha20SIMD = &ChaCha20_ctr32_vmx;
+	else {
+		ChaCha20_ctr32_int(dst, src, len, ctx->key, ctx->counter);
+		return true;
+	}
+	for (;;) {
+		if (len >= CHACHA20_BLOCK_SIZE * 3 && simd_use(simd_context)) {
+			const size_t bytes = min_t(size_t, len, PAGE_SIZE);
+
+			ChaCha20SIMD(dst, src, bytes, ctx->key, ctx->counter);
+			ctx->counter[0] += (bytes + 63) / 64;
+			len -= bytes;
+			if (!len)
+				break;
+			dst += bytes;
+			src += bytes;
+			simd_relax(simd_context);
+		} else {
+			ChaCha20_ctr32_int(dst, src, len, ctx->key, ctx->counter);
+			ctx->counter[0] += (len + 63) / 64;
+			return true;
+		}
+	}
+	return true;
+}
+
+static inline bool hchacha20_arch(u32 derived_key[CHACHA20_KEY_WORDS],
+				  const u8 nonce[HCHACHA20_NONCE_SIZE],
+				  const u8 key[HCHACHA20_KEY_SIZE],
+				  simd_context_t *simd_context)
+{
+	return false;
+}
diff --git a/src/crypto/zinc/chacha20/chacha20-ppc.pl b/src/crypto/zinc/chacha20/chacha20-ppc.pl
new file mode 100644
index 0000000..07468c8
--- /dev/null
+++ b/src/crypto/zinc/chacha20/chacha20-ppc.pl
@@ -0,0 +1,1355 @@
+#! /usr/bin/env perl
+# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+#
+# This code is taken from the OpenSSL project but the author, Andy Polyakov,
+# has relicensed it under the licenses specified in the SPDX header above.
+# The original headers, including the original license headers, are
+# included below for completeness.
+#
+# Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the Apache License 2.0 (the "License").  You may not use
+# this file except in compliance with the License.  You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
+#
+# ====================================================================
+# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
+# project. The module is, however, dual licensed under OpenSSL and
+# CRYPTOGAMS licenses depending on where you obtain it. For further
+# details see http://www.openssl.org/~appro/cryptogams/.
+# ====================================================================
+#
+# October 2015
+#
+# ChaCha20 for PowerPC/AltiVec.
+#
+# June 2018
+#
+# Add VSX 2.07 code path. Original 3xAltiVec+1xIALU is well-suited for
+# processors that can't issue more than one vector instruction per
+# cycle. But POWER8 (and POWER9) can issue a pair, and vector-only 4x
+# interleave would perform better. Incidentally PowerISA 2.07 (first
+# implemented by POWER8) defined new usable instructions, hence 4xVSX
+# code path...
+#
+# Performance in cycles per byte out of large buffer.
+#
+#			IALU/gcc-4.x    3xAltiVec+1xIALU	4xVSX
+#
+# Freescale e300	13.6/+115%	-			-
+# PPC74x0/G4e		6.81/+310%	3.81			-
+# PPC970/G5		9.29/+160%	?			-
+# POWER7		8.62/+61%	3.35			-
+# POWER8		8.70/+51%	2.91			2.09
+# POWER9		8.80/+29%	4.44(*)			2.45(**)
+#
+# (*)	this is trade-off result, it's possible to improve it, but
+#	then it would negatively affect all others;
+# (**)	POWER9 seems to be "allergic" to mixing vector and integer
+#	instructions, which is why switch to vector-only code pays
+#	off that much;
+
+$flavour = shift;
+
+if ($flavour =~ /64/) {
+	$SIZE_T	=8;
+	$LRSAVE	=2*$SIZE_T;
+	$STU	="stdu";
+	$POP	="ld";
+	$PUSH	="std";
+	$UCMP	="cmpld";
+} elsif ($flavour =~ /32/) {
+	$SIZE_T	=4;
+	$LRSAVE	=$SIZE_T;
+	$STU	="stwu";
+	$POP	="lwz";
+	$PUSH	="stw";
+	$UCMP	="cmplw";
+} else { die "nonsense $flavour"; }
+
+$LITTLE_ENDIAN = ($flavour=~/le$/) ? 1 : 0;
+
+$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
+( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
+( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
+die "can't locate ppc-xlate.pl";
+
+open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";
+
+$LOCALS=6*$SIZE_T;
+$FRAME=$LOCALS+64+18*$SIZE_T;	# 64 is for local variables
+
+sub AUTOLOAD()		# thunk [simplified] x86-style perlasm
+{ my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./;
+    $code .= "\t$opcode\t".join(',',@_)."\n";
+}
+
+my $sp = "r1";
+
+my ($out,$inp,$len,$key,$ctr) = map("r$_",(3..7));
+
+my @x=map("r$_",(16..31));
+my @d=map("r$_",(11,12,14,15));
+my @t=map("r$_",(7..10));
+
+sub ROUND {
+my ($a0,$b0,$c0,$d0)=@_;
+my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
+my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
+my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
+
+    (
+	"&add		(@x[$a0],@x[$a0],@x[$b0])",
+	 "&add		(@x[$a1],@x[$a1],@x[$b1])",
+	  "&add		(@x[$a2],@x[$a2],@x[$b2])",
+	   "&add	(@x[$a3],@x[$a3],@x[$b3])",
+	"&xor		(@x[$d0],@x[$d0],@x[$a0])",
+	 "&xor		(@x[$d1],@x[$d1],@x[$a1])",
+	  "&xor		(@x[$d2],@x[$d2],@x[$a2])",
+	   "&xor	(@x[$d3],@x[$d3],@x[$a3])",
+	"&rotlwi	(@x[$d0],@x[$d0],16)",
+	 "&rotlwi	(@x[$d1],@x[$d1],16)",
+	  "&rotlwi	(@x[$d2],@x[$d2],16)",
+	   "&rotlwi	(@x[$d3],@x[$d3],16)",
+
+	"&add		(@x[$c0],@x[$c0],@x[$d0])",
+	 "&add		(@x[$c1],@x[$c1],@x[$d1])",
+	  "&add		(@x[$c2],@x[$c2],@x[$d2])",
+	   "&add	(@x[$c3],@x[$c3],@x[$d3])",
+	"&xor		(@x[$b0],@x[$b0],@x[$c0])",
+	 "&xor		(@x[$b1],@x[$b1],@x[$c1])",
+	  "&xor		(@x[$b2],@x[$b2],@x[$c2])",
+	   "&xor	(@x[$b3],@x[$b3],@x[$c3])",
+	"&rotlwi	(@x[$b0],@x[$b0],12)",
+	 "&rotlwi	(@x[$b1],@x[$b1],12)",
+	  "&rotlwi	(@x[$b2],@x[$b2],12)",
+	   "&rotlwi	(@x[$b3],@x[$b3],12)",
+
+	"&add		(@x[$a0],@x[$a0],@x[$b0])",
+	 "&add		(@x[$a1],@x[$a1],@x[$b1])",
+	  "&add		(@x[$a2],@x[$a2],@x[$b2])",
+	   "&add	(@x[$a3],@x[$a3],@x[$b3])",
+	"&xor		(@x[$d0],@x[$d0],@x[$a0])",
+	 "&xor		(@x[$d1],@x[$d1],@x[$a1])",
+	  "&xor		(@x[$d2],@x[$d2],@x[$a2])",
+	   "&xor	(@x[$d3],@x[$d3],@x[$a3])",
+	"&rotlwi	(@x[$d0],@x[$d0],8)",
+	 "&rotlwi	(@x[$d1],@x[$d1],8)",
+	  "&rotlwi	(@x[$d2],@x[$d2],8)",
+	   "&rotlwi	(@x[$d3],@x[$d3],8)",
+
+	"&add		(@x[$c0],@x[$c0],@x[$d0])",
+	 "&add		(@x[$c1],@x[$c1],@x[$d1])",
+	  "&add		(@x[$c2],@x[$c2],@x[$d2])",
+	   "&add	(@x[$c3],@x[$c3],@x[$d3])",
+	"&xor		(@x[$b0],@x[$b0],@x[$c0])",
+	 "&xor		(@x[$b1],@x[$b1],@x[$c1])",
+	  "&xor		(@x[$b2],@x[$b2],@x[$c2])",
+	   "&xor	(@x[$b3],@x[$b3],@x[$c3])",
+	"&rotlwi	(@x[$b0],@x[$b0],7)",
+	 "&rotlwi	(@x[$b1],@x[$b1],7)",
+	  "&rotlwi	(@x[$b2],@x[$b2],7)",
+	   "&rotlwi	(@x[$b3],@x[$b3],7)"
+    );
+}
+
+$code.=<<___;
+.machine	"any"
+.text
+
+.globl	.ChaCha20_ctr32_int
+.align	5
+.ChaCha20_ctr32_int:
+__ChaCha20_ctr32_int:
+	${UCMP}i $len,0
+	beqlr-
+
+	$STU	$sp,-$FRAME($sp)
+	mflr	r0
+
+	$PUSH	r14,`$FRAME-$SIZE_T*18`($sp)
+	$PUSH	r15,`$FRAME-$SIZE_T*17`($sp)
+	$PUSH	r16,`$FRAME-$SIZE_T*16`($sp)
+	$PUSH	r17,`$FRAME-$SIZE_T*15`($sp)
+	$PUSH	r18,`$FRAME-$SIZE_T*14`($sp)
+	$PUSH	r19,`$FRAME-$SIZE_T*13`($sp)
+	$PUSH	r20,`$FRAME-$SIZE_T*12`($sp)
+	$PUSH	r21,`$FRAME-$SIZE_T*11`($sp)
+	$PUSH	r22,`$FRAME-$SIZE_T*10`($sp)
+	$PUSH	r23,`$FRAME-$SIZE_T*9`($sp)
+	$PUSH	r24,`$FRAME-$SIZE_T*8`($sp)
+	$PUSH	r25,`$FRAME-$SIZE_T*7`($sp)
+	$PUSH	r26,`$FRAME-$SIZE_T*6`($sp)
+	$PUSH	r27,`$FRAME-$SIZE_T*5`($sp)
+	$PUSH	r28,`$FRAME-$SIZE_T*4`($sp)
+	$PUSH	r29,`$FRAME-$SIZE_T*3`($sp)
+	$PUSH	r30,`$FRAME-$SIZE_T*2`($sp)
+	$PUSH	r31,`$FRAME-$SIZE_T*1`($sp)
+	$PUSH	r0,`$FRAME+$LRSAVE`($sp)
+
+	lwz	@d[0],0($ctr)			# load counter
+	lwz	@d[1],4($ctr)
+	lwz	@d[2],8($ctr)
+	lwz	@d[3],12($ctr)
+
+	bl	__ChaCha20_1x
+
+	$POP	r0,`$FRAME+$LRSAVE`($sp)
+	$POP	r14,`$FRAME-$SIZE_T*18`($sp)
+	$POP	r15,`$FRAME-$SIZE_T*17`($sp)
+	$POP	r16,`$FRAME-$SIZE_T*16`($sp)
+	$POP	r17,`$FRAME-$SIZE_T*15`($sp)
+	$POP	r18,`$FRAME-$SIZE_T*14`($sp)
+	$POP	r19,`$FRAME-$SIZE_T*13`($sp)
+	$POP	r20,`$FRAME-$SIZE_T*12`($sp)
+	$POP	r21,`$FRAME-$SIZE_T*11`($sp)
+	$POP	r22,`$FRAME-$SIZE_T*10`($sp)
+	$POP	r23,`$FRAME-$SIZE_T*9`($sp)
+	$POP	r24,`$FRAME-$SIZE_T*8`($sp)
+	$POP	r25,`$FRAME-$SIZE_T*7`($sp)
+	$POP	r26,`$FRAME-$SIZE_T*6`($sp)
+	$POP	r27,`$FRAME-$SIZE_T*5`($sp)
+	$POP	r28,`$FRAME-$SIZE_T*4`($sp)
+	$POP	r29,`$FRAME-$SIZE_T*3`($sp)
+	$POP	r30,`$FRAME-$SIZE_T*2`($sp)
+	$POP	r31,`$FRAME-$SIZE_T*1`($sp)
+	mtlr	r0
+	addi	$sp,$sp,$FRAME
+	blr
+	.long	0
+	.byte	0,12,4,1,0x80,18,5,0
+	.long	0
+.size	.ChaCha20_ctr32_int,.-.ChaCha20_ctr32_int
+
+.align	5
+__ChaCha20_1x:
+Loop_outer:
+	lis	@x[0],0x6170			# synthesize sigma
+	lis	@x[1],0x3320
+	lis	@x[2],0x7962
+	lis	@x[3],0x6b20
+	ori	@x[0],@x[0],0x7865
+	ori	@x[1],@x[1],0x646e
+	ori	@x[2],@x[2],0x2d32
+	ori	@x[3],@x[3],0x6574
+
+	li	r0,10				# inner loop counter
+	lwz	@x[4],0($key)			# load key
+	lwz	@x[5],4($key)
+	lwz	@x[6],8($key)
+	lwz	@x[7],12($key)
+	lwz	@x[8],16($key)
+	mr	@x[12],@d[0]			# copy counter
+	lwz	@x[9],20($key)
+	mr	@x[13],@d[1]
+	lwz	@x[10],24($key)
+	mr	@x[14],@d[2]
+	lwz	@x[11],28($key)
+	mr	@x[15],@d[3]
+
+	mr	@t[0],@x[4]
+	mr	@t[1],@x[5]
+	mr	@t[2],@x[6]
+	mr	@t[3],@x[7]
+
+	mtctr	r0
+Loop:
+___
+	foreach (&ROUND(0, 4, 8,12)) { eval; }
+	foreach (&ROUND(0, 5,10,15)) { eval; }
+$code.=<<___;
+	bdnz	Loop
+
+	subic	$len,$len,64			# $len-=64
+	addi	@x[0],@x[0],0x7865		# accumulate key block
+	addi	@x[1],@x[1],0x646e
+	addi	@x[2],@x[2],0x2d32
+	addi	@x[3],@x[3],0x6574
+	addis	@x[0],@x[0],0x6170
+	addis	@x[1],@x[1],0x3320
+	addis	@x[2],@x[2],0x7962
+	addis	@x[3],@x[3],0x6b20
+
+	subfe.	r0,r0,r0			# borrow?-1:0
+	add	@x[4],@x[4],@t[0]
+	lwz	@t[0],16($key)
+	add	@x[5],@x[5],@t[1]
+	lwz	@t[1],20($key)
+	add	@x[6],@x[6],@t[2]
+	lwz	@t[2],24($key)
+	add	@x[7],@x[7],@t[3]
+	lwz	@t[3],28($key)
+	add	@x[8],@x[8],@t[0]
+	add	@x[9],@x[9],@t[1]
+	add	@x[10],@x[10],@t[2]
+	add	@x[11],@x[11],@t[3]
+
+	add	@x[12],@x[12],@d[0]
+	add	@x[13],@x[13],@d[1]
+	add	@x[14],@x[14],@d[2]
+	add	@x[15],@x[15],@d[3]
+	addi	@d[0],@d[0],1			# increment counter
+___
+if (!$LITTLE_ENDIAN) { for($i=0;$i<16;$i++) {	# flip byte order
+$code.=<<___;
+	mr	@t[$i&3],@x[$i]
+	rotlwi	@x[$i],@x[$i],8
+	rlwimi	@x[$i],@t[$i&3],24,0,7
+	rlwimi	@x[$i],@t[$i&3],24,16,23
+___
+} }
+$code.=<<___;
+	bne	Ltail				# $len-=64 borrowed
+
+	lwz	@t[0],0($inp)			# load input, aligned or not
+	lwz	@t[1],4($inp)
+	${UCMP}i $len,0				# done already?
+	lwz	@t[2],8($inp)
+	lwz	@t[3],12($inp)
+	xor	@x[0],@x[0],@t[0]		# xor with input
+	lwz	@t[0],16($inp)
+	xor	@x[1],@x[1],@t[1]
+	lwz	@t[1],20($inp)
+	xor	@x[2],@x[2],@t[2]
+	lwz	@t[2],24($inp)
+	xor	@x[3],@x[3],@t[3]
+	lwz	@t[3],28($inp)
+	xor	@x[4],@x[4],@t[0]
+	lwz	@t[0],32($inp)
+	xor	@x[5],@x[5],@t[1]
+	lwz	@t[1],36($inp)
+	xor	@x[6],@x[6],@t[2]
+	lwz	@t[2],40($inp)
+	xor	@x[7],@x[7],@t[3]
+	lwz	@t[3],44($inp)
+	xor	@x[8],@x[8],@t[0]
+	lwz	@t[0],48($inp)
+	xor	@x[9],@x[9],@t[1]
+	lwz	@t[1],52($inp)
+	xor	@x[10],@x[10],@t[2]
+	lwz	@t[2],56($inp)
+	xor	@x[11],@x[11],@t[3]
+	lwz	@t[3],60($inp)
+	xor	@x[12],@x[12],@t[0]
+	stw	@x[0],0($out)			# store output, aligned or not
+	xor	@x[13],@x[13],@t[1]
+	stw	@x[1],4($out)
+	xor	@x[14],@x[14],@t[2]
+	stw	@x[2],8($out)
+	xor	@x[15],@x[15],@t[3]
+	stw	@x[3],12($out)
+	stw	@x[4],16($out)
+	stw	@x[5],20($out)
+	stw	@x[6],24($out)
+	stw	@x[7],28($out)
+	stw	@x[8],32($out)
+	stw	@x[9],36($out)
+	stw	@x[10],40($out)
+	stw	@x[11],44($out)
+	stw	@x[12],48($out)
+	stw	@x[13],52($out)
+	stw	@x[14],56($out)
+	addi	$inp,$inp,64
+	stw	@x[15],60($out)
+	addi	$out,$out,64
+
+	bne	Loop_outer
+
+	blr
+
+.align	4
+Ltail:
+	addi	$len,$len,64			# restore tail length
+	subi	$inp,$inp,1			# prepare for *++ptr
+	subi	$out,$out,1
+	addi	@t[0],$sp,$LOCALS-1
+	mtctr	$len
+
+	stw	@x[0],`$LOCALS+0`($sp)		# save whole block to stack
+	stw	@x[1],`$LOCALS+4`($sp)
+	stw	@x[2],`$LOCALS+8`($sp)
+	stw	@x[3],`$LOCALS+12`($sp)
+	stw	@x[4],`$LOCALS+16`($sp)
+	stw	@x[5],`$LOCALS+20`($sp)
+	stw	@x[6],`$LOCALS+24`($sp)
+	stw	@x[7],`$LOCALS+28`($sp)
+	stw	@x[8],`$LOCALS+32`($sp)
+	stw	@x[9],`$LOCALS+36`($sp)
+	stw	@x[10],`$LOCALS+40`($sp)
+	stw	@x[11],`$LOCALS+44`($sp)
+	stw	@x[12],`$LOCALS+48`($sp)
+	stw	@x[13],`$LOCALS+52`($sp)
+	stw	@x[14],`$LOCALS+56`($sp)
+	stw	@x[15],`$LOCALS+60`($sp)
+
+Loop_tail:					# byte-by-byte loop
+	lbzu	@d[0],1($inp)
+	lbzu	@x[0],1(@t[0])
+	xor	@d[1],@d[0],@x[0]
+	stbu	@d[1],1($out)
+	bdnz	Loop_tail
+
+	stw	$sp,`$LOCALS+0`($sp)		# wipe block on stack
+	stw	$sp,`$LOCALS+4`($sp)
+	stw	$sp,`$LOCALS+8`($sp)
+	stw	$sp,`$LOCALS+12`($sp)
+	stw	$sp,`$LOCALS+16`($sp)
+	stw	$sp,`$LOCALS+20`($sp)
+	stw	$sp,`$LOCALS+24`($sp)
+	stw	$sp,`$LOCALS+28`($sp)
+	stw	$sp,`$LOCALS+32`($sp)
+	stw	$sp,`$LOCALS+36`($sp)
+	stw	$sp,`$LOCALS+40`($sp)
+	stw	$sp,`$LOCALS+44`($sp)
+	stw	$sp,`$LOCALS+48`($sp)
+	stw	$sp,`$LOCALS+52`($sp)
+	stw	$sp,`$LOCALS+56`($sp)
+	stw	$sp,`$LOCALS+60`($sp)
+
+	blr
+	.long	0
+	.byte	0,12,0x14,0,0,0,0,0
+___
+
+{{{
+my ($A0,$B0,$C0,$D0,$A1,$B1,$C1,$D1,$A2,$B2,$C2,$D2)
+				= map("v$_",(0..11));
+my @K				= map("v$_",(12..17));
+my ($FOUR,$sixteen,$twenty4)	= map("v$_",(18..19,23));
+my ($inpperm,$outperm,$outmask)	= map("v$_",(24..26));
+my @D				= map("v$_",(27..31));
+my ($twelve,$seven,$T0,$T1) = @D;
+
+my $FRAME=$LOCALS+64+10*16+18*$SIZE_T;	# 10*16 is for v23-v31 offload
+
+sub VMXROUND {
+my $odd = pop;
+my ($a,$b,$c,$d)=@_;
+
+	(
+	"&vadduwm	('$a','$a','$b')",
+	"&vxor		('$d','$d','$a')",
+	"&vperm		('$d','$d','$d','$sixteen')",
+
+	"&vadduwm	('$c','$c','$d')",
+	"&vxor		('$b','$b','$c')",
+	"&vrlw		('$b','$b','$twelve')",
+
+	"&vadduwm	('$a','$a','$b')",
+	"&vxor		('$d','$d','$a')",
+	"&vperm		('$d','$d','$d','$twenty4')",
+
+	"&vadduwm	('$c','$c','$d')",
+	"&vxor		('$b','$b','$c')",
+	"&vrlw		('$b','$b','$seven')",
+
+	"&vrldoi	('$c','$c',8)",
+	"&vrldoi	('$b','$b',$odd?4:12)",
+	"&vrldoi	('$d','$d',$odd?12:4)"
+	);
+}
+
+$code.=<<___;
+
+.globl	.ChaCha20_ctr32_vmx
+.align	5
+.ChaCha20_ctr32_vmx:
+	${UCMP}i $len,256
+	blt	__ChaCha20_ctr32_int
+
+	$STU	$sp,-$FRAME($sp)
+	mflr	r0
+	li	r10,`15+$LOCALS+64`
+	li	r11,`31+$LOCALS+64`
+	mfspr	r12,256
+	stvx	v23,r10,$sp
+	addi	r10,r10,32
+	stvx	v24,r11,$sp
+	addi	r11,r11,32
+	stvx	v25,r10,$sp
+	addi	r10,r10,32
+	stvx	v26,r11,$sp
+	addi	r11,r11,32
+	stvx	v27,r10,$sp
+	addi	r10,r10,32
+	stvx	v28,r11,$sp
+	addi	r11,r11,32
+	stvx	v29,r10,$sp
+	addi	r10,r10,32
+	stvx	v30,r11,$sp
+	stvx	v31,r10,$sp
+	stw	r12,`$FRAME-$SIZE_T*18-4`($sp)	# save vrsave
+	$PUSH	r14,`$FRAME-$SIZE_T*18`($sp)
+	$PUSH	r15,`$FRAME-$SIZE_T*17`($sp)
+	$PUSH	r16,`$FRAME-$SIZE_T*16`($sp)
+	$PUSH	r17,`$FRAME-$SIZE_T*15`($sp)
+	$PUSH	r18,`$FRAME-$SIZE_T*14`($sp)
+	$PUSH	r19,`$FRAME-$SIZE_T*13`($sp)
+	$PUSH	r20,`$FRAME-$SIZE_T*12`($sp)
+	$PUSH	r21,`$FRAME-$SIZE_T*11`($sp)
+	$PUSH	r22,`$FRAME-$SIZE_T*10`($sp)
+	$PUSH	r23,`$FRAME-$SIZE_T*9`($sp)
+	$PUSH	r24,`$FRAME-$SIZE_T*8`($sp)
+	$PUSH	r25,`$FRAME-$SIZE_T*7`($sp)
+	$PUSH	r26,`$FRAME-$SIZE_T*6`($sp)
+	$PUSH	r27,`$FRAME-$SIZE_T*5`($sp)
+	$PUSH	r28,`$FRAME-$SIZE_T*4`($sp)
+	$PUSH	r29,`$FRAME-$SIZE_T*3`($sp)
+	$PUSH	r30,`$FRAME-$SIZE_T*2`($sp)
+	$PUSH	r31,`$FRAME-$SIZE_T*1`($sp)
+	li	r12,-4096+511
+	$PUSH	r0, `$FRAME+$LRSAVE`($sp)
+	mtspr	256,r12				# preserve 29 AltiVec registers
+
+	bl	Lconsts				# returns pointer Lsigma in r12
+	li	@x[0],16
+	li	@x[1],32
+	li	@x[2],48
+	li	@x[3],64
+	li	@x[4],31			# 31 is not a typo
+	li	@x[5],15			# nor is 15
+
+	lvx	@K[1],0,$key			# load key
+	?lvsr	$T0,0,$key			# prepare unaligned load
+	lvx	@K[2],@x[0],$key
+	lvx	@D[0],@x[4],$key
+
+	lvx	@K[3],0,$ctr			# load counter
+	?lvsr	$T1,0,$ctr			# prepare unaligned load
+	lvx	@D[1],@x[5],$ctr
+
+	lvx	@K[0],0,r12			# load constants
+	lvx	@K[5],@x[0],r12			# one
+	lvx	$FOUR,@x[1],r12
+	lvx	$sixteen,@x[2],r12
+	lvx	$twenty4,@x[3],r12
+
+	?vperm	@K[1],@K[2],@K[1],$T0		# align key
+	?vperm	@K[2],@D[0],@K[2],$T0
+	?vperm	@K[3],@D[1],@K[3],$T1		# align counter
+
+	lwz	@d[0],0($ctr)			# load counter to GPR
+	lwz	@d[1],4($ctr)
+	vadduwm	@K[3],@K[3],@K[5]		# adjust AltiVec counter
+	lwz	@d[2],8($ctr)
+	vadduwm	@K[4],@K[3],@K[5]
+	lwz	@d[3],12($ctr)
+	vadduwm	@K[5],@K[4],@K[5]
+
+	vxor	$T0,$T0,$T0			# 0x00..00
+	vspltisw $outmask,-1			# 0xff..ff
+	?lvsr	$inpperm,0,$inp			# prepare for unaligned load
+	?lvsl	$outperm,0,$out			# prepare for unaligned store
+	?vperm	$outmask,$outmask,$T0,$outperm
+
+	be?lvsl	$T0,0,@x[0]			# 0x00..0f
+	be?vspltisb $T1,3			# 0x03..03
+	be?vxor	$T0,$T0,$T1			# swap bytes within words
+	be?vxor	$outperm,$outperm,$T1
+	be?vperm $inpperm,$inpperm,$inpperm,$T0
+
+	li	r0,10				# inner loop counter
+	b	Loop_outer_vmx
+
+.align	4
+Loop_outer_vmx:
+	lis	@x[0],0x6170			# synthesize sigma
+	lis	@x[1],0x3320
+	 vmr	$A0,@K[0]
+	lis	@x[2],0x7962
+	lis	@x[3],0x6b20
+	 vmr	$A1,@K[0]
+	ori	@x[0],@x[0],0x7865
+	ori	@x[1],@x[1],0x646e
+	 vmr	$A2,@K[0]
+	ori	@x[2],@x[2],0x2d32
+	ori	@x[3],@x[3],0x6574
+	 vmr	$B0,@K[1]
+
+	lwz	@x[4],0($key)			# load key to GPR
+	 vmr	$B1,@K[1]
+	lwz	@x[5],4($key)
+	 vmr	$B2,@K[1]
+	lwz	@x[6],8($key)
+	 vmr	$C0,@K[2]
+	lwz	@x[7],12($key)
+	 vmr	$C1,@K[2]
+	lwz	@x[8],16($key)
+	 vmr	$C2,@K[2]
+	mr	@x[12],@d[0]			# copy GPR counter
+	lwz	@x[9],20($key)
+	 vmr	$D0,@K[3]
+	mr	@x[13],@d[1]
+	lwz	@x[10],24($key)
+	 vmr	$D1,@K[4]
+	mr	@x[14],@d[2]
+	lwz	@x[11],28($key)
+	 vmr	$D2,@K[5]
+	mr	@x[15],@d[3]
+
+	mr	@t[0],@x[4]
+	mr	@t[1],@x[5]
+	mr	@t[2],@x[6]
+	mr	@t[3],@x[7]
+
+	vspltisw $twelve,12			# synthesize constants
+	vspltisw $seven,7
+
+	mtctr	r0
+	nop
+Loop_vmx:
+___
+	my @thread0=&VMXROUND($A0,$B0,$C0,$D0,0);
+	my @thread1=&VMXROUND($A1,$B1,$C1,$D1,0);
+	my @thread2=&VMXROUND($A2,$B2,$C2,$D2,0);
+	my @thread3=&ROUND(0,4,8,12);
+
+	foreach (@thread0) {
+		eval;
+		eval(shift(@thread1));
+		eval(shift(@thread2));
+
+		eval(shift(@thread3));
+		eval(shift(@thread3));
+		eval(shift(@thread3));
+	}
+	foreach (@thread3) { eval; }
+
+	@thread0=&VMXROUND($A0,$B0,$C0,$D0,1);
+	@thread1=&VMXROUND($A1,$B1,$C1,$D1,1);
+	@thread2=&VMXROUND($A2,$B2,$C2,$D2,1);
+	@thread3=&ROUND(0,5,10,15);
+
+	foreach (@thread0) {
+		eval;
+		eval(shift(@thread1));
+		eval(shift(@thread2));
+
+		eval(shift(@thread3));
+		eval(shift(@thread3));
+		eval(shift(@thread3));
+	}
+	foreach (@thread3) { eval; }
+$code.=<<___;
+	bdnz	Loop_vmx
+
+	subi	$len,$len,256			# $len-=256
+	addi	@x[0],@x[0],0x7865		# accumulate key block
+	addi	@x[1],@x[1],0x646e
+	addi	@x[2],@x[2],0x2d32
+	addi	@x[3],@x[3],0x6574
+	addis	@x[0],@x[0],0x6170
+	addis	@x[1],@x[1],0x3320
+	addis	@x[2],@x[2],0x7962
+	addis	@x[3],@x[3],0x6b20
+	add	@x[4],@x[4],@t[0]
+	lwz	@t[0],16($key)
+	add	@x[5],@x[5],@t[1]
+	lwz	@t[1],20($key)
+	add	@x[6],@x[6],@t[2]
+	lwz	@t[2],24($key)
+	add	@x[7],@x[7],@t[3]
+	lwz	@t[3],28($key)
+	add	@x[8],@x[8],@t[0]
+	add	@x[9],@x[9],@t[1]
+	add	@x[10],@x[10],@t[2]
+	add	@x[11],@x[11],@t[3]
+	add	@x[12],@x[12],@d[0]
+	add	@x[13],@x[13],@d[1]
+	add	@x[14],@x[14],@d[2]
+	add	@x[15],@x[15],@d[3]
+
+	vadduwm	$A0,$A0,@K[0]			# accumulate key block
+	vadduwm	$A1,$A1,@K[0]
+	vadduwm	$A2,$A2,@K[0]
+	vadduwm	$B0,$B0,@K[1]
+	vadduwm	$B1,$B1,@K[1]
+	vadduwm	$B2,$B2,@K[1]
+	vadduwm	$C0,$C0,@K[2]
+	vadduwm	$C1,$C1,@K[2]
+	vadduwm	$C2,$C2,@K[2]
+	vadduwm	$D0,$D0,@K[3]
+	vadduwm	$D1,$D1,@K[4]
+	vadduwm	$D2,$D2,@K[5]
+
+	addi	@d[0],@d[0],4			# increment counter
+	vadduwm	@K[3],@K[3],$FOUR
+	vadduwm	@K[4],@K[4],$FOUR
+	vadduwm	@K[5],@K[5],$FOUR
+
+___
+if (!$LITTLE_ENDIAN) { for($i=0;$i<16;$i++) {	# flip byte order
+$code.=<<___;
+	mr	@t[$i&3],@x[$i]
+	rotlwi	@x[$i],@x[$i],8
+	rlwimi	@x[$i],@t[$i&3],24,0,7
+	rlwimi	@x[$i],@t[$i&3],24,16,23
+___
+} }
+$code.=<<___;
+	lwz	@t[0],0($inp)			# load input, aligned or not
+	lwz	@t[1],4($inp)
+	lwz	@t[2],8($inp)
+	lwz	@t[3],12($inp)
+	xor	@x[0],@x[0],@t[0]		# xor with input
+	lwz	@t[0],16($inp)
+	xor	@x[1],@x[1],@t[1]
+	lwz	@t[1],20($inp)
+	xor	@x[2],@x[2],@t[2]
+	lwz	@t[2],24($inp)
+	xor	@x[3],@x[3],@t[3]
+	lwz	@t[3],28($inp)
+	xor	@x[4],@x[4],@t[0]
+	lwz	@t[0],32($inp)
+	xor	@x[5],@x[5],@t[1]
+	lwz	@t[1],36($inp)
+	xor	@x[6],@x[6],@t[2]
+	lwz	@t[2],40($inp)
+	xor	@x[7],@x[7],@t[3]
+	lwz	@t[3],44($inp)
+	xor	@x[8],@x[8],@t[0]
+	lwz	@t[0],48($inp)
+	xor	@x[9],@x[9],@t[1]
+	lwz	@t[1],52($inp)
+	xor	@x[10],@x[10],@t[2]
+	lwz	@t[2],56($inp)
+	xor	@x[11],@x[11],@t[3]
+	lwz	@t[3],60($inp)
+	xor	@x[12],@x[12],@t[0]
+	stw	@x[0],0($out)			# store output, aligned or not
+	xor	@x[13],@x[13],@t[1]
+	stw	@x[1],4($out)
+	xor	@x[14],@x[14],@t[2]
+	stw	@x[2],8($out)
+	xor	@x[15],@x[15],@t[3]
+	stw	@x[3],12($out)
+	addi	$inp,$inp,64
+	stw	@x[4],16($out)
+	li	@t[0],16
+	stw	@x[5],20($out)
+	li	@t[1],32
+	stw	@x[6],24($out)
+	li	@t[2],48
+	stw	@x[7],28($out)
+	li	@t[3],64
+	stw	@x[8],32($out)
+	stw	@x[9],36($out)
+	stw	@x[10],40($out)
+	stw	@x[11],44($out)
+	stw	@x[12],48($out)
+	stw	@x[13],52($out)
+	stw	@x[14],56($out)
+	stw	@x[15],60($out)
+	addi	$out,$out,64
+
+	lvx	@D[0],0,$inp			# load input
+	lvx	@D[1],@t[0],$inp
+	lvx	@D[2],@t[1],$inp
+	lvx	@D[3],@t[2],$inp
+	lvx	@D[4],@t[3],$inp
+	addi	$inp,$inp,64
+
+	?vperm	@D[0],@D[1],@D[0],$inpperm	# align input
+	?vperm	@D[1],@D[2],@D[1],$inpperm
+	?vperm	@D[2],@D[3],@D[2],$inpperm
+	?vperm	@D[3],@D[4],@D[3],$inpperm
+	vxor	$A0,$A0,@D[0]			# xor with input
+	vxor	$B0,$B0,@D[1]
+	lvx	@D[1],@t[0],$inp		# keep loading input
+	vxor	$C0,$C0,@D[2]
+	lvx	@D[2],@t[1],$inp
+	vxor	$D0,$D0,@D[3]
+	lvx	@D[3],@t[2],$inp
+	lvx	@D[0],@t[3],$inp
+	addi	$inp,$inp,64
+	li	@t[3],63			# 63 is not a typo
+	vperm	$A0,$A0,$A0,$outperm		# pre-misalign output
+	vperm	$B0,$B0,$B0,$outperm
+	vperm	$C0,$C0,$C0,$outperm
+	vperm	$D0,$D0,$D0,$outperm
+
+	?vperm	@D[4],@D[1],@D[4],$inpperm	# align input
+	?vperm	@D[1],@D[2],@D[1],$inpperm
+	?vperm	@D[2],@D[3],@D[2],$inpperm
+	?vperm	@D[3],@D[0],@D[3],$inpperm
+	vxor	$A1,$A1,@D[4]
+	vxor	$B1,$B1,@D[1]
+	lvx	@D[1],@t[0],$inp		# keep loading input
+	vxor	$C1,$C1,@D[2]
+	lvx	@D[2],@t[1],$inp
+	vxor	$D1,$D1,@D[3]
+	lvx	@D[3],@t[2],$inp
+	lvx	@D[4],@t[3],$inp		# redundant in aligned case
+	addi	$inp,$inp,64
+	vperm	$A1,$A1,$A1,$outperm		# pre-misalign output
+	vperm	$B1,$B1,$B1,$outperm
+	vperm	$C1,$C1,$C1,$outperm
+	vperm	$D1,$D1,$D1,$outperm
+
+	?vperm	@D[0],@D[1],@D[0],$inpperm	# align input
+	?vperm	@D[1],@D[2],@D[1],$inpperm
+	?vperm	@D[2],@D[3],@D[2],$inpperm
+	?vperm	@D[3],@D[4],@D[3],$inpperm
+	vxor	$A2,$A2,@D[0]
+	vxor	$B2,$B2,@D[1]
+	vxor	$C2,$C2,@D[2]
+	vxor	$D2,$D2,@D[3]
+	vperm	$A2,$A2,$A2,$outperm		# pre-misalign output
+	vperm	$B2,$B2,$B2,$outperm
+	vperm	$C2,$C2,$C2,$outperm
+	vperm	$D2,$D2,$D2,$outperm
+
+	andi.	@x[1],$out,15			# is $out aligned?
+	mr	@x[0],$out
+
+	vsel	@D[0],$A0,$B0,$outmask		# collect pre-misaligned output
+	vsel	@D[1],$B0,$C0,$outmask
+	vsel	@D[2],$C0,$D0,$outmask
+	vsel	@D[3],$D0,$A1,$outmask
+	vsel	$B0,$A1,$B1,$outmask
+	vsel	$C0,$B1,$C1,$outmask
+	vsel	$D0,$C1,$D1,$outmask
+	vsel	$A1,$D1,$A2,$outmask
+	vsel	$B1,$A2,$B2,$outmask
+	vsel	$C1,$B2,$C2,$outmask
+	vsel	$D1,$C2,$D2,$outmask
+
+	#stvx	$A0,0,$out			# take it easy on the edges
+	stvx	@D[0],@t[0],$out		# store output
+	stvx	@D[1],@t[1],$out
+	stvx	@D[2],@t[2],$out
+	addi	$out,$out,64
+	stvx	@D[3],0,$out
+	stvx	$B0,@t[0],$out
+	stvx	$C0,@t[1],$out
+	stvx	$D0,@t[2],$out
+	addi	$out,$out,64
+	stvx	$A1,0,$out
+	stvx	$B1,@t[0],$out
+	stvx	$C1,@t[1],$out
+	stvx	$D1,@t[2],$out
+	addi	$out,$out,64
+
+	beq	Laligned_vmx
+
+	sub	@x[2],$out,@x[1]		# in misaligned case edges
+	li	@x[3],0				# are written byte-by-byte
+Lunaligned_tail_vmx:
+	stvebx	$D2,@x[3],@x[2]
+	addi	@x[3],@x[3],1
+	cmpw	@x[3],@x[1]
+	bne	Lunaligned_tail_vmx
+
+	sub	@x[2],@x[0],@x[1]
+Lunaligned_head_vmx:
+	stvebx	$A0,@x[1],@x[2]
+	cmpwi	@x[1],15
+	addi	@x[1],@x[1],1
+	bne	Lunaligned_head_vmx
+
+	${UCMP}i $len,255			# done with 256-byte blocks yet?
+	bgt	Loop_outer_vmx
+
+	b	Ldone_vmx
+
+.align	4
+Laligned_vmx:
+	stvx	$A0,0,@x[0]			# head hexaword was not stored
+
+	${UCMP}i $len,255			# done with 256-byte blocks yet?
+	bgt	Loop_outer_vmx
+	nop
+
+Ldone_vmx:
+	${UCMP}i $len,0				# done yet?
+	bnel	__ChaCha20_1x
+
+	lwz	r12,`$FRAME-$SIZE_T*18-4`($sp)	# pull vrsave
+	li	r10,`15+$LOCALS+64`
+	li	r11,`31+$LOCALS+64`
+	mtspr	256,r12				# restore vrsave
+	lvx	v23,r10,$sp
+	addi	r10,r10,32
+	lvx	v24,r11,$sp
+	addi	r11,r11,32
+	lvx	v25,r10,$sp
+	addi	r10,r10,32
+	lvx	v26,r11,$sp
+	addi	r11,r11,32
+	lvx	v27,r10,$sp
+	addi	r10,r10,32
+	lvx	v28,r11,$sp
+	addi	r11,r11,32
+	lvx	v29,r10,$sp
+	addi	r10,r10,32
+	lvx	v30,r11,$sp
+	lvx	v31,r10,$sp
+	$POP	r0, `$FRAME+$LRSAVE`($sp)
+	$POP	r14,`$FRAME-$SIZE_T*18`($sp)
+	$POP	r15,`$FRAME-$SIZE_T*17`($sp)
+	$POP	r16,`$FRAME-$SIZE_T*16`($sp)
+	$POP	r17,`$FRAME-$SIZE_T*15`($sp)
+	$POP	r18,`$FRAME-$SIZE_T*14`($sp)
+	$POP	r19,`$FRAME-$SIZE_T*13`($sp)
+	$POP	r20,`$FRAME-$SIZE_T*12`($sp)
+	$POP	r21,`$FRAME-$SIZE_T*11`($sp)
+	$POP	r22,`$FRAME-$SIZE_T*10`($sp)
+	$POP	r23,`$FRAME-$SIZE_T*9`($sp)
+	$POP	r24,`$FRAME-$SIZE_T*8`($sp)
+	$POP	r25,`$FRAME-$SIZE_T*7`($sp)
+	$POP	r26,`$FRAME-$SIZE_T*6`($sp)
+	$POP	r27,`$FRAME-$SIZE_T*5`($sp)
+	$POP	r28,`$FRAME-$SIZE_T*4`($sp)
+	$POP	r29,`$FRAME-$SIZE_T*3`($sp)
+	$POP	r30,`$FRAME-$SIZE_T*2`($sp)
+	$POP	r31,`$FRAME-$SIZE_T*1`($sp)
+	mtlr	r0
+	addi	$sp,$sp,$FRAME
+	blr
+	.long	0
+	.byte	0,12,0x04,1,0x80,18,5,0
+	.long	0
+.size	.ChaCha20_ctr32_vmx,.-.ChaCha20_ctr32_vmx
+___
+}}}
+{{{
+my ($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3,
+    $xc0,$xc1,$xc2,$xc3, $xd0,$xd1,$xd2,$xd3) = map("v$_",(0..15));
+my @K = map("v$_",(16..19));
+my $CTR = "v26";
+my ($xt0,$xt1,$xt2,$xt3) = map("v$_",(27..30));
+my ($sixteen,$twelve,$eight,$seven) = ($xt0,$xt1,$xt2,$xt3);
+my $beperm = "v31";
+
+my ($x00,$x10,$x20,$x30) = (0, map("r$_",(8..10)));
+
+my $FRAME=$LOCALS+64+7*16;	# 7*16 is for v26-v31 offload
+
+sub VSX_lane_ROUND {
+my ($a0,$b0,$c0,$d0)=@_;
+my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
+my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
+my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
+my @x=map("\"v$_\"",(0..15));
+
+	(
+	"&vadduwm	(@x[$a0],@x[$a0],@x[$b0])",	# Q1
+	 "&vadduwm	(@x[$a1],@x[$a1],@x[$b1])",	# Q2
+	  "&vadduwm	(@x[$a2],@x[$a2],@x[$b2])",	# Q3
+	   "&vadduwm	(@x[$a3],@x[$a3],@x[$b3])",	# Q4
+	"&vxor		(@x[$d0],@x[$d0],@x[$a0])",
+	 "&vxor		(@x[$d1],@x[$d1],@x[$a1])",
+	  "&vxor	(@x[$d2],@x[$d2],@x[$a2])",
+	   "&vxor	(@x[$d3],@x[$d3],@x[$a3])",
+	"&vrlw		(@x[$d0],@x[$d0],'$sixteen')",
+	 "&vrlw		(@x[$d1],@x[$d1],'$sixteen')",
+	  "&vrlw	(@x[$d2],@x[$d2],'$sixteen')",
+	   "&vrlw	(@x[$d3],@x[$d3],'$sixteen')",
+
+	"&vadduwm	(@x[$c0],@x[$c0],@x[$d0])",
+	 "&vadduwm	(@x[$c1],@x[$c1],@x[$d1])",
+	  "&vadduwm	(@x[$c2],@x[$c2],@x[$d2])",
+	   "&vadduwm	(@x[$c3],@x[$c3],@x[$d3])",
+	"&vxor		(@x[$b0],@x[$b0],@x[$c0])",
+	 "&vxor		(@x[$b1],@x[$b1],@x[$c1])",
+	  "&vxor	(@x[$b2],@x[$b2],@x[$c2])",
+	   "&vxor	(@x[$b3],@x[$b3],@x[$c3])",
+	"&vrlw		(@x[$b0],@x[$b0],'$twelve')",
+	 "&vrlw		(@x[$b1],@x[$b1],'$twelve')",
+	  "&vrlw	(@x[$b2],@x[$b2],'$twelve')",
+	   "&vrlw	(@x[$b3],@x[$b3],'$twelve')",
+
+	"&vadduwm	(@x[$a0],@x[$a0],@x[$b0])",
+	 "&vadduwm	(@x[$a1],@x[$a1],@x[$b1])",
+	  "&vadduwm	(@x[$a2],@x[$a2],@x[$b2])",
+	   "&vadduwm	(@x[$a3],@x[$a3],@x[$b3])",
+	"&vxor		(@x[$d0],@x[$d0],@x[$a0])",
+	 "&vxor		(@x[$d1],@x[$d1],@x[$a1])",
+	  "&vxor	(@x[$d2],@x[$d2],@x[$a2])",
+	   "&vxor	(@x[$d3],@x[$d3],@x[$a3])",
+	"&vrlw		(@x[$d0],@x[$d0],'$eight')",
+	 "&vrlw		(@x[$d1],@x[$d1],'$eight')",
+	  "&vrlw	(@x[$d2],@x[$d2],'$eight')",
+	   "&vrlw	(@x[$d3],@x[$d3],'$eight')",
+
+	"&vadduwm	(@x[$c0],@x[$c0],@x[$d0])",
+	 "&vadduwm	(@x[$c1],@x[$c1],@x[$d1])",
+	  "&vadduwm	(@x[$c2],@x[$c2],@x[$d2])",
+	   "&vadduwm	(@x[$c3],@x[$c3],@x[$d3])",
+	"&vxor		(@x[$b0],@x[$b0],@x[$c0])",
+	 "&vxor		(@x[$b1],@x[$b1],@x[$c1])",
+	  "&vxor	(@x[$b2],@x[$b2],@x[$c2])",
+	   "&vxor	(@x[$b3],@x[$b3],@x[$c3])",
+	"&vrlw		(@x[$b0],@x[$b0],'$seven')",
+	 "&vrlw		(@x[$b1],@x[$b1],'$seven')",
+	  "&vrlw	(@x[$b2],@x[$b2],'$seven')",
+	   "&vrlw	(@x[$b3],@x[$b3],'$seven')"
+	);
+}
+
+$code.=<<___;
+
+.globl	.ChaCha20_ctr32_vsx
+.align	5
+.ChaCha20_ctr32_vsx:
+	$STU	$sp,-$FRAME($sp)
+	mflr	r0
+	li	r10,`15+$LOCALS+64`
+	li	r11,`31+$LOCALS+64`
+	mfspr	r12,256
+	stvx	v26,r10,$sp
+	addi	r10,r10,32
+	stvx	v27,r11,$sp
+	addi	r11,r11,32
+	stvx	v28,r10,$sp
+	addi	r10,r10,32
+	stvx	v29,r11,$sp
+	addi	r11,r11,32
+	stvx	v30,r10,$sp
+	stvx	v31,r11,$sp
+	stw	r12,`$FRAME-4`($sp)		# save vrsave
+	li	r12,-4096+63
+	$PUSH	r0, `$FRAME+$LRSAVE`($sp)
+	mtspr	256,r12				# preserve 29 AltiVec registers
+
+	bl	Lconsts				# returns pointer Lsigma in r12
+	lvx_4w	@K[0],0,r12			# load sigma
+	addi	r12,r12,0x50
+	li	$x10,16
+	li	$x20,32
+	li	$x30,48
+	li	r11,64
+
+	lvx_4w	@K[1],0,$key			# load key
+	lvx_4w	@K[2],$x10,$key
+	lvx_4w	@K[3],0,$ctr			# load counter
+
+	vxor	$xt0,$xt0,$xt0
+	lvx_4w	$xt1,r11,r12
+	vspltw	$CTR,@K[3],0
+	vsldoi	@K[3],@K[3],$xt0,4
+	vsldoi	@K[3],$xt0,@K[3],12		# clear @K[3].word[0]
+	vadduwm	$CTR,$CTR,$xt1
+
+	be?lvsl	$beperm,0,$x10			# 0x00..0f
+	be?vspltisb $xt0,3			# 0x03..03
+	be?vxor	$beperm,$beperm,$xt0		# swap bytes within words
+
+	li	r0,10				# inner loop counter
+	mtctr	r0
+	b	Loop_outer_vsx
+
+.align	5
+Loop_outer_vsx:
+	lvx	$xa0,$x00,r12			# load [smashed] sigma
+	lvx	$xa1,$x10,r12
+	lvx	$xa2,$x20,r12
+	lvx	$xa3,$x30,r12
+
+	vspltw	$xb0,@K[1],0			# smash the key
+	vspltw	$xb1,@K[1],1
+	vspltw	$xb2,@K[1],2
+	vspltw	$xb3,@K[1],3
+
+	vspltw	$xc0,@K[2],0
+	vspltw	$xc1,@K[2],1
+	vspltw	$xc2,@K[2],2
+	vspltw	$xc3,@K[2],3
+
+	vmr	$xd0,$CTR			# smash the counter
+	vspltw	$xd1,@K[3],1
+	vspltw	$xd2,@K[3],2
+	vspltw	$xd3,@K[3],3
+
+	vspltisw $sixteen,-16			# synthesize constants
+	vspltisw $twelve,12
+	vspltisw $eight,8
+	vspltisw $seven,7
+
+Loop_vsx:
+___
+	foreach (&VSX_lane_ROUND(0, 4, 8,12)) { eval; }
+	foreach (&VSX_lane_ROUND(0, 5,10,15)) { eval; }
+$code.=<<___;
+	bdnz	Loop_vsx
+
+	vadduwm	$xd0,$xd0,$CTR
+
+	vmrgew	$xt0,$xa0,$xa1			# transpose data
+	vmrgew	$xt1,$xa2,$xa3
+	vmrgow	$xa0,$xa0,$xa1
+	vmrgow	$xa2,$xa2,$xa3
+	 vmrgew	$xt2,$xb0,$xb1
+	 vmrgew	$xt3,$xb2,$xb3
+	vpermdi	$xa1,$xa0,$xa2,0b00
+	vpermdi	$xa3,$xa0,$xa2,0b11
+	vpermdi	$xa0,$xt0,$xt1,0b00
+	vpermdi	$xa2,$xt0,$xt1,0b11
+
+	vmrgow	$xb0,$xb0,$xb1
+	vmrgow	$xb2,$xb2,$xb3
+	 vmrgew	$xt0,$xc0,$xc1
+	 vmrgew	$xt1,$xc2,$xc3
+	vpermdi	$xb1,$xb0,$xb2,0b00
+	vpermdi	$xb3,$xb0,$xb2,0b11
+	vpermdi	$xb0,$xt2,$xt3,0b00
+	vpermdi	$xb2,$xt2,$xt3,0b11
+
+	vmrgow	$xc0,$xc0,$xc1
+	vmrgow	$xc2,$xc2,$xc3
+	 vmrgew	$xt2,$xd0,$xd1
+	 vmrgew	$xt3,$xd2,$xd3
+	vpermdi	$xc1,$xc0,$xc2,0b00
+	vpermdi	$xc3,$xc0,$xc2,0b11
+	vpermdi	$xc0,$xt0,$xt1,0b00
+	vpermdi	$xc2,$xt0,$xt1,0b11
+
+	vmrgow	$xd0,$xd0,$xd1
+	vmrgow	$xd2,$xd2,$xd3
+	 vspltisw $xt0,4
+	 vadduwm  $CTR,$CTR,$xt0		# next counter value
+	vpermdi	$xd1,$xd0,$xd2,0b00
+	vpermdi	$xd3,$xd0,$xd2,0b11
+	vpermdi	$xd0,$xt2,$xt3,0b00
+	vpermdi	$xd2,$xt2,$xt3,0b11
+
+	vadduwm	$xa0,$xa0,@K[0]
+	vadduwm	$xb0,$xb0,@K[1]
+	vadduwm	$xc0,$xc0,@K[2]
+	vadduwm	$xd0,$xd0,@K[3]
+
+	be?vperm $xa0,$xa0,$xa0,$beperm
+	be?vperm $xb0,$xb0,$xb0,$beperm
+	be?vperm $xc0,$xc0,$xc0,$beperm
+	be?vperm $xd0,$xd0,$xd0,$beperm
+
+	${UCMP}i $len,0x40
+	blt	Ltail_vsx
+
+	lvx_4w	$xt0,$x00,$inp
+	lvx_4w	$xt1,$x10,$inp
+	lvx_4w	$xt2,$x20,$inp
+	lvx_4w	$xt3,$x30,$inp
+
+	vxor	$xt0,$xt0,$xa0
+	vxor	$xt1,$xt1,$xb0
+	vxor	$xt2,$xt2,$xc0
+	vxor	$xt3,$xt3,$xd0
+
+	stvx_4w	$xt0,$x00,$out
+	stvx_4w	$xt1,$x10,$out
+	addi	$inp,$inp,0x40
+	stvx_4w	$xt2,$x20,$out
+	subi	$len,$len,0x40
+	stvx_4w	$xt3,$x30,$out
+	addi	$out,$out,0x40
+	beq	Ldone_vsx
+
+	vadduwm	$xa0,$xa1,@K[0]
+	vadduwm	$xb0,$xb1,@K[1]
+	vadduwm	$xc0,$xc1,@K[2]
+	vadduwm	$xd0,$xd1,@K[3]
+
+	be?vperm $xa0,$xa0,$xa0,$beperm
+	be?vperm $xb0,$xb0,$xb0,$beperm
+	be?vperm $xc0,$xc0,$xc0,$beperm
+	be?vperm $xd0,$xd0,$xd0,$beperm
+
+	${UCMP}i $len,0x40
+	blt	Ltail_vsx
+
+	lvx_4w	$xt0,$x00,$inp
+	lvx_4w	$xt1,$x10,$inp
+	lvx_4w	$xt2,$x20,$inp
+	lvx_4w	$xt3,$x30,$inp
+
+	vxor	$xt0,$xt0,$xa0
+	vxor	$xt1,$xt1,$xb0
+	vxor	$xt2,$xt2,$xc0
+	vxor	$xt3,$xt3,$xd0
+
+	stvx_4w	$xt0,$x00,$out
+	stvx_4w	$xt1,$x10,$out
+	addi	$inp,$inp,0x40
+	stvx_4w	$xt2,$x20,$out
+	subi	$len,$len,0x40
+	stvx_4w	$xt3,$x30,$out
+	addi	$out,$out,0x40
+	beq	Ldone_vsx
+
+	vadduwm	$xa0,$xa2,@K[0]
+	vadduwm	$xb0,$xb2,@K[1]
+	vadduwm	$xc0,$xc2,@K[2]
+	vadduwm	$xd0,$xd2,@K[3]
+
+	be?vperm $xa0,$xa0,$xa0,$beperm
+	be?vperm $xb0,$xb0,$xb0,$beperm
+	be?vperm $xc0,$xc0,$xc0,$beperm
+	be?vperm $xd0,$xd0,$xd0,$beperm
+
+	${UCMP}i $len,0x40
+	blt	Ltail_vsx
+
+	lvx_4w	$xt0,$x00,$inp
+	lvx_4w	$xt1,$x10,$inp
+	lvx_4w	$xt2,$x20,$inp
+	lvx_4w	$xt3,$x30,$inp
+
+	vxor	$xt0,$xt0,$xa0
+	vxor	$xt1,$xt1,$xb0
+	vxor	$xt2,$xt2,$xc0
+	vxor	$xt3,$xt3,$xd0
+
+	stvx_4w	$xt0,$x00,$out
+	stvx_4w	$xt1,$x10,$out
+	addi	$inp,$inp,0x40
+	stvx_4w	$xt2,$x20,$out
+	subi	$len,$len,0x40
+	stvx_4w	$xt3,$x30,$out
+	addi	$out,$out,0x40
+	beq	Ldone_vsx
+
+	vadduwm	$xa0,$xa3,@K[0]
+	vadduwm	$xb0,$xb3,@K[1]
+	vadduwm	$xc0,$xc3,@K[2]
+	vadduwm	$xd0,$xd3,@K[3]
+
+	be?vperm $xa0,$xa0,$xa0,$beperm
+	be?vperm $xb0,$xb0,$xb0,$beperm
+	be?vperm $xc0,$xc0,$xc0,$beperm
+	be?vperm $xd0,$xd0,$xd0,$beperm
+
+	${UCMP}i $len,0x40
+	blt	Ltail_vsx
+
+	lvx_4w	$xt0,$x00,$inp
+	lvx_4w	$xt1,$x10,$inp
+	lvx_4w	$xt2,$x20,$inp
+	lvx_4w	$xt3,$x30,$inp
+
+	vxor	$xt0,$xt0,$xa0
+	vxor	$xt1,$xt1,$xb0
+	vxor	$xt2,$xt2,$xc0
+	vxor	$xt3,$xt3,$xd0
+
+	stvx_4w	$xt0,$x00,$out
+	stvx_4w	$xt1,$x10,$out
+	addi	$inp,$inp,0x40
+	stvx_4w	$xt2,$x20,$out
+	subi	$len,$len,0x40
+	stvx_4w	$xt3,$x30,$out
+	addi	$out,$out,0x40
+	mtctr	r0
+	bne	Loop_outer_vsx
+
+Ldone_vsx:
+	lwz	r12,`$FRAME-4`($sp)		# pull vrsave
+	li	r10,`15+$LOCALS+64`
+	li	r11,`31+$LOCALS+64`
+	$POP	r0, `$FRAME+$LRSAVE`($sp)
+	mtspr	256,r12				# restore vrsave
+	lvx	v26,r10,$sp
+	addi	r10,r10,32
+	lvx	v27,r11,$sp
+	addi	r11,r11,32
+	lvx	v28,r10,$sp
+	addi	r10,r10,32
+	lvx	v29,r11,$sp
+	addi	r11,r11,32
+	lvx	v30,r10,$sp
+	lvx	v31,r11,$sp
+	mtlr	r0
+	addi	$sp,$sp,$FRAME
+	blr
+
+.align	4
+Ltail_vsx:
+	addi	r11,$sp,$LOCALS
+	mtctr	$len
+	stvx_4w	$xa0,$x00,r11			# offload block to stack
+	stvx_4w	$xb0,$x10,r11
+	stvx_4w	$xc0,$x20,r11
+	stvx_4w	$xd0,$x30,r11
+	subi	r12,r11,1			# prepare for *++ptr
+	subi	$inp,$inp,1
+	subi	$out,$out,1
+
+Loop_tail_vsx:
+	lbzu	r6,1(r12)
+	lbzu	r7,1($inp)
+	xor	r6,r6,r7
+	stbu	r6,1($out)
+	bdnz	Loop_tail_vsx
+
+	stvx_4w	$K[0],$x00,r11			# wipe copy of the block
+	stvx_4w	$K[0],$x10,r11
+	stvx_4w	$K[0],$x20,r11
+	stvx_4w	$K[0],$x30,r11
+
+	b	Ldone_vsx
+	.long	0
+	.byte	0,12,0x04,1,0x80,0,5,0
+	.long	0
+.size	.ChaCha20_ctr32_vsx,.-.ChaCha20_ctr32_vsx
+___
+}}}
+$code.=<<___;
+.align	5
+Lconsts:
+	mflr	r0
+	bcl	20,31,\$+4
+	mflr	r12	#vvvvv "distance between . and Lsigma
+	addi	r12,r12,`64-8`
+	mtlr	r0
+	blr
+	.long	0
+	.byte	0,12,0x14,0,0,0,0,0
+	.space	`64-9*4`
+Lsigma:
+	.long   0x61707865,0x3320646e,0x79622d32,0x6b206574
+	.long	1,0,0,0
+	.long	4,0,0,0
+___
+$code.=<<___ 	if ($LITTLE_ENDIAN);
+	.long	0x0e0f0c0d,0x0a0b0809,0x06070405,0x02030001
+	.long	0x0d0e0f0c,0x090a0b08,0x05060704,0x01020300
+___
+$code.=<<___ 	if (!$LITTLE_ENDIAN);	# flipped words
+	.long	0x02030001,0x06070405,0x0a0b0809,0x0e0f0c0d
+	.long	0x01020300,0x05060704,0x090a0b08,0x0d0e0f0c
+___
+$code.=<<___;
+	.long	0x61707865,0x61707865,0x61707865,0x61707865
+	.long	0x3320646e,0x3320646e,0x3320646e,0x3320646e
+	.long	0x79622d32,0x79622d32,0x79622d32,0x79622d32
+	.long	0x6b206574,0x6b206574,0x6b206574,0x6b206574
+	.long	0,1,2,3
+.asciz  "ChaCha20 for PowerPC/AltiVec, CRYPTOGAMS by <appro\@openssl.org>"
+.align	2
+___
+
+foreach (split("\n",$code)) {
+	s/\`([^\`]*)\`/eval $1/ge;
+
+	# instructions prefixed with '?' are endian-specific and need
+	# to be adjusted accordingly...
+	if ($flavour !~ /le$/) {	# big-endian
+	    s/be\?//		or
+	    s/le\?/#le#/	or
+	    s/\?lvsr/lvsl/	or
+	    s/\?lvsl/lvsr/	or
+	    s/\?(vperm\s+v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+)/$1$3$2$4/ or
+	    s/vrldoi(\s+v[0-9]+,\s*)(v[0-9]+,)\s*([0-9]+)/vsldoi$1$2$2 16-$3/;
+	} else {			# little-endian
+	    s/le\?//		or
+	    s/be\?/#be#/	or
+	    s/\?([a-z]+)/$1/	or
+	    s/vrldoi(\s+v[0-9]+,\s*)(v[0-9]+,)\s*([0-9]+)/vsldoi$1$2$2 $3/;
+	}
+
+	print $_,"\n";
+}
+
+close STDOUT;
diff --git a/src/crypto/zinc/chacha20/chacha20.c b/src/crypto/zinc/chacha20/chacha20.c
index b4763c8..42e5360 100644
--- a/src/crypto/zinc/chacha20/chacha20.c
+++ b/src/crypto/zinc/chacha20/chacha20.c
@@ -20,10 +20,12 @@
 #include "chacha20-x86_64-glue.c"
 #elif defined(CONFIG_ZINC_ARCH_ARM) || defined(CONFIG_ZINC_ARCH_ARM64)
 #include "chacha20-arm-glue.c"
 #elif defined(CONFIG_ZINC_ARCH_MIPS)
 #include "chacha20-mips-glue.c"
+#elif defined(CONFIG_ZINC_ARCH_PPC32) || defined(CONFIG_ZINC_ARCH_PPC64)
+#include "chacha20-ppc-glue.c"
 #else
 static bool *const chacha20_nobs[] __initconst = { };
 static void __init chacha20_fpu_init(void)
 {
 }
diff --git a/src/crypto/zinc/chacha20/ppc-xlate.pl b/src/crypto/zinc/chacha20/ppc-xlate.pl
new file mode 100644
index 0000000..2362071
--- /dev/null
+++ b/src/crypto/zinc/chacha20/ppc-xlate.pl
@@ -0,0 +1,353 @@
+#! /usr/bin/env perl
+# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+#
+# This code is taken from the OpenSSL project but the author, Andy Polyakov,
+# has relicensed it under the licenses specified in the SPDX header above.
+# The original headers, including the original license headers, are
+# included below for completeness.
+#
+# Copyright 2006-2018 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the Apache License 2.0 (the "License").  You may not use
+# this file except in compliance with the License.  You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
+my $flavour = shift;
+my $output = shift;
+open STDOUT,">$output" || die "can't open $output: $!";
+
+my %GLOBALS;
+my %TYPES;
+my $dotinlocallabels=($flavour=~/linux/)?1:0;
+
+################################################################
+# directives which need special treatment on different platforms
+################################################################
+my $type = sub {
+    my ($dir,$name,$type) = @_;
+
+    $TYPES{$name} = $type;
+    if ($flavour =~ /linux/) {
+	$name =~ s|^\.||;
+	".type	$name,$type";
+    } else {
+	"";
+    }
+};
+my $globl = sub {
+    my $junk = shift;
+    my $name = shift;
+    my $global = \$GLOBALS{$name};
+    my $type = \$TYPES{$name};
+    my $ret;
+
+    $name =~ s|^\.||;
+
+    SWITCH: for ($flavour) {
+	/aix/		&& do { if (!$$type) {
+				    $$type = "\@function";
+				}
+				if ($$type =~ /function/) {
+				    $name = ".$name";
+				}
+				last;
+			      };
+	/osx/		&& do { $name = "_$name";
+				last;
+			      };
+	/linux.*(32|64le)/
+			&& do {	$ret .= ".globl	$name";
+				if (!$$type) {
+				    $ret .= "\n.type	$name,\@function";
+				    $$type = "\@function";
+				}
+				last;
+			      };
+	/linux.*64/	&& do {	$ret .= ".globl	$name";
+				if (!$$type) {
+				    $ret .= "\n.type	$name,\@function";
+				    $$type = "\@function";
+				}
+				if ($$type =~ /function/) {
+				    $ret .= "\n.section	\".opd\",\"aw\"";
+				    $ret .= "\n.align	3";
+				    $ret .= "\n$name:";
+				    $ret .= "\n.quad	.$name,.TOC.\@tocbase,0";
+				    $ret .= "\n.previous";
+				    $name = ".$name";
+				}
+				last;
+			      };
+    }
+
+    $ret = ".globl	$name" if (!$ret);
+    $$global = $name;
+    $ret;
+};
+my $text = sub {
+    my $ret = ($flavour =~ /aix/) ? ".csect\t.text[PR],7" : ".text";
+    $ret = ".abiversion	2\n".$ret	if ($flavour =~ /linux.*64le/);
+    $ret;
+};
+my $machine = sub {
+    my $junk = shift;
+    my $arch = shift;
+    if ($flavour =~ /osx/)
+    {	$arch =~ s/\"//g;
+	$arch = ($flavour=~/64/) ? "ppc970-64" : "ppc970" if ($arch eq "any");
+    }
+    ".machine	$arch";
+};
+my $size = sub {
+    if ($flavour =~ /linux/)
+    {	shift;
+	my $name = shift;
+	my $real = $GLOBALS{$name} ? \$GLOBALS{$name} : \$name;
+	my $ret  = ".size	$$real,.-$$real";
+	$name =~ s|^\.||;
+	if ($$real ne $name) {
+	    $ret .= "\n.size	$name,.-$$real";
+	}
+	$ret;
+    }
+    else
+    {	"";	}
+};
+my $asciz = sub {
+    shift;
+    my $line = join(",",@_);
+    if ($line =~ /^"(.*)"$/)
+    {	".byte	" . join(",",unpack("C*",$1),0) . "\n.align	2";	}
+    else
+    {	"";	}
+};
+my $quad = sub {
+    shift;
+    my @ret;
+    my ($hi,$lo);
+    for (@_) {
+	if (/^0x([0-9a-f]*?)([0-9a-f]{1,8})$/io)
+	{  $hi=$1?"0x$1":"0"; $lo="0x$2";  }
+	elsif (/^([0-9]+)$/o)
+	{  $hi=$1>>32; $lo=$1&0xffffffff;  } # error-prone with 32-bit perl
+	else
+	{  $hi=undef; $lo=$_; }
+
+	if (defined($hi))
+	{  push(@ret,$flavour=~/le$/o?".long\t$lo,$hi":".long\t$hi,$lo");  }
+	else
+	{  push(@ret,".quad	$lo");  }
+    }
+    join("\n",@ret);
+};
+
+################################################################
+# simplified mnemonics not handled by at least one assembler
+################################################################
+my $cmplw = sub {
+    my $f = shift;
+    my $cr = 0; $cr = shift if ($#_>1);
+    # Some out-of-date 32-bit GNU assembler just can't handle cmplw...
+    ($flavour =~ /linux.*32/) ?
+	"	.long	".sprintf "0x%x",31<<26|$cr<<23|$_[0]<<16|$_[1]<<11|64 :
+	"	cmplw	".join(',',$cr,@_);
+};
+my $bdnz = sub {
+    my $f = shift;
+    my $bo = $f=~/[\+\-]/ ? 16+9 : 16;	# optional "to be taken" hint
+    "	bc	$bo,0,".shift;
+} if ($flavour!~/linux/);
+my $bltlr = sub {
+    my $f = shift;
+    my $bo = $f=~/\-/ ? 12+2 : 12;	# optional "not to be taken" hint
+    ($flavour =~ /linux/) ?		# GNU as doesn't allow most recent hints
+	"	.long	".sprintf "0x%x",19<<26|$bo<<21|16<<1 :
+	"	bclr	$bo,0";
+};
+my $bnelr = sub {
+    my $f = shift;
+    my $bo = $f=~/\-/ ? 4+2 : 4;	# optional "not to be taken" hint
+    ($flavour =~ /linux/) ?		# GNU as doesn't allow most recent hints
+	"	.long	".sprintf "0x%x",19<<26|$bo<<21|2<<16|16<<1 :
+	"	bclr	$bo,2";
+};
+my $beqlr = sub {
+    my $f = shift;
+    my $bo = $f=~/-/ ? 12+2 : 12;	# optional "not to be taken" hint
+    ($flavour =~ /linux/) ?		# GNU as doesn't allow most recent hints
+	"	.long	".sprintf "0x%X",19<<26|$bo<<21|2<<16|16<<1 :
+	"	bclr	$bo,2";
+};
+# GNU assembler can't handle extrdi rA,rS,16,48, or when sum of last two
+# arguments is 64, with "operand out of range" error.
+my $extrdi = sub {
+    my ($f,$ra,$rs,$n,$b) = @_;
+    $b = ($b+$n)&63; $n = 64-$n;
+    "	rldicl	$ra,$rs,$b,$n";
+};
+my $vmr = sub {
+    my ($f,$vx,$vy) = @_;
+    "	vor	$vx,$vy,$vy";
+};
+
+# Some ABIs specify vrsave, special-purpose register #256, as reserved
+# for system use.
+my $no_vrsave = ($flavour =~ /aix|linux64le/);
+my $mtspr = sub {
+    my ($f,$idx,$ra) = @_;
+    if ($idx == 256 && $no_vrsave) {
+	"	or	$ra,$ra,$ra";
+    } else {
+	"	mtspr	$idx,$ra";
+    }
+};
+my $mfspr = sub {
+    my ($f,$rd,$idx) = @_;
+    if ($idx == 256 && $no_vrsave) {
+	"	li	$rd,-1";
+    } else {
+	"	mfspr	$rd,$idx";
+    }
+};
+
+# PowerISA 2.06 stuff
+sub vsxmem_op {
+    my ($f, $vrt, $ra, $rb, $op) = @_;
+    "	.long	".sprintf "0x%X",(31<<26)|($vrt<<21)|($ra<<16)|($rb<<11)|($op*2+1);
+}
+# made-up unaligned memory reference AltiVec/VMX instructions
+my $lvx_u	= sub {	vsxmem_op(@_, 844); };	# lxvd2x
+my $stvx_u	= sub {	vsxmem_op(@_, 972); };	# stxvd2x
+my $lvdx_u	= sub {	vsxmem_op(@_, 588); };	# lxsdx
+my $stvdx_u	= sub {	vsxmem_op(@_, 716); };	# stxsdx
+my $lvx_4w	= sub { vsxmem_op(@_, 780); };	# lxvw4x
+my $stvx_4w	= sub { vsxmem_op(@_, 908); };	# stxvw4x
+my $lvx_splt	= sub { vsxmem_op(@_, 332); };	# lxvdsx
+# VSX instruction[s] masqueraded as made-up AltiVec/VMX
+my $vpermdi	= sub {				# xxpermdi
+    my ($f, $vrt, $vra, $vrb, $dm) = @_;
+    $dm = oct($dm) if ($dm =~ /^0/);
+    "	.long	".sprintf "0x%X",(60<<26)|($vrt<<21)|($vra<<16)|($vrb<<11)|($dm<<8)|(10<<3)|7;
+};
+
+# PowerISA 2.07 stuff
+sub vcrypto_op {
+    my ($f, $vrt, $vra, $vrb, $op) = @_;
+    "	.long	".sprintf "0x%X",(4<<26)|($vrt<<21)|($vra<<16)|($vrb<<11)|$op;
+}
+sub vfour {
+    my ($f, $vrt, $vra, $vrb, $vrc, $op) = @_;
+    "	.long	".sprintf "0x%X",(4<<26)|($vrt<<21)|($vra<<16)|($vrb<<11)|($vrc<<6)|$op;
+};
+my $vcipher	= sub { vcrypto_op(@_, 1288); };
+my $vcipherlast	= sub { vcrypto_op(@_, 1289); };
+my $vncipher	= sub { vcrypto_op(@_, 1352); };
+my $vncipherlast= sub { vcrypto_op(@_, 1353); };
+my $vsbox	= sub { vcrypto_op(@_, 0, 1480); };
+my $vshasigmad	= sub { my ($st,$six)=splice(@_,-2); vcrypto_op(@_, $st<<4|$six, 1730); };
+my $vshasigmaw	= sub { my ($st,$six)=splice(@_,-2); vcrypto_op(@_, $st<<4|$six, 1666); };
+my $vpmsumb	= sub { vcrypto_op(@_, 1032); };
+my $vpmsumd	= sub { vcrypto_op(@_, 1224); };
+my $vpmsubh	= sub { vcrypto_op(@_, 1096); };
+my $vpmsumw	= sub { vcrypto_op(@_, 1160); };
+# These are not really crypto, but vcrypto_op template works
+my $vaddudm	= sub { vcrypto_op(@_, 192);  };
+my $vadduqm	= sub { vcrypto_op(@_, 256);  };
+my $vmuleuw	= sub { vcrypto_op(@_, 648);  };
+my $vmulouw	= sub { vcrypto_op(@_, 136);  };
+my $vrld	= sub { vcrypto_op(@_, 196);  };
+my $vsld	= sub { vcrypto_op(@_, 1476); };
+my $vsrd	= sub { vcrypto_op(@_, 1732); };
+my $vsubudm	= sub { vcrypto_op(@_, 1216); };
+my $vaddcuq	= sub { vcrypto_op(@_, 320);  };
+my $vaddeuqm	= sub { vfour(@_,60); };
+my $vaddecuq	= sub { vfour(@_,61); };
+my $vmrgew	= sub { vfour(@_,0,1932); };
+my $vmrgow	= sub { vfour(@_,0,1676); };
+
+my $mtsle	= sub {
+    my ($f, $arg) = @_;
+    "	.long	".sprintf "0x%X",(31<<26)|($arg<<21)|(147*2);
+};
+
+# VSX instructions masqueraded as AltiVec/VMX
+my $mtvrd	= sub {
+    my ($f, $vrt, $ra) = @_;
+    "	.long	".sprintf "0x%X",(31<<26)|($vrt<<21)|($ra<<16)|(179<<1)|1;
+};
+my $mtvrwz	= sub {
+    my ($f, $vrt, $ra) = @_;
+    "	.long	".sprintf "0x%X",(31<<26)|($vrt<<21)|($ra<<16)|(243<<1)|1;
+};
+my $lvwzx_u	= sub { vsxmem_op(@_, 12); };	# lxsiwzx
+my $stvwx_u	= sub { vsxmem_op(@_, 140); };	# stxsiwx
+
+# PowerISA 3.0 stuff
+my $maddhdu	= sub { vfour(@_,49); };
+my $maddld	= sub { vfour(@_,51); };
+my $darn = sub {
+    my ($f, $rt, $l) = @_;
+    "	.long	".sprintf "0x%X",(31<<26)|($rt<<21)|($l<<16)|(755<<1);
+};
+my $iseleq = sub {
+    my ($f, $rt, $ra, $rb) = @_;
+    "	.long	".sprintf "0x%X",(31<<26)|($rt<<21)|($ra<<16)|($rb<<11)|(2<<6)|30;
+};
+# VSX instruction[s] masqueraded as made-up AltiVec/VMX
+my $vspltib	= sub {				# xxspltib
+    my ($f, $vrt, $imm8) = @_;
+    $imm8 = oct($imm8) if ($imm8 =~ /^0/);
+    $imm8 &= 0xff;
+    "	.long	".sprintf "0x%X",(60<<26)|($vrt<<21)|($imm8<<11)|(360<<1)|1;
+};
+
+# PowerISA 3.0B stuff
+my $addex = sub {
+    my ($f, $rt, $ra, $rb, $cy) = @_;	# only cy==0 is specified in 3.0B
+    "	.long	".sprintf "0x%X",(31<<26)|($rt<<21)|($ra<<16)|($rb<<11)|($cy<<9)|(170<<1);
+};
+my $vmsumudm	= sub { vfour(@_,35); };
+
+while($line=<>) {
+
+    $line =~ s|[#!;].*$||;	# get rid of asm-style comments...
+    $line =~ s|/\*.*\*/||;	# ... and C-style comments...
+    $line =~ s|^\s+||;		# ... and skip white spaces in beginning...
+    $line =~ s|\s+$||;		# ... and at the end
+
+    {
+	$line =~ s|\.L(\w+)|L$1|g;	# common denominator for Locallabel
+	$line =~ s|\bL(\w+)|\.L$1|g	if ($dotinlocallabels);
+    }
+
+    {
+	$line =~ s|(^[\.\w]+)\:\s*||;
+	my $label = $1;
+	if ($label) {
+	    my $xlated = ($GLOBALS{$label} or $label);
+	    print "$xlated:";
+	    if ($flavour =~ /linux.*64le/) {
+		if ($TYPES{$label} =~ /function/) {
+		    printf "\n.localentry	%s,0\n",$xlated;
+		}
+	    }
+	}
+    }
+
+    {
+	$line =~ s|^\s*(\.?)(\w+)([\.\+\-]?)\s*||;
+	my $c = $1; $c = "\t" if ($c eq "");
+	my $mnemonic = $2;
+	my $f = $3;
+	my $opcode = eval("\$$mnemonic");
+	$line =~ s/\b(c?[rf]|v|vs)([0-9]+)\b/$2/g if ($c ne "." and $flavour !~ /osx/);
+	if (ref($opcode) eq 'CODE') { $line = &$opcode($f,split(/,\s*/,$line)); }
+	elsif ($mnemonic)           { $line = $c.$mnemonic.$f."\t".$line; }
+    }
+
+    print $line if ($line);
+    print "\n";
+}
+
+close STDOUT;
-- 
2.20.1

_______________________________________________
WireGuard mailing list
WireGuard@lists.zx2c4.com
https://lists.zx2c4.com/mailman/listinfo/wireguard

^ permalink raw reply	[flat|nested] 5+ messages in thread

* [PATCH 2/2] [zinc] add accelerated poly1305 from openssl/cryptograms
  2019-05-11 18:03 ` [PATCH 1/2] " Shawn Landden
@ 2019-05-11 18:03   ` " Shawn Landden
  2019-05-13 21:31   ` [PATCH 1/2 v3] [Zinc] Add PowerPC chacha20 implementation " Shawn Landden
  1 sibling, 0 replies; 5+ messages in thread
From: Shawn Landden @ 2019-05-11 18:03 UTC (permalink / raw)
  To: wireguard

Unfortunately I am not seeing a speed up with this patch,
but it does decrease CPU usage.

Again, this will not work until ppc has its own version of the
may_use_simd() function that allows simd in the softirq context
that WireGuard runs in.

Signed-off-by: Shawn Landden <shawn@git.icu>
---
 src/crypto/Kbuild.include                     |   10 +-
 src/crypto/zinc/chacha20/chacha20-ppc.pl      |    3 +
 .../zinc/{chacha20 => perlasm}/ppc-xlate.pl   |    0
 src/crypto/zinc/poly1305/poly1305-arm-glue.c  |   65 -
 src/crypto/zinc/poly1305/poly1305-ppc-glue.c  |   94 +
 src/crypto/zinc/poly1305/poly1305-ppc.pl      | 1989 +++++++++++++++++
 src/crypto/zinc/poly1305/poly1305-ppcfp.pl    |  749 +++++++
 src/crypto/zinc/poly1305/poly1305.c           |   69 +
 8 files changed, 2910 insertions(+), 69 deletions(-)
 rename src/crypto/zinc/{chacha20 => perlasm}/ppc-xlate.pl (100%)
 create mode 100644 src/crypto/zinc/poly1305/poly1305-ppc-glue.c
 create mode 100644 src/crypto/zinc/poly1305/poly1305-ppc.pl
 create mode 100755 src/crypto/zinc/poly1305/poly1305-ppcfp.pl

diff --git a/src/crypto/Kbuild.include b/src/crypto/Kbuild.include
index 4e05181..ea18ce5 100644
--- a/src/crypto/Kbuild.include
+++ b/src/crypto/Kbuild.include
@@ -34,30 +34,32 @@ zinc-$(CONFIG_ZINC_ARCH_X86_64) += poly1305/poly1305-x86_64.o
 zinc-$(CONFIG_ZINC_ARCH_ARM) += poly1305/poly1305-arm.o
 zinc-$(CONFIG_ZINC_ARCH_ARM64) += poly1305/poly1305-arm64.o
 zinc-$(CONFIG_ZINC_ARCH_MIPS) += poly1305/poly1305-mips.o
 AFLAGS_poly1305-mips.o += -O2 # This is required to fill the branch delay slots
 zinc-$(CONFIG_ZINC_ARCH_MIPS64) += poly1305/poly1305-mips64.o
+zinc-$(CONFIG_ZINC_ARCH_PPC32) += poly1305/poly1305-ppc.o poly1305/poly1305-ppcfp.o
+zinc-$(CONFIG_ZINC_ARCH_PPC64) += poly1305/poly1305-ppc.o poly1305/poly1305-ppcfp.o
 
 zinc-y += chacha20poly1305.o
 
 zinc-y += blake2s/blake2s.o
 zinc-$(CONFIG_ZINC_ARCH_X86_64) += blake2s/blake2s-x86_64.o
 
 zinc-y += curve25519/curve25519.o
 zinc-$(CONFIG_ZINC_ARCH_ARM) += curve25519/curve25519-arm.o
 
 quiet_cmd_perlasm = PERLASM $@
-      cmd_perlasm = $(PERL) $< $(perlflags-y) > $@
+      cmd_perlasm = $(PERL) $(perlflags-y) $< $(perlargs-y) > $@
 $(obj)/%.S: $(src)/%.pl FORCE
 	$(call if_changed,perlasm)
 kbuild-dir := $(if $(filter /%,$(src)),$(src),$(srctree)/$(src))
 targets := $(patsubst $(kbuild-dir)/%.pl,%.S,$(wildcard $(patsubst %.o,$(kbuild-dir)/crypto/zinc/%.pl,$(zinc-y) $(zinc-m) $(zinc-))))
 
-perlflags-$(CONFIG_ZINC_ARCH_PPC32) += linux32
+perlargs-$(CONFIG_ZINC_ARCH_PPC32) += linux32
 ifeq ($(CONFIG_ZINC_ARCH_PPC64),y)
-perlflags-$(CONFIG_CPU_BIG_ENDIAN) += linux64
-perlflags-$(CONFIG_CPU_LITTLE_ENDIAN) += linux64le
+perlargs-$(CONFIG_CPU_BIG_ENDIAN) += linux64
+perlargs-$(CONFIG_CPU_LITTLE_ENDIAN) += linux64le
 endif
 
 # Old kernels don't set this, which causes trouble.
 .SECONDARY:
 
diff --git a/src/crypto/zinc/chacha20/chacha20-ppc.pl b/src/crypto/zinc/chacha20/chacha20-ppc.pl
index 07468c8..fa8f6bc 100644
--- a/src/crypto/zinc/chacha20/chacha20-ppc.pl
+++ b/src/crypto/zinc/chacha20/chacha20-ppc.pl
@@ -4,10 +4,12 @@
 # This code is taken from the OpenSSL project but the author, Andy Polyakov,
 # has relicensed it under the licenses specified in the SPDX header above.
 # The original headers, including the original license headers, are
 # included below for completeness.
 #
+# Changes: search in more places for ppc-xlate.pl
+#
 # Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved.
 #
 # Licensed under the Apache License 2.0 (the "License").  You may not use
 # this file except in compliance with the License.  You can obtain a copy
 # in the file LICENSE in the source distribution or at
@@ -71,10 +73,11 @@ if ($flavour =~ /64/) {
 
 $LITTLE_ENDIAN = ($flavour=~/le$/) ? 1 : 0;
 
 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
 ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
+( $xlate="${dir}../perlasm/ppc-xlate.pl" and -f $xlate) or
 ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
 die "can't locate ppc-xlate.pl";
 
 open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";
 
diff --git a/src/crypto/zinc/chacha20/ppc-xlate.pl b/src/crypto/zinc/perlasm/ppc-xlate.pl
similarity index 100%
rename from src/crypto/zinc/chacha20/ppc-xlate.pl
rename to src/crypto/zinc/perlasm/ppc-xlate.pl
diff --git a/src/crypto/zinc/poly1305/poly1305-arm-glue.c b/src/crypto/zinc/poly1305/poly1305-arm-glue.c
index a80f046..6100700 100644
--- a/src/crypto/zinc/poly1305/poly1305-arm-glue.c
+++ b/src/crypto/zinc/poly1305/poly1305-arm-glue.c
@@ -24,75 +24,10 @@ static void __init poly1305_fpu_init(void)
 #elif defined(CONFIG_ZINC_ARCH_ARM)
 	poly1305_use_neon = elf_hwcap & HWCAP_NEON;
 #endif
 }
 
-#if defined(CONFIG_ZINC_ARCH_ARM64)
-struct poly1305_arch_internal {
-	union {
-		u32 h[5];
-		struct {
-			u64 h0, h1, h2;
-		};
-	};
-	u64 is_base2_26;
-	u64 r[2];
-};
-#elif defined(CONFIG_ZINC_ARCH_ARM)
-struct poly1305_arch_internal {
-	union {
-		u32 h[5];
-		struct {
-			u64 h0, h1;
-			u32 h2;
-		} __packed;
-	};
-	u32 r[4];
-	u32 is_base2_26;
-};
-#endif
-
-/* The NEON code uses base 2^26, while the scalar code uses base 2^64 on 64-bit
- * and base 2^32 on 32-bit. If we hit the unfortunate situation of using NEON
- * and then having to go back to scalar -- because the user is silly and has
- * called the update function from two separate contexts -- then we need to
- * convert back to the original base before proceeding. The below function is
- * written for 64-bit integers, and so we have to swap words at the end on
- * big-endian 32-bit. It is possible to reason that the initial reduction below
- * is sufficient given the implementation invariants. However, for an avoidance
- * of doubt and because this is not performance critical, we do the full
- * reduction anyway.
- */
-static void convert_to_base2_64(void *ctx)
-{
-	struct poly1305_arch_internal *state = ctx;
-	u32 cy;
-
-	if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !state->is_base2_26)
-		return;
-
-	cy = state->h[0] >> 26; state->h[0] &= 0x3ffffff; state->h[1] += cy;
-	cy = state->h[1] >> 26; state->h[1] &= 0x3ffffff; state->h[2] += cy;
-	cy = state->h[2] >> 26; state->h[2] &= 0x3ffffff; state->h[3] += cy;
-	cy = state->h[3] >> 26; state->h[3] &= 0x3ffffff; state->h[4] += cy;
-	state->h0 = ((u64)state->h[2] << 52) | ((u64)state->h[1] << 26) | state->h[0];
-	state->h1 = ((u64)state->h[4] << 40) | ((u64)state->h[3] << 14) | (state->h[2] >> 12);
-	state->h2 = state->h[4] >> 24;
-	if (IS_ENABLED(CONFIG_ZINC_ARCH_ARM) && IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) {
-		state->h0 = rol64(state->h0, 32);
-		state->h1 = rol64(state->h1, 32);
-	}
-#define ULT(a, b) ((a ^ ((a ^ b) | ((a - b) ^ b))) >> (sizeof(a) * 8 - 1))
-	cy = (state->h2 >> 2) + (state->h2 & ~3ULL);
-	state->h2 &= 3;
-	state->h0 += cy;
-	state->h1 += (cy = ULT(state->h0, cy));
-	state->h2 += ULT(state->h1, cy);
-#undef ULT
-	state->is_base2_26 = 0;
-}
-
 static inline bool poly1305_init_arch(void *ctx,
 				      const u8 key[POLY1305_KEY_SIZE])
 {
 	poly1305_init_arm(ctx, key);
 	return true;
diff --git a/src/crypto/zinc/poly1305/poly1305-ppc-glue.c b/src/crypto/zinc/poly1305/poly1305-ppc-glue.c
new file mode 100644
index 0000000..ae6c7fb
--- /dev/null
+++ b/src/crypto/zinc/poly1305/poly1305-ppc-glue.c
@@ -0,0 +1,94 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#include <asm/cpufeature.h>
+
+asmlinkage void poly1305_init_int(void *ctx, const u8 key[16]);
+asmlinkage void poly1305_blocks_int(void *ctx, const u8 *inp, size_t len,
+				    u32 padbit);
+asmlinkage void poly1305_emit_int(void *ctx, u8 mac[16],
+				  const u32 nonce[4]);
+asmlinkage void poly1305_init_fpu(void *ctx, const u8 key[16]);
+asmlinkage void poly1305_blocks_fpu(void *ctx, const u8 *inp, size_t len,
+				    u32 padbit);
+asmlinkage void poly1305_emit_fpu(void *ctx, u8 mac[16],
+				  const u32 nonce[4]);
+asmlinkage void poly1305_blocks_vsx(void *ctx, const u8 *inp, size_t len,
+				    u32 padbit);
+static void (*poly1305_init_ppc)(void *ctx, const u8 key[16]) __ro_after_init;
+static void (*poly1305_emit_ppc)(void *ctx, u8 mac[16],
+				 const u32 nonce[4]) __ro_after_init;
+static void (*poly1305_blocks_ppc)(void *ctx, const u8 *inp, size_t len,
+				   u32 padbit) __ro_after_init;
+static bool *const poly1305_nobs[] __initconst = {};
+
+static void inline do_invalid_op(void) {
+	BUG();
+}
+
+static void __init poly1305_fpu_init(void)
+{
+	if (cpu_have_feature(PPC_MODULE_FEATURE_VEC_CRYPTO)) {
+		poly1305_init_ppc = &poly1305_init_int;	/* No special init */
+		poly1305_emit_ppc = &poly1305_emit_int;	/* No special emit */
+		poly1305_blocks_ppc = &poly1305_blocks_vsx;
+	} else if (!cpu_has_feature(CPU_FTR_FPU_UNAVAILABLE)) {
+		poly1305_init_ppc = &poly1305_init_fpu;
+		poly1305_emit_ppc = &poly1305_emit_fpu;
+		poly1305_blocks_ppc = &poly1305_blocks_fpu;
+	} else {
+		poly1305_init_ppc = &poly1305_init_int;
+		poly1305_emit_ppc = &poly1305_emit_int;
+		poly1305_blocks_ppc = (void (*)(void *ctx, const u8 *inp, size_t len, u32 padbit))&do_invalid_op;
+	}
+}
+
+static inline bool poly1305_init_arch(void *ctx,
+				      const u8 key[POLY1305_KEY_SIZE])
+{
+	poly1305_init_ppc(ctx, key);
+	return true;
+}
+
+static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp,
+					size_t len, const u32 padbit,
+					simd_context_t *simd_context)
+{
+	/* SIMD disables preemption, so relax after processing each page. */
+	BUILD_BUG_ON(PAGE_SIZE < POLY1305_BLOCK_SIZE ||
+		     PAGE_SIZE % POLY1305_BLOCK_SIZE);
+
+	if (!IS_ENABLED(CONFIG_PPC_FPU) ||
+	    !simd_use(simd_context)) {
+		convert_to_base2_64(ctx);
+		poly1305_blocks_int(ctx, inp, len, padbit);
+		return true;
+	}
+
+	for (;;) {
+		const size_t bytes = min_t(size_t, len, PAGE_SIZE);
+
+		poly1305_blocks_ppc(ctx, inp, bytes, padbit);
+		len -= bytes;
+		if (!len)
+			break;
+		inp += bytes;
+		simd_relax(simd_context);
+	}
+	return true;
+}
+
+static inline bool poly1305_emit_arch(void *ctx, u8 mac[POLY1305_MAC_SIZE],
+				      const u32 nonce[4],
+				      simd_context_t *simd_context)
+{
+	if (!IS_ENABLED(CONFIG_PPC_FPU) ||
+	    !simd_use(simd_context)) {
+		convert_to_base2_64(ctx);
+		poly1305_emit_int(ctx, mac, nonce);
+	} else
+		poly1305_emit_ppc(ctx, mac, nonce);
+	return true;
+}
diff --git a/src/crypto/zinc/poly1305/poly1305-ppc.pl b/src/crypto/zinc/poly1305/poly1305-ppc.pl
new file mode 100644
index 0000000..dd4e3fb
--- /dev/null
+++ b/src/crypto/zinc/poly1305/poly1305-ppc.pl
@@ -0,0 +1,1989 @@
+#! /usr/bin/env perl
+# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+#
+# This code is taken from the OpenSSL project but the author, Andy Polyakov,
+# has relicensed it under the licenses specified in the SPDX header above.
+# The original headers, including the original license headers, are
+# included below for completeness.
+#
+# Changes: renamed poly1305_emit to poly1305_emit_int,
+#          renamed poly1305_blocks to poly1305_blocks_int
+#          Look in more places for ppc-xlate.pl
+#
+# Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the Apache License 2.0 (the "License").  You may not use
+# this file except in compliance with the License.  You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
+#
+# ====================================================================
+# Written by Andy Polyakov, @dot-asm, initially for use in the OpenSSL
+# project. The module is dual licensed under OpenSSL and CRYPTOGAMS
+# licenses depending on where you obtain it. For further details see
+# https://github.com/dot-asm/cryptogams/.
+# ====================================================================
+#
+# This module implements Poly1305 hash for PowerPC.
+#
+# June 2015
+#
+# Numbers are cycles per processed byte with poly1305_blocks alone,
+# and improvement coefficients relative to gcc-generated code.
+#
+#			-m32		-m64
+#
+# Freescale e300	14.8/+80%	-
+# PPC74x0		7.60/+60%	-
+# PPC970		7.00/+114%	3.51/+205%
+# POWER7		3.75/+260%	1.93/+100%
+# POWER8		-		2.03/+200%
+# POWER9		-		2.00/+150%
+#
+# Do we need floating-point implementation for PPC? Results presented
+# in poly1305_ieee754.c are tricky to compare to, because they are for
+# compiler-generated code. On the other hand it's known that floating-
+# point performance can be dominated by FPU latency, which means that
+# there is limit even for ideally optimized (and even vectorized) code.
+# And this limit is estimated to be higher than above -m64 results. Or
+# in other words floating-point implementation can be meaningful to
+# consider only in 32-bit application context. We probably have to
+# recognize that 32-bit builds are getting less popular on high-end
+# systems and therefore tend to target embedded ones, which might not
+# even have FPU...
+#
+# On side note, Power ISA 2.07 enables vector base 2^26 implementation,
+# and POWER8 might have capacity to break 1.0 cycle per byte barrier...
+#
+# January 2019
+#
+# ... Unfortunately not:-( Estimate was a projection of ARM result,
+# but ARM has vector multiply-n-add instruction, while PowerISA does
+# not, not one usable in the context. Improvement is ~40% over -m64
+# result above and is ~1.43 on little-endian systems.
+
+$flavour = shift;
+
+if ($flavour =~ /64/) {
+	$SIZE_T	=8;
+	$LRSAVE	=2*$SIZE_T;
+	$UCMP	="cmpld";
+	$STU	="stdu";
+	$POP	="ld";
+	$PUSH	="std";
+} elsif ($flavour =~ /32/) {
+	$SIZE_T	=4;
+	$LRSAVE	=$SIZE_T;
+	$UCMP	="cmplw";
+	$STU	="stwu";
+	$POP	="lwz";
+	$PUSH	="stw";
+} else { die "nonsense $flavour"; }
+
+# Define endianness based on flavour
+# i.e.: linux64le
+$LITTLE_ENDIAN = ($flavour=~/le$/) ? $SIZE_T : 0;
+
+$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
+( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
+( $xlate="${dir}../perlasm/ppc-xlate.pl" and -f $xlate) or
+( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
+die "can't locate ppc-xlate.pl";
+
+open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";
+
+$FRAME=24*$SIZE_T;
+
+$sp="r1";
+my ($ctx,$inp,$len,$padbit) = map("r$_",(3..6));
+my ($mac,$nonce)=($inp,$len);
+my $mask = "r0";
+
+$code=<<___;
+.machine	"any"
+.text
+___
+							if ($flavour =~ /64/) {
+###############################################################################
+# base 2^64 implementation
+
+my ($h0,$h1,$h2,$d0,$d1,$d2, $r0,$r1,$s1, $t0,$t1) = map("r$_",(7..12,27..31));
+
+$code.=<<___;
+.globl	.poly1305_init_int
+.align	4
+.poly1305_init_int:
+	xor	r0,r0,r0
+	std	r0,0($ctx)		# zero hash value
+	std	r0,8($ctx)
+	std	r0,16($ctx)
+	stw	r0,24($ctx)		# clear is_base2_26
+
+	$UCMP	$inp,r0
+	beq-	Lno_key
+___
+$code.=<<___	if ($LITTLE_ENDIAN);
+	ld	$d0,0($inp)		# load key material
+	ld	$d1,8($inp)
+___
+$code.=<<___	if (!$LITTLE_ENDIAN);
+	li	$h0,4
+	lwbrx	$d0,0,$inp		# load key material
+	li	$d1,8
+	lwbrx	$h0,$h0,$inp
+	li	$h1,12
+	lwbrx	$d1,$d1,$inp
+	lwbrx	$h1,$h1,$inp
+	insrdi	$d0,$h0,32,0
+	insrdi	$d1,$h1,32,0
+___
+$code.=<<___;
+	lis	$h1,0xfff		# 0x0fff0000
+	ori	$h1,$h1,0xfffc		# 0x0ffffffc
+	insrdi	$h1,$h1,32,0		# 0x0ffffffc0ffffffc
+	ori	$h0,$h1,3		# 0x0ffffffc0fffffff
+
+	and	$d0,$d0,$h0
+	and	$d1,$d1,$h1
+
+	std	$d0,32($ctx)		# store key
+	std	$d1,40($ctx)
+
+Lno_key:
+	xor	r3,r3,r3
+	blr
+	.long	0
+	.byte	0,12,0x14,0,0,0,2,0
+.size	.poly1305_init_int,.-.poly1305_init_int
+
+.globl	.poly1305_blocks_int
+.align	4
+.poly1305_blocks_int:
+Lpoly1305_blocks:
+	srdi.	$len,$len,4
+	beq-	Labort
+
+	$STU	$sp,-$FRAME($sp)
+	mflr	r0
+	$PUSH	r27,`$FRAME-$SIZE_T*5`($sp)
+	$PUSH	r28,`$FRAME-$SIZE_T*4`($sp)
+	$PUSH	r29,`$FRAME-$SIZE_T*3`($sp)
+	$PUSH	r30,`$FRAME-$SIZE_T*2`($sp)
+	$PUSH	r31,`$FRAME-$SIZE_T*1`($sp)
+	$PUSH	r0,`$FRAME+$LRSAVE`($sp)
+
+	ld	$r0,32($ctx)		# load key
+	ld	$r1,40($ctx)
+
+	ld	$h0,0($ctx)		# load hash value
+	ld	$h1,8($ctx)
+	ld	$h2,16($ctx)
+
+	srdi	$s1,$r1,2
+	mtctr	$len
+	add	$s1,$s1,$r1		# s1 = r1 + r1>>2
+	li	$mask,3
+	b	Loop
+
+.align	4
+Loop:
+___
+$code.=<<___	if ($LITTLE_ENDIAN);
+	ld	$t0,0($inp)		# load input
+	ld	$t1,8($inp)
+___
+$code.=<<___	if (!$LITTLE_ENDIAN);
+	li	$d0,4
+	lwbrx	$t0,0,$inp		# load input
+	li	$t1,8
+	lwbrx	$d0,$d0,$inp
+	li	$d1,12
+	lwbrx	$t1,$t1,$inp
+	lwbrx	$d1,$d1,$inp
+	insrdi	$t0,$d0,32,0
+	insrdi	$t1,$d1,32,0
+___
+$code.=<<___;
+	addi	$inp,$inp,16
+
+	addc	$h0,$h0,$t0		# accumulate input
+	adde	$h1,$h1,$t1
+
+	mulld	$d0,$h0,$r0		# h0*r0
+	mulhdu	$d1,$h0,$r0
+	adde	$h2,$h2,$padbit
+
+	mulld	$t0,$h1,$s1		# h1*5*r1
+	mulhdu	$t1,$h1,$s1
+	addc	$d0,$d0,$t0
+	adde	$d1,$d1,$t1
+
+	mulld	$t0,$h0,$r1		# h0*r1
+	mulhdu	$d2,$h0,$r1
+	addc	$d1,$d1,$t0
+	addze	$d2,$d2
+
+	mulld	$t0,$h1,$r0		# h1*r0
+	mulhdu	$t1,$h1,$r0
+	addc	$d1,$d1,$t0
+	adde	$d2,$d2,$t1
+
+	mulld	$t0,$h2,$s1		# h2*5*r1
+	mulld	$t1,$h2,$r0		# h2*r0
+	addc	$d1,$d1,$t0
+	adde	$d2,$d2,$t1
+
+	andc	$t0,$d2,$mask		# final reduction step
+	and	$h2,$d2,$mask
+	srdi	$t1,$t0,2
+	add	$t0,$t0,$t1
+	addc	$h0,$d0,$t0
+	addze	$h1,$d1
+	addze	$h2,$h2
+
+	bdnz	Loop
+
+	std	$h0,0($ctx)		# store hash value
+	std	$h1,8($ctx)
+	std	$h2,16($ctx)
+
+	$POP	r27,`$FRAME-$SIZE_T*5`($sp)
+	$POP	r28,`$FRAME-$SIZE_T*4`($sp)
+	$POP	r29,`$FRAME-$SIZE_T*3`($sp)
+	$POP	r30,`$FRAME-$SIZE_T*2`($sp)
+	$POP	r31,`$FRAME-$SIZE_T*1`($sp)
+	addi	$sp,$sp,$FRAME
+Labort:
+	blr
+	.long	0
+	.byte	0,12,4,1,0x80,5,4,0
+.size	.poly1305_blocks_int,.-.poly1305_blocks_int
+___
+{
+my ($h0,$h1,$h2,$h3,$h4,$t0) = map("r$_",(7..12));
+
+$code.=<<___;
+.globl	.poly1305_emit_int
+.align	5
+.poly1305_emit_int:
+	lwz	$h0,0($ctx)	# load hash value base 2^26
+	lwz	$h1,4($ctx)
+	lwz	$h2,8($ctx)
+	lwz	$h3,12($ctx)
+	lwz	$h4,16($ctx)
+	lwz	r0,24($ctx)	# is_base2_26
+
+	sldi	$h1,$h1,26	# base 2^26 -> base 2^64
+	sldi	$t0,$h2,52
+	srdi	$h2,$h2,12
+	sldi	$h3,$h3,14
+	add	$h0,$h0,$h1
+	addc	$h0,$h0,$t0
+	sldi	$t0,$h4,40
+	srdi	$h4,$h4,24
+	adde	$h1,$h2,$h3
+	addc	$h1,$h1,$t0
+	addze	$h2,$h4
+
+	ld	$h3,0($ctx)	# load hash value base 2^64
+	ld	$h4,8($ctx)
+	ld	$t0,16($ctx)
+
+	neg	r0,r0
+	xor	$h0,$h0,$h3	# choose between radixes
+	xor	$h1,$h1,$h4
+	xor	$h2,$h2,$t0
+	and	$h0,$h0,r0
+	and	$h1,$h1,r0
+	and	$h2,$h2,r0
+	xor	$h0,$h0,$h3
+	xor	$h1,$h1,$h4
+	xor	$h2,$h2,$t0
+
+	addic	$h3,$h0,5	# compare to modulus
+	addze	$h4,$h1
+	addze	$t0,$h2
+
+	srdi	$t0,$t0,2	# see if it carried/borrowed
+	neg	$t0,$t0
+
+	andc	$h0,$h0,$t0
+	and	$h3,$h3,$t0
+	andc	$h1,$h1,$t0
+	and	$h4,$h4,$t0
+	or	$h0,$h0,$h3
+	or	$h1,$h1,$h4
+
+	lwz	$t0,4($nonce)
+	lwz	$h2,12($nonce)
+	lwz	$h3,0($nonce)
+	lwz	$h4,8($nonce)
+
+	insrdi	$h3,$t0,32,0
+	insrdi	$h4,$h2,32,0
+
+	addc	$h0,$h0,$h3	# accumulate nonce
+	adde	$h1,$h1,$h4
+
+	addi	$ctx,$mac,-1
+	addi	$mac,$mac,7
+
+	stbu	$h0,1($ctx)	# write [little-endian] result
+	srdi	$h0,$h0,8
+	stbu	$h1,1($mac)
+	srdi	$h1,$h1,8
+
+	stbu	$h0,1($ctx)
+	srdi	$h0,$h0,8
+	stbu	$h1,1($mac)
+	srdi	$h1,$h1,8
+
+	stbu	$h0,1($ctx)
+	srdi	$h0,$h0,8
+	stbu	$h1,1($mac)
+	srdi	$h1,$h1,8
+
+	stbu	$h0,1($ctx)
+	srdi	$h0,$h0,8
+	stbu	$h1,1($mac)
+	srdi	$h1,$h1,8
+
+	stbu	$h0,1($ctx)
+	srdi	$h0,$h0,8
+	stbu	$h1,1($mac)
+	srdi	$h1,$h1,8
+
+	stbu	$h0,1($ctx)
+	srdi	$h0,$h0,8
+	stbu	$h1,1($mac)
+	srdi	$h1,$h1,8
+
+	stbu	$h0,1($ctx)
+	srdi	$h0,$h0,8
+	stbu	$h1,1($mac)
+	srdi	$h1,$h1,8
+
+	stbu	$h0,1($ctx)
+	stbu	$h1,1($mac)
+
+	blr
+	.long	0
+	.byte	0,12,0x14,0,0,0,3,0
+.size	.poly1305_emit_int,.-.poly1305_emit_int
+___
+}							} else {
+###############################################################################
+# base 2^32 implementation
+
+my ($h0,$h1,$h2,$h3,$h4, $r0,$r1,$r2,$r3, $s1,$s2,$s3,
+    $t0,$t1,$t2,$t3, $D0,$D1,$D2,$D3, $d0,$d1,$d2,$d3
+   ) = map("r$_",(7..12,14..31));
+
+$code.=<<___;
+.globl	.poly1305_init_int
+.align	4
+.poly1305_init_int:
+	xor	r0,r0,r0
+	stw	r0,0($ctx)		# zero hash value
+	stw	r0,4($ctx)
+	stw	r0,8($ctx)
+	stw	r0,12($ctx)
+	stw	r0,16($ctx)
+	stw	r0,24($ctx)		# clear is_base2_26
+
+	$UCMP	$inp,r0
+	beq-	Lno_key
+___
+$code.=<<___	if ($LITTLE_ENDIAN);
+	lw	$h0,0($inp)		# load key material
+	lw	$h1,4($inp)
+	lw	$h2,8($inp)
+	lw	$h3,12($inp)
+___
+$code.=<<___	if (!$LITTLE_ENDIAN);
+	li	$h1,4
+	lwbrx	$h0,0,$inp		# load key material
+	li	$h2,8
+	lwbrx	$h1,$h1,$inp
+	li	$h3,12
+	lwbrx	$h2,$h2,$inp
+	lwbrx	$h3,$h3,$inp
+___
+$code.=<<___;
+	lis	$mask,0xf000		# 0xf0000000
+	li	$r0,-4
+	andc	$r0,$r0,$mask		# 0x0ffffffc
+
+	andc	$h0,$h0,$mask
+	and	$h1,$h1,$r0
+	and	$h2,$h2,$r0
+	and	$h3,$h3,$r0
+
+	stw	$h0,32($ctx)		# store key
+	stw	$h1,36($ctx)
+	stw	$h2,40($ctx)
+	stw	$h3,44($ctx)
+
+Lno_key:
+	xor	r3,r3,r3
+	blr
+	.long	0
+	.byte	0,12,0x14,0,0,0,2,0
+.size	.poly1305_init_int,.-.poly1305_init_int
+
+.globl	.poly1305_blocks_int
+.align	4
+.poly1305_blocks_int:
+Lpoly1305_blocks:
+	srwi.	$len,$len,4
+	beq-	Labort
+
+	$STU	$sp,-$FRAME($sp)
+	mflr	r0
+	$PUSH	r14,`$FRAME-$SIZE_T*18`($sp)
+	$PUSH	r15,`$FRAME-$SIZE_T*17`($sp)
+	$PUSH	r16,`$FRAME-$SIZE_T*16`($sp)
+	$PUSH	r17,`$FRAME-$SIZE_T*15`($sp)
+	$PUSH	r18,`$FRAME-$SIZE_T*14`($sp)
+	$PUSH	r19,`$FRAME-$SIZE_T*13`($sp)
+	$PUSH	r20,`$FRAME-$SIZE_T*12`($sp)
+	$PUSH	r21,`$FRAME-$SIZE_T*11`($sp)
+	$PUSH	r22,`$FRAME-$SIZE_T*10`($sp)
+	$PUSH	r23,`$FRAME-$SIZE_T*9`($sp)
+	$PUSH	r24,`$FRAME-$SIZE_T*8`($sp)
+	$PUSH	r25,`$FRAME-$SIZE_T*7`($sp)
+	$PUSH	r26,`$FRAME-$SIZE_T*6`($sp)
+	$PUSH	r27,`$FRAME-$SIZE_T*5`($sp)
+	$PUSH	r28,`$FRAME-$SIZE_T*4`($sp)
+	$PUSH	r29,`$FRAME-$SIZE_T*3`($sp)
+	$PUSH	r30,`$FRAME-$SIZE_T*2`($sp)
+	$PUSH	r31,`$FRAME-$SIZE_T*1`($sp)
+	$PUSH	r0,`$FRAME+$LRSAVE`($sp)
+
+	lwz	$r0,32($ctx)		# load key
+	lwz	$r1,36($ctx)
+	lwz	$r2,40($ctx)
+	lwz	$r3,44($ctx)
+
+	lwz	$h0,0($ctx)		# load hash value
+	lwz	$h1,4($ctx)
+	lwz	$h2,8($ctx)
+	lwz	$h3,12($ctx)
+	lwz	$h4,16($ctx)
+
+	srwi	$s1,$r1,2
+	srwi	$s2,$r2,2
+	srwi	$s3,$r3,2
+	add	$s1,$s1,$r1		# si = ri + ri>>2
+	add	$s2,$s2,$r2
+	add	$s3,$s3,$r3
+	mtctr	$len
+	li	$mask,3
+	b	Loop
+
+.align	4
+Loop:
+___
+$code.=<<___	if ($LITTLE_ENDIAN);
+	lwz	$d0,0($inp)		# load input
+	lwz	$d1,4($inp)
+	lwz	$d2,8($inp)
+	lwz	$d3,12($inp)
+___
+$code.=<<___	if (!$LITTLE_ENDIAN);
+	li	$d1,4
+	lwbrx	$d0,0,$inp		# load input
+	li	$d2,8
+	lwbrx	$d1,$d1,$inp
+	li	$d3,12
+	lwbrx	$d2,$d2,$inp
+	lwbrx	$d3,$d3,$inp
+___
+$code.=<<___;
+	addi	$inp,$inp,16
+
+	addc	$h0,$h0,$d0		# accumulate input
+	adde	$h1,$h1,$d1
+	adde	$h2,$h2,$d2
+
+	mullw	$d0,$h0,$r0		# h0*r0
+	mulhwu	$D0,$h0,$r0
+
+	mullw	$d1,$h0,$r1		# h0*r1
+	mulhwu	$D1,$h0,$r1
+
+	mullw	$d2,$h0,$r2		# h0*r2
+	mulhwu	$D2,$h0,$r2
+
+	 adde	$h3,$h3,$d3
+	 adde	$h4,$h4,$padbit
+
+	mullw	$d3,$h0,$r3		# h0*r3
+	mulhwu	$D3,$h0,$r3
+
+	mullw	$t0,$h1,$s3		# h1*s3
+	mulhwu	$t1,$h1,$s3
+
+	mullw	$t2,$h1,$r0		# h1*r0
+	mulhwu	$t3,$h1,$r0
+	 addc	$d0,$d0,$t0
+	 adde	$D0,$D0,$t1
+
+	mullw	$t0,$h1,$r1		# h1*r1
+	mulhwu	$t1,$h1,$r1
+	 addc	$d1,$d1,$t2
+	 adde	$D1,$D1,$t3
+
+	mullw	$t2,$h1,$r2		# h1*r2
+	mulhwu	$t3,$h1,$r2
+	 addc	$d2,$d2,$t0
+	 adde	$D2,$D2,$t1
+
+	mullw	$t0,$h2,$s2		# h2*s2
+	mulhwu	$t1,$h2,$s2
+	 addc	$d3,$d3,$t2
+	 adde	$D3,$D3,$t3
+
+	mullw	$t2,$h2,$s3		# h2*s3
+	mulhwu	$t3,$h2,$s3
+	 addc	$d0,$d0,$t0
+	 adde	$D0,$D0,$t1
+
+	mullw	$t0,$h2,$r0		# h2*r0
+	mulhwu	$t1,$h2,$r0
+	 addc	$d1,$d1,$t2
+	 adde	$D1,$D1,$t3
+
+	mullw	$t2,$h2,$r1		# h2*r1
+	mulhwu	$t3,$h2,$r1
+	 addc	$d2,$d2,$t0
+	 adde	$D2,$D2,$t1
+
+	mullw	$t0,$h3,$s1		# h3*s1
+	mulhwu	$t1,$h3,$s1
+	 addc	$d3,$d3,$t2
+	 adde	$D3,$D3,$t3
+
+	mullw	$t2,$h3,$s2		# h3*s2
+	mulhwu	$t3,$h3,$s2
+	 addc	$d0,$d0,$t0
+	 adde	$D0,$D0,$t1
+
+	mullw	$t0,$h3,$s3		# h3*s3
+	mulhwu	$t1,$h3,$s3
+	 addc	$d1,$d1,$t2
+	 adde	$D1,$D1,$t3
+
+	mullw	$t2,$h3,$r0		# h3*r0
+	mulhwu	$t3,$h3,$r0
+	 addc	$d2,$d2,$t0
+	 adde	$D2,$D2,$t1
+
+	mullw	$t0,$h4,$s1		# h4*s1
+	 addc	$d3,$d3,$t2
+	 adde	$D3,$D3,$t3
+	addc	$d1,$d1,$t0
+
+	mullw	$t1,$h4,$s2		# h4*s2
+	 addze	$D1,$D1
+	addc	$d2,$d2,$t1
+	addze	$D2,$D2
+
+	mullw	$t2,$h4,$s3		# h4*s3
+	addc	$d3,$d3,$t2
+	addze	$D3,$D3
+
+	mullw	$h4,$h4,$r0		# h4*r0
+
+	addc	$h1,$d1,$D0
+	adde	$h2,$d2,$D1
+	adde	$h3,$d3,$D2
+	adde	$h4,$h4,$D3
+
+	andc	$D0,$h4,$mask		# final reduction step
+	and	$h4,$h4,$mask
+	srwi	$D1,$D0,2
+	add	$D0,$D0,$D1
+	addc	$h0,$d0,$D0
+	addze	$h1,$h1
+	addze	$h2,$h2
+	addze	$h3,$h3
+	addze	$h4,$h4
+
+	bdnz	Loop
+
+	stw	$h0,0($ctx)		# store hash value
+	stw	$h1,4($ctx)
+	stw	$h2,8($ctx)
+	stw	$h3,12($ctx)
+	stw	$h4,16($ctx)
+
+	$POP	r14,`$FRAME-$SIZE_T*18`($sp)
+	$POP	r15,`$FRAME-$SIZE_T*17`($sp)
+	$POP	r16,`$FRAME-$SIZE_T*16`($sp)
+	$POP	r17,`$FRAME-$SIZE_T*15`($sp)
+	$POP	r18,`$FRAME-$SIZE_T*14`($sp)
+	$POP	r19,`$FRAME-$SIZE_T*13`($sp)
+	$POP	r20,`$FRAME-$SIZE_T*12`($sp)
+	$POP	r21,`$FRAME-$SIZE_T*11`($sp)
+	$POP	r22,`$FRAME-$SIZE_T*10`($sp)
+	$POP	r23,`$FRAME-$SIZE_T*9`($sp)
+	$POP	r24,`$FRAME-$SIZE_T*8`($sp)
+	$POP	r25,`$FRAME-$SIZE_T*7`($sp)
+	$POP	r26,`$FRAME-$SIZE_T*6`($sp)
+	$POP	r27,`$FRAME-$SIZE_T*5`($sp)
+	$POP	r28,`$FRAME-$SIZE_T*4`($sp)
+	$POP	r29,`$FRAME-$SIZE_T*3`($sp)
+	$POP	r30,`$FRAME-$SIZE_T*2`($sp)
+	$POP	r31,`$FRAME-$SIZE_T*1`($sp)
+	addi	$sp,$sp,$FRAME
+Labort:
+	blr
+	.long	0
+	.byte	0,12,4,1,0x80,18,4,0
+.size	.poly1305_blocks_int,.-.poly1305_blocks_int
+___
+{
+my ($h0,$h1,$h2,$h3,$h4,$t0,$t1) = map("r$_",(6..12));
+
+$code.=<<___;
+.globl	.poly1305_emit_int
+.align	5
+.poly1305_emit_int:
+	lwz	r0,24($ctx)	# is_base2_26
+	lwz	$h0,0($ctx)	# load hash value
+	lwz	$h1,4($ctx)
+	lwz	$h2,8($ctx)
+	lwz	$h3,12($ctx)
+	lwz	$h4,16($ctx)
+	cmplwi	r0,0
+	beq	Lemit_base2_32
+
+	slwi	$t0,$h1,26	# base 2^26 -> base 2^32
+	srwi	$h1,$h1,6
+	slwi	$t1,$h2,20
+	srwi	$h2,$h2,12
+	addc	$h0,$h0,$t0
+	slwi	$t0,$h3,14
+	srwi	$h3,$h3,18
+	adde	$h1,$h1,$t1
+	slwi	$t1,$h4,8
+	srwi	$h4,$h4,24
+	adde	$h2,$h2,$t0
+	adde	$h3,$h3,$t1
+	addze	$h4,$h4
+
+Lemit_base2_32:
+	addic	r0,$h0,5	# compare to modulus
+	addze	r0,$h1
+	addze	r0,$h2
+	addze	r0,$h3
+	addze	r0,$h4
+
+	srwi	r0,r0,2		# see if it carried/borrowed
+	neg	r0,r0
+	andi.	r0,r0,5
+
+	addc	$h0,$h0,r0
+	lwz	r0,0($nonce)
+	addze	$h1,$h1
+	lwz	$t0,4($nonce)
+	addze	$h2,$h2
+	lwz	$t1,8($nonce)
+	addze	$h3,$h3
+	lwz	$h4,12($nonce)
+
+	addc	$h0,$h0,r0	# accumulate nonce
+	adde	$h1,$h1,$t0
+	adde	$h2,$h2,$t1
+	adde	$h3,$h3,$h4
+
+	addi	$ctx,$mac,-1
+	addi	$mac,$mac,7
+
+	stbu	$h0,1($ctx)	# write [little-endian] result
+	srwi	$h0,$h0,8
+	stbu	$h2,1($mac)
+	srwi	$h2,$h2,8
+
+	stbu	$h0,1($ctx)
+	srwi	$h0,$h0,8
+	stbu	$h2,1($mac)
+	srwi	$h2,$h2,8
+
+	stbu	$h0,1($ctx)
+	srwi	$h0,$h0,8
+	stbu	$h2,1($mac)
+	srwi	$h2,$h2,8
+
+	stbu	$h0,1($ctx)
+	stbu	$h2,1($mac)
+
+	stbu	$h1,1($ctx)
+	srwi	$h1,$h1,8
+	stbu	$h3,1($mac)
+	srwi	$h3,$h3,8
+
+	stbu	$h1,1($ctx)
+	srwi	$h1,$h1,8
+	stbu	$h3,1($mac)
+	srwi	$h3,$h3,8
+
+	stbu	$h1,1($ctx)
+	srwi	$h1,$h1,8
+	stbu	$h3,1($mac)
+	srwi	$h3,$h3,8
+
+	stbu	$h1,1($ctx)
+	stbu	$h3,1($mac)
+
+	blr
+	.long	0
+	.byte	0,12,0x14,0,0,0,3,0
+.size	.poly1305_emit_int,.-.poly1305_emit_int
+___
+}							}
+{{{
+########################################################################
+# PowerISA 2.07/VSX section                                            #
+########################################################################
+
+my $LOCALS= 6*$SIZE_T;
+my $VSXFRAME = $LOCALS + 6*$SIZE_T;
+   $VSXFRAME += 128;	# local variables
+   $VSXFRAME += 13*16;	# v20-v31 offload
+
+my $BIG_ENDIAN = ($flavour !~ /le/) ? 4 : 0;
+
+########################################################################
+# Layout of opaque area is following:
+#
+#	unsigned __int32 h[5];		# current hash value base 2^26
+#	unsigned __int32 pad;
+#	unsigned __int32 is_base2_26, pad;
+#	unsigned __int64 r[2];		# key value base 2^64
+#	struct { unsigned __int32 r^2, r^4, r^1, r^3; } r[9];
+#
+# where r^n are base 2^26 digits of powers of multiplier key. There are
+# 5 digits, but last four are interleaved with multiples of 5, totalling
+# in 9 elements: r0, r1, 5*r1, r2, 5*r2, r3, 5*r3, r4, 5*r4. Order of
+# powers is as they appear in register, not memory.
+
+my ($H0, $H1, $H2, $H3, $H4) = map("v$_",(0..4));
+my ($I0, $I1, $I2, $I3, $I4) = map("v$_",(5..9));
+my ($R0, $R1, $S1, $R2, $S2) = map("v$_",(10..14));
+my      ($R3, $S3, $R4, $S4) = ($R1, $S1, $R2, $S2);
+my ($ACC0, $ACC1, $ACC2, $ACC3, $ACC4) = map("v$_",(15..19));
+my ($T0, $T1, $T2, $T3, $T4) = map("v$_",(20..24));
+my ($_26,$_4,$_40,$_14,$mask26,$padbits,$I2perm) = map("v$_",(25..31));
+my ($x00,$x60,$x70,$x10,$x20,$x30,$x40,$x50) = (0, map("r$_",(7,8,27..31)));
+my ($ctx_,$_ctx,$const) = map("r$_",(10..12));
+
+							if ($flavour =~ /64/) {
+###############################################################################
+# setup phase of poly1305_blocks_vsx is different on 32- and 64-bit platforms,
+# but the base 2^26 computational part is same...
+
+my ($h0,$h1,$h2,$d0,$d1,$d2, $r0,$r1,$s1, $t0,$t1) = map("r$_",(6..11,27..31));
+my $mask = "r0";
+
+$code.=<<___;
+.globl	.poly1305_blocks_vsx
+.align	5
+.poly1305_blocks_vsx:
+	lwz	r7,24($ctx)		# is_base2_26
+	cmpldi	$len,128
+	bge	__poly1305_blocks_vsx
+
+	neg	r0,r7			# is_base2_26 as mask
+	lwz	r7,0($ctx)		# load hash base 2^26
+	lwz	r8,4($ctx)
+	lwz	r9,8($ctx)
+	lwz	r10,12($ctx)
+	lwz	r11,16($ctx)
+
+	sldi	r8,r8,26		# base 2^26 -> base 2^64
+	sldi	r12,r9,52
+	add	r7,r7,r8
+	srdi	r9,r9,12
+	sldi	r10,r10,14
+	addc	r7,r7,r12
+	sldi	r8,r11,40
+	adde	r9,r9,r10
+	srdi	r11,r11,24
+	addc	r9,r9,r8
+	addze	r11,r11
+
+	ld	r8,0($ctx)		# load hash base 2^64
+	ld	r10,8($ctx)
+	ld	r12,16($ctx)
+
+	xor	r7,r7,r8		# select between radixes
+	xor	r9,r9,r10
+	xor	r11,r11,r12
+	and	r7,r7,r0
+	and	r9,r9,r0
+	and	r11,r11,r0
+	xor	r7,r7,r8
+	xor	r9,r9,r10
+	xor	r11,r11,r12
+
+	li	r0,0
+	std	r7,0($ctx)		# store hash base 2^64
+	std	r9,8($ctx)
+	std	r11,16($ctx)
+	stw	r0,24($ctx)		# clear is_base2_26
+
+	b	Lpoly1305_blocks
+	.long	0
+	.byte	0,12,0x14,0,0,0,4,0
+.size	.poly1305_blocks_vsx,.-.poly1305_blocks_vsx
+
+.align	5
+__poly1305_mul:
+	mulld	$d0,$h0,$r0		# h0*r0
+	mulhdu	$d1,$h0,$r0
+
+	mulld	$t0,$h1,$s1		# h1*5*r1
+	mulhdu	$t1,$h1,$s1
+	addc	$d0,$d0,$t0
+	adde	$d1,$d1,$t1
+
+	mulld	$t0,$h0,$r1		# h0*r1
+	mulhdu	$d2,$h0,$r1
+	addc	$d1,$d1,$t0
+	addze	$d2,$d2
+
+	mulld	$t0,$h1,$r0		# h1*r0
+	mulhdu	$t1,$h1,$r0
+	addc	$d1,$d1,$t0
+	adde	$d2,$d2,$t1
+
+	mulld	$t0,$h2,$s1		# h2*5*r1
+	mulld	$t1,$h2,$r0		# h2*r0
+	addc	$d1,$d1,$t0
+	adde	$d2,$d2,$t1
+
+	andc	$t0,$d2,$mask		# final reduction step
+	and	$h2,$d2,$mask
+	srdi	$t1,$t0,2
+	add	$t0,$t0,$t1
+	addc	$h0,$d0,$t0
+	addze	$h1,$d1
+	addze	$h2,$h2
+
+	blr
+	.long	0
+	.byte	0,12,0x14,0,0,0,0,0
+.size	__poly1305_mul,.-__poly1305_mul
+
+.align	5
+__poly1305_splat:
+	extrdi	$d0,$h0,26,38
+	extrdi	$d1,$h0,26,12
+	stw	$d0,0x00($t1)
+
+	extrdi	$d2,$h0,12,0
+	slwi	$d0,$d1,2
+	stw	$d1,0x10($t1)
+	add	$d0,$d0,$d1		# * 5
+	stw	$d0,0x20($t1)
+
+	insrdi	$d2,$h1,14,38
+	slwi	$d0,$d2,2
+	stw	$d2,0x30($t1)
+	add	$d0,$d0,$d2		# * 5
+	stw	$d0,0x40($t1)
+
+	extrdi	$d1,$h1,26,24
+	extrdi	$d2,$h1,24,0
+	slwi	$d0,$d1,2
+	stw	$d1,0x50($t1)
+	add	$d0,$d0,$d1		# * 5
+	stw	$d0,0x60($t1)
+
+	insrdi	$d2,$h2,3,37
+	slwi	$d0,$d2,2
+	stw	$d2,0x70($t1)
+	add	$d0,$d0,$d2		# * 5
+	stw	$d0,0x80($t1)
+
+	blr
+	.long	0
+	.byte	0,12,0x14,0,0,0,0,0
+.size	__poly1305_splat,.-__poly1305_splat
+
+.align	5
+__poly1305_blocks_vsx:
+	$STU	$sp,-$VSXFRAME($sp)
+	mflr	r0
+	li	r10,`15+$LOCALS+128`
+	li	r11,`31+$LOCALS+128`
+	mfspr	r12,256
+	stvx	v20,r10,$sp
+	addi	r10,r10,32
+	stvx	v21,r11,$sp
+	addi	r11,r11,32
+	stvx	v22,r10,$sp
+	addi	r10,r10,32
+	stvx	v23,r10,$sp
+	addi	r10,r10,32
+	stvx	v24,r11,$sp
+	addi	r11,r11,32
+	stvx	v25,r10,$sp
+	addi	r10,r10,32
+	stvx	v26,r10,$sp
+	addi	r10,r10,32
+	stvx	v27,r11,$sp
+	addi	r11,r11,32
+	stvx	v28,r10,$sp
+	addi	r10,r10,32
+	stvx	v29,r11,$sp
+	addi	r11,r11,32
+	stvx	v30,r10,$sp
+	stvx	v31,r11,$sp
+	stw	r12,`$VSXFRAME-$SIZE_T*5-4`($sp)# save vrsave
+	li	r12,-1
+	mtspr	256,r12			# preserve all AltiVec registers
+	$PUSH	r27,`$VSXFRAME-$SIZE_T*5`($sp)
+	$PUSH	r28,`$VSXFRAME-$SIZE_T*4`($sp)
+	$PUSH	r29,`$VSXFRAME-$SIZE_T*3`($sp)
+	$PUSH	r30,`$VSXFRAME-$SIZE_T*2`($sp)
+	$PUSH	r31,`$VSXFRAME-$SIZE_T*1`($sp)
+	$PUSH	r0,`$VSXFRAME+$LRSAVE`($sp)
+
+	bl	LPICmeup
+
+	li	$x10,0x10
+	li	$x20,0x20
+	li	$x30,0x30
+	li	$x40,0x40
+	li	$x50,0x50
+	lvx_u	$mask26,$x00,$const
+	lvx_u	$_26,$x10,$const
+	lvx_u	$_40,$x20,$const
+	lvx_u	$I2perm,$x30,$const
+	lvx_u	$padbits,$x40,$const
+
+	cmplwi	r7,0			# is_base2_26?
+	bne	Lskip_init_vsx
+
+	ld	$r0,32($ctx)		# load key base 2^64
+	ld	$r1,40($ctx)
+	srdi	$s1,$r1,2
+	li	$mask,3
+	add	$s1,$s1,$r1		# s1 = r1 + r1>>2
+
+	mr	$h0,$r0			# "calculate" r^1
+	mr	$h1,$r1
+	li	$h2,0
+	addi	$t1,$ctx,`48+(12^$BIG_ENDIAN)`
+	bl	__poly1305_splat
+
+	bl	__poly1305_mul		# caclulate r^2
+	addi	$t1,$ctx,`48+(4^$BIG_ENDIAN)`
+	bl	__poly1305_splat
+
+	bl	__poly1305_mul		# caclulate r^3
+	addi	$t1,$ctx,`48+(8^$BIG_ENDIAN)`
+	bl	__poly1305_splat
+
+	bl	__poly1305_mul		# caclulate r^4
+	addi	$t1,$ctx,`48+(0^$BIG_ENDIAN)`
+	bl	__poly1305_splat
+
+	ld	$h0,0($ctx)		# load hash
+	ld	$h1,8($ctx)
+	ld	$h2,16($ctx)
+
+	extrdi	$d0,$h0,26,38		# base 2^64 -> base 2^26
+	extrdi	$d1,$h0,26,12
+	extrdi	$d2,$h0,12,0
+	mtvrwz	$H0,$d0
+	insrdi	$d2,$h1,14,38
+	mtvrwz	$H1,$d1
+	extrdi	$d1,$h1,26,24
+	mtvrwz	$H2,$d2
+	extrdi	$d2,$h1,24,0
+	mtvrwz	$H3,$d1
+	insrdi	$d2,$h2,3,37
+	mtvrwz	$H4,$d2
+___
+							} else {
+###############################################################################
+# 32-bit initialization
+
+my ($h0,$h1,$h2,$h3,$h4,$t0,$t1) = map("r$_",(7..11,0,12));
+my ($R3,$S3,$R4,$S4)=($I1,$I2,$I3,$I4);
+
+$code.=<<___;
+.globl	.poly1305_blocks_vsx
+.align	5
+.poly1305_blocks_vsx:
+	lwz	r7,24($ctx)		# is_base2_26
+	cmplwi	$len,128
+	bge	__poly1305_blocks_vsx
+	cmplwi	r7,0
+	beq	Lpoly1305_blocks
+
+	lwz	$h0,0($ctx)		# load hash
+	lwz	$h1,4($ctx)
+	lwz	$h2,8($ctx)
+	lwz	$h3,12($ctx)
+	lwz	$h4,16($ctx)
+
+	slwi	$t0,$h1,26		# base 2^26 -> base 2^32
+	srwi	$h1,$h1,6
+	slwi	$t1,$h2,20
+	srwi	$h2,$h2,12
+	addc	$h0,$h0,$t0
+	slwi	$t0,$h3,14
+	srwi	$h3,$h3,18
+	adde	$h1,$h1,$t1
+	slwi	$t1,$h4,8
+	srwi	$h4,$h4,24
+	adde	$h2,$h2,$t0
+	li	$t0,0
+	adde	$h3,$h3,$t1
+	addze	$h4,$h4
+
+	stw	$h0,0($ctx)		# store hash base 2^32
+	stw	$h1,4($ctx)
+	stw	$h2,8($ctx)
+	stw	$h3,12($ctx)
+	stw	$h4,16($ctx)
+	stw	$t0,24($ctx)		# clear is_base2_26
+
+	b	Lpoly1305_blocks
+	.long	0
+	.byte	0,12,0x14,0,0,0,4,0
+.size	.poly1305_blocks_vsx,.-.poly1305_blocks_vsx
+
+.align	5
+__poly1305_mul:
+	vmulouw		$ACC0,$H0,$R0
+	vmulouw		$ACC1,$H1,$R0
+	vmulouw		$ACC2,$H2,$R0
+	vmulouw		$ACC3,$H3,$R0
+	vmulouw		$ACC4,$H4,$R0
+
+	vmulouw		$T0,$H4,$S1
+	vaddudm		$ACC0,$ACC0,$T0
+	vmulouw		$T0,$H0,$R1
+	vaddudm		$ACC1,$ACC1,$T0
+	vmulouw		$T0,$H1,$R1
+	vaddudm		$ACC2,$ACC2,$T0
+	vmulouw		$T0,$H2,$R1
+	vaddudm		$ACC3,$ACC3,$T0
+	vmulouw		$T0,$H3,$R1
+	vaddudm		$ACC4,$ACC4,$T0
+
+	vmulouw		$T0,$H3,$S2
+	vaddudm		$ACC0,$ACC0,$T0
+	vmulouw		$T0,$H4,$S2
+	vaddudm		$ACC1,$ACC1,$T0
+	vmulouw		$T0,$H0,$R2
+	vaddudm		$ACC2,$ACC2,$T0
+	vmulouw		$T0,$H1,$R2
+	vaddudm		$ACC3,$ACC3,$T0
+	vmulouw		$T0,$H2,$R2
+	vaddudm		$ACC4,$ACC4,$T0
+
+	vmulouw		$T0,$H2,$S3
+	vaddudm		$ACC0,$ACC0,$T0
+	vmulouw		$T0,$H3,$S3
+	vaddudm		$ACC1,$ACC1,$T0
+	vmulouw		$T0,$H4,$S3
+	vaddudm		$ACC2,$ACC2,$T0
+	vmulouw		$T0,$H0,$R3
+	vaddudm		$ACC3,$ACC3,$T0
+	vmulouw		$T0,$H1,$R3
+	vaddudm		$ACC4,$ACC4,$T0
+
+	vmulouw		$T0,$H1,$S4
+	vaddudm		$ACC0,$ACC0,$T0
+	vmulouw		$T0,$H2,$S4
+	vaddudm		$ACC1,$ACC1,$T0
+	vmulouw		$T0,$H3,$S4
+	vaddudm		$ACC2,$ACC2,$T0
+	vmulouw		$T0,$H4,$S4
+	vaddudm		$ACC3,$ACC3,$T0
+	vmulouw		$T0,$H0,$R4
+	vaddudm		$ACC4,$ACC4,$T0
+
+	################################################################
+	# lazy reduction
+
+	vspltisb	$T0,2
+	vsrd		$H4,$ACC3,$_26
+	vsrd		$H1,$ACC0,$_26
+	vand		$H3,$ACC3,$mask26
+	vand		$H0,$ACC0,$mask26
+	vaddudm		$H4,$H4,$ACC4		# h3 -> h4
+	vaddudm		$H1,$H1,$ACC1		# h0 -> h1
+
+	vsrd		$ACC4,$H4,$_26
+	vsrd		$ACC1,$H1,$_26
+	vand		$H4,$H4,$mask26
+	vand		$H1,$H1,$mask26
+	vaddudm		$H0,$H0,$ACC4
+	vaddudm		$H2,$ACC2,$ACC1		# h1 -> h2
+
+	vsld		$ACC4,$ACC4,$T0		# <<2
+	vsrd		$ACC2,$H2,$_26
+	vand		$H2,$H2,$mask26
+	vaddudm		$H0,$H0,$ACC4		# h4 -> h0
+	vaddudm		$H3,$H3,$ACC2		# h2 -> h3
+
+	vsrd		$ACC0,$H0,$_26
+	vsrd		$ACC3,$H3,$_26
+	vand		$H0,$H0,$mask26
+	vand		$H3,$H3,$mask26
+	vaddudm		$H1,$H1,$ACC0		# h0 -> h1
+	vaddudm		$H4,$H4,$ACC3		# h3 -> h4
+
+	blr
+	.long	0
+	.byte	0,12,0x14,0,0,0,0,0
+.size	__poly1305_mul,.-__poly1305_mul
+
+.align	5
+__poly1305_blocks_vsx:
+	$STU	$sp,-$VSXFRAME($sp)
+	mflr	r0
+	li	r10,`15+$LOCALS+128`
+	li	r11,`31+$LOCALS+128`
+	mfspr	r12,256
+	stvx	v20,r10,$sp
+	addi	r10,r10,32
+	stvx	v21,r11,$sp
+	addi	r11,r11,32
+	stvx	v22,r10,$sp
+	addi	r10,r10,32
+	stvx	v23,r10,$sp
+	addi	r10,r10,32
+	stvx	v24,r11,$sp
+	addi	r11,r11,32
+	stvx	v25,r10,$sp
+	addi	r10,r10,32
+	stvx	v26,r10,$sp
+	addi	r10,r10,32
+	stvx	v27,r11,$sp
+	addi	r11,r11,32
+	stvx	v28,r10,$sp
+	addi	r10,r10,32
+	stvx	v29,r11,$sp
+	addi	r11,r11,32
+	stvx	v30,r10,$sp
+	stvx	v31,r11,$sp
+	stw	r12,`$VSXFRAME-$SIZE_T*5-4`($sp)# save vrsave
+	li	r12,-1
+	mtspr	256,r12			# preserve all AltiVec registers
+	$PUSH	r27,`$VSXFRAME-$SIZE_T*5`($sp)
+	$PUSH	r28,`$VSXFRAME-$SIZE_T*4`($sp)
+	$PUSH	r29,`$VSXFRAME-$SIZE_T*3`($sp)
+	$PUSH	r30,`$VSXFRAME-$SIZE_T*2`($sp)
+	$PUSH	r31,`$VSXFRAME-$SIZE_T*1`($sp)
+	$PUSH	r0,`$VSXFRAME+$LRSAVE`($sp)
+
+	bl	LPICmeup
+
+	li	$x10,0x10
+	li	$x20,0x20
+	li	$x30,0x30
+	li	$x40,0x40
+	li	$x50,0x50
+	lvx_u	$mask26,$x00,$const
+	lvx_u	$_26,$x10,$const
+	lvx_u	$_40,$x20,$const
+	lvx_u	$I2perm,$x30,$const
+	lvx_u	$padbits,$x40,$const
+
+	cmplwi	r7,0			# is_base2_26?
+	bne	Lskip_init_vsx
+
+	lwz	$h1,32($ctx)		# load key base 2^32
+	lwz	$h2,36($ctx)
+	lwz	$h3,40($ctx)
+	lwz	$h4,44($ctx)
+
+	extrwi	$h0,$h1,26,6		# base 2^32 -> base 2^26
+	extrwi	$h1,$h1,6,0
+	insrwi	$h1,$h2,20,6
+	extrwi	$h2,$h2,12,0
+	insrwi	$h2,$h3,14,6
+	extrwi	$h3,$h3,18,0
+	insrwi	$h3,$h4,8,6
+	extrwi	$h4,$h4,24,0
+
+	mtvrwz	$R0,$h0
+	slwi	$h0,$h1,2
+	mtvrwz	$R1,$h1
+	add	$h1,$h1,$h0
+	mtvrwz	$S1,$h1
+	slwi	$h1,$h2,2
+	mtvrwz	$R2,$h2
+	add	$h2,$h2,$h1
+	mtvrwz	$S2,$h2
+	slwi	$h2,$h3,2
+	mtvrwz	$R3,$h3
+	add	$h3,$h3,$h2
+	mtvrwz	$S3,$h3
+	slwi	$h3,$h4,2
+	mtvrwz	$R4,$h4
+	add	$h4,$h4,$h3
+	mtvrwz	$S4,$h4
+
+	vmr	$H0,$R0
+	vmr	$H1,$R1
+	vmr	$H2,$R2
+	vmr	$H3,$R3
+	vmr	$H4,$R4
+
+	bl	__poly1305_mul		# r^1:- * r^1:-
+
+	vpermdi	$R0,$H0,$R0,0b00
+	vpermdi	$R1,$H1,$R1,0b00
+	vpermdi	$R2,$H2,$R2,0b00
+	vpermdi	$R3,$H3,$R3,0b00
+	vpermdi	$R4,$H4,$R4,0b00
+	vpermdi	$H0,$H0,$H0,0b00
+	vpermdi	$H1,$H1,$H1,0b00
+	vpermdi	$H2,$H2,$H2,0b00
+	vpermdi	$H3,$H3,$H3,0b00
+	vpermdi	$H4,$H4,$H4,0b00
+	vsld	$S1,$R1,$T0		# <<2
+	vsld	$S2,$R2,$T0
+	vsld	$S3,$R3,$T0
+	vsld	$S4,$R4,$T0
+	vaddudm	$S1,$S1,$R1
+	vaddudm	$S2,$S2,$R2
+	vaddudm	$S3,$S3,$R3
+	vaddudm	$S4,$S4,$R4
+
+	bl	__poly1305_mul		# r^2:r^2 * r^2:r^1
+
+	addi	$h0,$ctx,0x60
+	lwz	$h1,0($ctx)		# load hash
+	lwz	$h2,4($ctx)
+	lwz	$h3,8($ctx)
+	lwz	$h4,12($ctx)
+	lwz	$t0,16($ctx)
+
+	vmrgow	$R0,$R0,$H0		# r^2:r^4:r^1:r^3
+	vmrgow	$R1,$R1,$H1
+	vmrgow	$R2,$R2,$H2
+	vmrgow	$R3,$R3,$H3
+	vmrgow	$R4,$R4,$H4
+	vslw	$S1,$R1,$T0		# <<2
+	vslw	$S2,$R2,$T0
+	vslw	$S3,$R3,$T0
+	vslw	$S4,$R4,$T0
+	vadduwm	$S1,$S1,$R1
+	vadduwm	$S2,$S2,$R2
+	vadduwm	$S3,$S3,$R3
+	vadduwm	$S4,$S4,$R4
+
+	stvx_u	$R0,$x30,$ctx
+	stvx_u	$R1,$x40,$ctx
+	stvx_u	$S1,$x50,$ctx
+	stvx_u	$R2,$x00,$h0
+	stvx_u	$S2,$x10,$h0
+	stvx_u	$R3,$x20,$h0
+	stvx_u	$S3,$x30,$h0
+	stvx_u	$R4,$x40,$h0
+	stvx_u	$S4,$x50,$h0
+
+	extrwi	$h0,$h1,26,6		# base 2^32 -> base 2^26
+	extrwi	$h1,$h1,6,0
+	mtvrwz	$H0,$h0
+	insrwi	$h1,$h2,20,6
+	extrwi	$h2,$h2,12,0
+	mtvrwz	$H1,$h1
+	insrwi	$h2,$h3,14,6
+	extrwi	$h3,$h3,18,0
+	mtvrwz	$H2,$h2
+	insrwi	$h3,$h4,8,6
+	extrwi	$h4,$h4,24,0
+	mtvrwz	$H3,$h3
+	insrwi	$h4,$t0,3,5
+	mtvrwz	$H4,$h4
+___
+							}
+$code.=<<___;
+	li	r0,1
+	stw	r0,24($ctx)		# set is_base2_26
+	b	Loaded_vsx
+
+.align	4
+Lskip_init_vsx:
+	li		$x10,4
+	li		$x20,8
+	li		$x30,12
+	li		$x40,16
+	lvwzx_u		$H0,$x00,$ctx
+	lvwzx_u		$H1,$x10,$ctx
+	lvwzx_u		$H2,$x20,$ctx
+	lvwzx_u		$H3,$x30,$ctx
+	lvwzx_u		$H4,$x40,$ctx
+
+Loaded_vsx:
+	li		$x10,0x10
+	li		$x20,0x20
+	li		$x30,0x30
+	li		$x40,0x40
+	li		$x50,0x50
+	li		$x60,0x60
+	li		$x70,0x70
+	addi		$ctx_,$ctx,64		# &ctx->r[1]
+	addi		$_ctx,$sp,`$LOCALS+15`	# &ctx->r[1], r^2:r^4 shadow
+
+	vxor		$T0,$T0,$T0		# ensure second half is zero
+	vpermdi		$H0,$H0,$T0,0b00
+	vpermdi		$H1,$H1,$T0,0b00
+	vpermdi		$H2,$H2,$T0,0b00
+	vpermdi		$H3,$H3,$T0,0b00
+	vpermdi		$H4,$H4,$T0,0b00
+
+	be?lvx_u	$_4,$x50,$const		# byte swap mask
+	lvx_u		$T1,$x00,$inp		# load first input block
+	lvx_u		$T2,$x10,$inp
+	lvx_u		$T3,$x20,$inp
+	lvx_u		$T4,$x30,$inp
+	be?vperm	$T1,$T1,$T1,$_4
+	be?vperm	$T2,$T2,$T2,$_4
+	be?vperm	$T3,$T3,$T3,$_4
+	be?vperm	$T4,$T4,$T4,$_4
+
+	vpermdi		$I0,$T1,$T2,0b00	# smash input to base 2^26
+	vspltisb	$_4,4
+	vperm		$I2,$T1,$T2,$I2perm	# 0x...0e0f0001...1e1f1011
+	vspltisb	$_14,14
+	vpermdi		$I3,$T1,$T2,0b11
+
+	vsrd		$I1,$I0,$_26
+	vsrd		$I2,$I2,$_4
+	vsrd		$I4,$I3,$_40
+	vsrd		$I3,$I3,$_14
+	vand		$I0,$I0,$mask26
+	vand		$I1,$I1,$mask26
+	vand		$I2,$I2,$mask26
+	vand		$I3,$I3,$mask26
+
+	vpermdi		$T1,$T3,$T4,0b00
+	vperm		$T2,$T3,$T4,$I2perm	# 0x...0e0f0001...1e1f1011
+	vpermdi		$T3,$T3,$T4,0b11
+
+	vsrd		$T0,$T1,$_26
+	vsrd		$T2,$T2,$_4
+	vsrd		$T4,$T3,$_40
+	vsrd		$T3,$T3,$_14
+	vand		$T1,$T1,$mask26
+	vand		$T0,$T0,$mask26
+	vand		$T2,$T2,$mask26
+	vand		$T3,$T3,$mask26
+
+	# inp[2]:inp[0]:inp[3]:inp[1]
+	vmrgow		$I4,$T4,$I4
+	vmrgow		$I0,$T1,$I0
+	vmrgow		$I1,$T0,$I1
+	vmrgow		$I2,$T2,$I2
+	vmrgow		$I3,$T3,$I3
+	vor		$I4,$I4,$padbits
+
+	lvx_splt	$R0,$x30,$ctx		# taking lvx_vsplt out of loop
+	lvx_splt	$R1,$x00,$ctx_		# gives ~8% improvement
+	lvx_splt	$S1,$x10,$ctx_
+	lvx_splt	$R2,$x20,$ctx_
+	lvx_splt	$S2,$x30,$ctx_
+	lvx_splt	$T1,$x40,$ctx_
+	lvx_splt	$T2,$x50,$ctx_
+	lvx_splt	$T3,$x60,$ctx_
+	lvx_splt	$T4,$x70,$ctx_
+	stvx		$R1,$x00,$_ctx
+	stvx		$S1,$x10,$_ctx
+	stvx		$R2,$x20,$_ctx
+	stvx		$S2,$x30,$_ctx
+	stvx		$T1,$x40,$_ctx
+	stvx		$T2,$x50,$_ctx
+	stvx		$T3,$x60,$_ctx
+	stvx		$T4,$x70,$_ctx
+
+	addi		$inp,$inp,0x40
+	addi		$const,$const,0x50
+	addi		r0,$len,-64
+	srdi		r0,r0,6
+	mtctr		r0
+	b		Loop_vsx
+
+.align	4
+Loop_vsx:
+	################################################################
+	## ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
+	## ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
+	##   \___________________/
+	##
+	## Note that we start with inp[2:3]*r^2. This is because it
+	## doesn't depend on reduction in previous iteration.
+	################################################################
+	## d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
+	## d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
+	## d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
+	## d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
+	## d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
+
+	vmuleuw		$ACC0,$I0,$R0
+	vmuleuw		$ACC1,$I0,$R1
+	vmuleuw		$ACC2,$I0,$R2
+	vmuleuw		$ACC3,$I1,$R2
+
+	vmuleuw		$T0,$I1,$R0
+	vaddudm		$ACC1,$ACC1,$T0
+	vmuleuw		$T0,$I1,$R1
+	vaddudm		$ACC2,$ACC2,$T0
+	 vmuleuw	$ACC4,$I2,$R2
+	vmuleuw		$T0,$I4,$S1
+	vaddudm		$ACC0,$ACC0,$T0
+	vmuleuw		$T0,$I2,$R1
+	vaddudm		$ACC3,$ACC3,$T0
+	lvx		$S3,$x50,$_ctx
+	vmuleuw		$T0,$I3,$R1
+	vaddudm		$ACC4,$ACC4,$T0
+	lvx		$R3,$x40,$_ctx
+
+	 vaddudm	$H2,$H2,$I2
+	 vaddudm	$H0,$H0,$I0
+	 vaddudm	$H3,$H3,$I3
+	 vaddudm	$H1,$H1,$I1
+	 vaddudm	$H4,$H4,$I4
+
+	vmuleuw		$T0,$I3,$S2
+	vaddudm		$ACC0,$ACC0,$T0
+	vmuleuw		$T0,$I4,$S2
+	vaddudm		$ACC1,$ACC1,$T0
+	vmuleuw		$T0,$I2,$R0
+	vaddudm		$ACC2,$ACC2,$T0
+	vmuleuw		$T0,$I3,$R0
+	vaddudm		$ACC3,$ACC3,$T0
+	lvx		$S4,$x70,$_ctx
+	vmuleuw		$T0,$I4,$R0
+	vaddudm		$ACC4,$ACC4,$T0
+	lvx		$R4,$x60,$_ctx
+
+	vmuleuw		$T0,$I2,$S3
+	vaddudm		$ACC0,$ACC0,$T0
+	vmuleuw		$T0,$I3,$S3
+	vaddudm		$ACC1,$ACC1,$T0
+	vmuleuw		$T0,$I4,$S3
+	vaddudm		$ACC2,$ACC2,$T0
+	vmuleuw		$T0,$I0,$R3
+	vaddudm		$ACC3,$ACC3,$T0
+	vmuleuw		$T0,$I1,$R3
+	vaddudm		$ACC4,$ACC4,$T0
+
+	 be?lvx_u	$_4,$x00,$const		# byte swap mask
+	 lvx_u		$T1,$x00,$inp		# load next input block
+	 lvx_u		$T2,$x10,$inp
+	 lvx_u		$T3,$x20,$inp
+	 lvx_u		$T4,$x30,$inp
+	 be?vperm	$T1,$T1,$T1,$_4
+	 be?vperm	$T2,$T2,$T2,$_4
+	 be?vperm	$T3,$T3,$T3,$_4
+	 be?vperm	$T4,$T4,$T4,$_4
+
+	vmuleuw		$T0,$I1,$S4
+	vaddudm		$ACC0,$ACC0,$T0
+	vmuleuw		$T0,$I2,$S4
+	vaddudm		$ACC1,$ACC1,$T0
+	vmuleuw		$T0,$I3,$S4
+	vaddudm		$ACC2,$ACC2,$T0
+	vmuleuw		$T0,$I4,$S4
+	vaddudm		$ACC3,$ACC3,$T0
+	vmuleuw		$T0,$I0,$R4
+	vaddudm		$ACC4,$ACC4,$T0
+
+	 vpermdi	$I0,$T1,$T2,0b00	# smash input to base 2^26
+	 vspltisb	$_4,4
+	 vperm		$I2,$T1,$T2,$I2perm	# 0x...0e0f0001...1e1f1011
+	 vpermdi	$I3,$T1,$T2,0b11
+
+	# (hash + inp[0:1]) * r^4
+	vmulouw		$T0,$H0,$R0
+	vaddudm		$ACC0,$ACC0,$T0
+	vmulouw		$T0,$H1,$R0
+	vaddudm		$ACC1,$ACC1,$T0
+	vmulouw		$T0,$H2,$R0
+	vaddudm		$ACC2,$ACC2,$T0
+	vmulouw		$T0,$H3,$R0
+	vaddudm		$ACC3,$ACC3,$T0
+	vmulouw		$T0,$H4,$R0
+	vaddudm		$ACC4,$ACC4,$T0
+
+	 vpermdi	$T1,$T3,$T4,0b00
+	 vperm		$T2,$T3,$T4,$I2perm	# 0x...0e0f0001...1e1f1011
+	 vpermdi	$T3,$T3,$T4,0b11
+
+	vmulouw		$T0,$H2,$S3
+	vaddudm		$ACC0,$ACC0,$T0
+	vmulouw		$T0,$H3,$S3
+	vaddudm		$ACC1,$ACC1,$T0
+	vmulouw		$T0,$H4,$S3
+	vaddudm		$ACC2,$ACC2,$T0
+	vmulouw		$T0,$H0,$R3
+	vaddudm		$ACC3,$ACC3,$T0
+	lvx		$S1,$x10,$_ctx
+	vmulouw		$T0,$H1,$R3
+	vaddudm		$ACC4,$ACC4,$T0
+	lvx		$R1,$x00,$_ctx
+
+	 vsrd		$I1,$I0,$_26
+	 vsrd		$I2,$I2,$_4
+	 vsrd		$I4,$I3,$_40
+	 vsrd		$I3,$I3,$_14
+
+	vmulouw		$T0,$H1,$S4
+	vaddudm		$ACC0,$ACC0,$T0
+	vmulouw		$T0,$H2,$S4
+	vaddudm		$ACC1,$ACC1,$T0
+	vmulouw		$T0,$H3,$S4
+	vaddudm		$ACC2,$ACC2,$T0
+	vmulouw		$T0,$H4,$S4
+	vaddudm		$ACC3,$ACC3,$T0
+	lvx		$S2,$x30,$_ctx
+	vmulouw		$T0,$H0,$R4
+	vaddudm		$ACC4,$ACC4,$T0
+	lvx		$R2,$x20,$_ctx
+
+	 vand		$I0,$I0,$mask26
+	 vand		$I1,$I1,$mask26
+	 vand		$I2,$I2,$mask26
+	 vand		$I3,$I3,$mask26
+
+	vmulouw		$T0,$H4,$S1
+	vaddudm		$ACC0,$ACC0,$T0
+	vmulouw		$T0,$H0,$R1
+	vaddudm		$ACC1,$ACC1,$T0
+	vmulouw		$T0,$H1,$R1
+	vaddudm		$ACC2,$ACC2,$T0
+	vmulouw		$T0,$H2,$R1
+	vaddudm		$ACC3,$ACC3,$T0
+	vmulouw		$T0,$H3,$R1
+	vaddudm		$ACC4,$ACC4,$T0
+
+	 vsrd		$T2,$T2,$_4
+	 vsrd		$_4,$T1,$_26
+	 vsrd		$T4,$T3,$_40
+	 vsrd		$T3,$T3,$_14
+
+	vmulouw		$T0,$H3,$S2
+	vaddudm		$ACC0,$ACC0,$T0
+	vmulouw		$T0,$H4,$S2
+	vaddudm		$ACC1,$ACC1,$T0
+	vmulouw		$T0,$H0,$R2
+	vaddudm		$ACC2,$ACC2,$T0
+	vmulouw		$T0,$H1,$R2
+	vaddudm		$ACC3,$ACC3,$T0
+	vmulouw		$T0,$H2,$R2
+	vaddudm		$ACC4,$ACC4,$T0
+
+	 vand		$T1,$T1,$mask26
+	 vand		$_4,$_4,$mask26
+	 vand		$T2,$T2,$mask26
+	 vand		$T3,$T3,$mask26
+
+	################################################################
+	# lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
+	# and P. Schwabe
+
+	vspltisb	$T0,2
+	vsrd		$H4,$ACC3,$_26
+	vsrd		$H1,$ACC0,$_26
+	vand		$H3,$ACC3,$mask26
+	vand		$H0,$ACC0,$mask26
+	vaddudm		$H4,$H4,$ACC4		# h3 -> h4
+	vaddudm		$H1,$H1,$ACC1		# h0 -> h1
+
+	 vmrgow		$I4,$T4,$I4
+	 vmrgow		$I0,$T1,$I0
+	 vmrgow		$I1,$_4,$I1
+	 vmrgow		$I2,$T2,$I2
+	 vmrgow		$I3,$T3,$I3
+	 vor		$I4,$I4,$padbits
+
+	vsrd		$ACC4,$H4,$_26
+	vsrd		$ACC1,$H1,$_26
+	vand		$H4,$H4,$mask26
+	vand		$H1,$H1,$mask26
+	vaddudm		$H0,$H0,$ACC4
+	vaddudm		$H2,$ACC2,$ACC1		# h1 -> h2
+
+	vsld		$ACC4,$ACC4,$T0		# <<2
+	vsrd		$ACC2,$H2,$_26
+	vand		$H2,$H2,$mask26
+	vaddudm		$H0,$H0,$ACC4		# h4 -> h0
+	vaddudm		$H3,$H3,$ACC2		# h2 -> h3
+
+	vsrd		$ACC0,$H0,$_26
+	vsrd		$ACC3,$H3,$_26
+	vand		$H0,$H0,$mask26
+	vand		$H3,$H3,$mask26
+	vaddudm		$H1,$H1,$ACC0		# h0 -> h1
+	vaddudm		$H4,$H4,$ACC3		# h3 -> h4
+
+	addi		$inp,$inp,0x40
+	bdnz		Loop_vsx
+
+	neg		$len,$len
+	andi.		$len,$len,0x30
+	sub		$inp,$inp,$len
+
+	lvx_u		$R0,$x30,$ctx		# load all powers
+	lvx_u		$R1,$x00,$ctx_
+	lvx_u		$S1,$x10,$ctx_
+	lvx_u		$R2,$x20,$ctx_
+	lvx_u		$S2,$x30,$ctx_
+
+Last_vsx:
+	vmuleuw		$ACC0,$I0,$R0
+	vmuleuw		$ACC1,$I1,$R0
+	vmuleuw		$ACC2,$I2,$R0
+	vmuleuw		$ACC3,$I3,$R0
+	vmuleuw		$ACC4,$I4,$R0
+
+	vmuleuw		$T0,$I4,$S1
+	vaddudm		$ACC0,$ACC0,$T0
+	vmuleuw		$T0,$I0,$R1
+	vaddudm		$ACC1,$ACC1,$T0
+	vmuleuw		$T0,$I1,$R1
+	vaddudm		$ACC2,$ACC2,$T0
+	vmuleuw		$T0,$I2,$R1
+	vaddudm		$ACC3,$ACC3,$T0
+	lvx_u		$S3,$x50,$ctx_
+	vmuleuw		$T0,$I3,$R1
+	vaddudm		$ACC4,$ACC4,$T0
+	lvx_u		$R3,$x40,$ctx_
+
+	 vaddudm	$H2,$H2,$I2
+	 vaddudm	$H0,$H0,$I0
+	 vaddudm	$H3,$H3,$I3
+	 vaddudm	$H1,$H1,$I1
+	 vaddudm	$H4,$H4,$I4
+
+	vmuleuw		$T0,$I3,$S2
+	vaddudm		$ACC0,$ACC0,$T0
+	vmuleuw		$T0,$I4,$S2
+	vaddudm		$ACC1,$ACC1,$T0
+	vmuleuw		$T0,$I0,$R2
+	vaddudm		$ACC2,$ACC2,$T0
+	vmuleuw		$T0,$I1,$R2
+	vaddudm		$ACC3,$ACC3,$T0
+	lvx_u		$S4,$x70,$ctx_
+	vmuleuw		$T0,$I2,$R2
+	vaddudm		$ACC4,$ACC4,$T0
+	lvx_u		$R4,$x60,$ctx_
+
+	vmuleuw		$T0,$I2,$S3
+	vaddudm		$ACC0,$ACC0,$T0
+	vmuleuw		$T0,$I3,$S3
+	vaddudm		$ACC1,$ACC1,$T0
+	vmuleuw		$T0,$I4,$S3
+	vaddudm		$ACC2,$ACC2,$T0
+	vmuleuw		$T0,$I0,$R3
+	vaddudm		$ACC3,$ACC3,$T0
+	vmuleuw		$T0,$I1,$R3
+	vaddudm		$ACC4,$ACC4,$T0
+
+	vmuleuw		$T0,$I1,$S4
+	vaddudm		$ACC0,$ACC0,$T0
+	vmuleuw		$T0,$I2,$S4
+	vaddudm		$ACC1,$ACC1,$T0
+	vmuleuw		$T0,$I3,$S4
+	vaddudm		$ACC2,$ACC2,$T0
+	vmuleuw		$T0,$I4,$S4
+	vaddudm		$ACC3,$ACC3,$T0
+	vmuleuw		$T0,$I0,$R4
+	vaddudm		$ACC4,$ACC4,$T0
+
+	# (hash + inp[0:1]) * r^4
+	vmulouw		$T0,$H0,$R0
+	vaddudm		$ACC0,$ACC0,$T0
+	vmulouw		$T0,$H1,$R0
+	vaddudm		$ACC1,$ACC1,$T0
+	vmulouw		$T0,$H2,$R0
+	vaddudm		$ACC2,$ACC2,$T0
+	vmulouw		$T0,$H3,$R0
+	vaddudm		$ACC3,$ACC3,$T0
+	vmulouw		$T0,$H4,$R0
+	vaddudm		$ACC4,$ACC4,$T0
+
+	vmulouw		$T0,$H2,$S3
+	vaddudm		$ACC0,$ACC0,$T0
+	vmulouw		$T0,$H3,$S3
+	vaddudm		$ACC1,$ACC1,$T0
+	vmulouw		$T0,$H4,$S3
+	vaddudm		$ACC2,$ACC2,$T0
+	vmulouw		$T0,$H0,$R3
+	vaddudm		$ACC3,$ACC3,$T0
+	lvx_u		$S1,$x10,$ctx_
+	vmulouw		$T0,$H1,$R3
+	vaddudm		$ACC4,$ACC4,$T0
+	lvx_u		$R1,$x00,$ctx_
+
+	vmulouw		$T0,$H1,$S4
+	vaddudm		$ACC0,$ACC0,$T0
+	vmulouw		$T0,$H2,$S4
+	vaddudm		$ACC1,$ACC1,$T0
+	vmulouw		$T0,$H3,$S4
+	vaddudm		$ACC2,$ACC2,$T0
+	vmulouw		$T0,$H4,$S4
+	vaddudm		$ACC3,$ACC3,$T0
+	lvx_u		$S2,$x30,$ctx_
+	vmulouw		$T0,$H0,$R4
+	vaddudm		$ACC4,$ACC4,$T0
+	lvx_u		$R2,$x20,$ctx_
+
+	vmulouw		$T0,$H4,$S1
+	vaddudm		$ACC0,$ACC0,$T0
+	vmulouw		$T0,$H0,$R1
+	vaddudm		$ACC1,$ACC1,$T0
+	vmulouw		$T0,$H1,$R1
+	vaddudm		$ACC2,$ACC2,$T0
+	vmulouw		$T0,$H2,$R1
+	vaddudm		$ACC3,$ACC3,$T0
+	vmulouw		$T0,$H3,$R1
+	vaddudm		$ACC4,$ACC4,$T0
+
+	vmulouw		$T0,$H3,$S2
+	vaddudm		$ACC0,$ACC0,$T0
+	vmulouw		$T0,$H4,$S2
+	vaddudm		$ACC1,$ACC1,$T0
+	vmulouw		$T0,$H0,$R2
+	vaddudm		$ACC2,$ACC2,$T0
+	vmulouw		$T0,$H1,$R2
+	vaddudm		$ACC3,$ACC3,$T0
+	vmulouw		$T0,$H2,$R2
+	vaddudm		$ACC4,$ACC4,$T0
+
+	################################################################
+	# horizontal addition
+
+	vpermdi		$H0,$ACC0,$ACC0,0b10
+	vpermdi		$H1,$ACC1,$ACC1,0b10
+	vpermdi		$H2,$ACC2,$ACC2,0b10
+	vpermdi		$H3,$ACC3,$ACC3,0b10
+	vpermdi		$H4,$ACC4,$ACC4,0b10
+	vaddudm		$ACC0,$ACC0,$H0
+	vaddudm		$ACC1,$ACC1,$H1
+	vaddudm		$ACC2,$ACC2,$H2
+	vaddudm		$ACC3,$ACC3,$H3
+	vaddudm		$ACC4,$ACC4,$H4
+
+	################################################################
+	# lazy reduction
+
+	vspltisb	$T0,2
+	vsrd		$H4,$ACC3,$_26
+	vsrd		$H1,$ACC0,$_26
+	vand		$H3,$ACC3,$mask26
+	vand		$H0,$ACC0,$mask26
+	vaddudm		$H4,$H4,$ACC4		# h3 -> h4
+	vaddudm		$H1,$H1,$ACC1		# h0 -> h1
+
+	vsrd		$ACC4,$H4,$_26
+	vsrd		$ACC1,$H1,$_26
+	vand		$H4,$H4,$mask26
+	vand		$H1,$H1,$mask26
+	vaddudm		$H0,$H0,$ACC4
+	vaddudm		$H2,$ACC2,$ACC1		# h1 -> h2
+
+	vsld		$ACC4,$ACC4,$T0		# <<2
+	vsrd		$ACC2,$H2,$_26
+	vand		$H2,$H2,$mask26
+	vaddudm		$H0,$H0,$ACC4		# h4 -> h0
+	vaddudm		$H3,$H3,$ACC2		# h2 -> h3
+
+	vsrd		$ACC0,$H0,$_26
+	vsrd		$ACC3,$H3,$_26
+	vand		$H0,$H0,$mask26
+	vand		$H3,$H3,$mask26
+	vaddudm		$H1,$H1,$ACC0		# h0 -> h1
+	vaddudm		$H4,$H4,$ACC3		# h3 -> h4
+
+	beq		Ldone_vsx
+
+	add		r6,$const,$len
+
+	be?lvx_u	$_4,$x00,$const		# byte swap mask
+	lvx_u		$T1,$x00,$inp		# load last partial input block
+	lvx_u		$T2,$x10,$inp
+	lvx_u		$T3,$x20,$inp
+	lvx_u		$T4,$x30,$inp
+	be?vperm	$T1,$T1,$T1,$_4
+	be?vperm	$T2,$T2,$T2,$_4
+	be?vperm	$T3,$T3,$T3,$_4
+	be?vperm	$T4,$T4,$T4,$_4
+
+	vpermdi		$I0,$T1,$T2,0b00	# smash input to base 2^26
+	vspltisb	$_4,4
+	vperm		$I2,$T1,$T2,$I2perm	# 0x...0e0f0001...1e1f1011
+	vpermdi		$I3,$T1,$T2,0b11
+
+	vsrd		$I1,$I0,$_26
+	vsrd		$I2,$I2,$_4
+	vsrd		$I4,$I3,$_40
+	vsrd		$I3,$I3,$_14
+	vand		$I0,$I0,$mask26
+	vand		$I1,$I1,$mask26
+	vand		$I2,$I2,$mask26
+	vand		$I3,$I3,$mask26
+
+	vpermdi		$T0,$T3,$T4,0b00
+	vperm		$T1,$T3,$T4,$I2perm	# 0x...0e0f0001...1e1f1011
+	vpermdi		$T2,$T3,$T4,0b11
+
+	lvx_u		$ACC0,$x00,r6
+	lvx_u		$ACC1,$x30,r6
+
+	vsrd		$T3,$T0,$_26
+	vsrd		$T1,$T1,$_4
+	vsrd		$T4,$T2,$_40
+	vsrd		$T2,$T2,$_14
+	vand		$T0,$T0,$mask26
+	vand		$T3,$T3,$mask26
+	vand		$T1,$T1,$mask26
+	vand		$T2,$T2,$mask26
+
+	# inp[2]:inp[0]:inp[3]:inp[1]
+	vmrgow		$I4,$T4,$I4
+	vmrgow		$I0,$T0,$I0
+	vmrgow		$I1,$T3,$I1
+	vmrgow		$I2,$T1,$I2
+	vmrgow		$I3,$T2,$I3
+	vor		$I4,$I4,$padbits
+
+	vperm		$H0,$H0,$H0,$ACC0	# move hash to right lane
+	vand		$I0,$I0,    $ACC1	# mask redundant input lane[s]
+	vperm		$H1,$H1,$H1,$ACC0
+	vand		$I1,$I1,    $ACC1
+	vperm		$H2,$H2,$H2,$ACC0
+	vand		$I2,$I2,    $ACC1
+	vperm		$H3,$H3,$H3,$ACC0
+	vand		$I3,$I3,    $ACC1
+	vperm		$H4,$H4,$H4,$ACC0
+	vand		$I4,$I4,    $ACC1
+
+	vaddudm		$I0,$I0,$H0		# accumulate hash
+	vxor		$H0,$H0,$H0		# wipe hash value
+	vaddudm		$I1,$I1,$H1
+	vxor		$H1,$H1,$H1
+	vaddudm		$I2,$I2,$H2
+	vxor		$H2,$H2,$H2
+	vaddudm		$I3,$I3,$H3
+	vxor		$H3,$H3,$H3
+	vaddudm		$I4,$I4,$H4
+	vxor		$H4,$H4,$H4
+
+	xor.		$len,$len,$len
+	b		Last_vsx
+
+.align	4
+Ldone_vsx:
+	$POP	r0,`$VSXFRAME+$LRSAVE`($sp)
+	li	$x10,4
+	li	$x20,8
+	li	$x30,12
+	li	$x40,16
+	stvwx_u	$H0,$x00,$ctx			# store hash
+	stvwx_u	$H1,$x10,$ctx
+	stvwx_u	$H2,$x20,$ctx
+	stvwx_u	$H3,$x30,$ctx
+	stvwx_u	$H4,$x40,$ctx
+
+	lwz	r12,`$VSXFRAME-$SIZE_T*5-4`($sp)# pull vrsave
+	mtlr	r0
+	li	r10,`15+$LOCALS+128`
+	li	r11,`31+$LOCALS+128`
+	mtspr	256,r12				# restore vrsave
+	lvx	v20,r10,$sp
+	addi	r10,r10,32
+	lvx	v21,r10,$sp
+	addi	r10,r10,32
+	lvx	v22,r11,$sp
+	addi	r11,r11,32
+	lvx	v23,r10,$sp
+	addi	r10,r10,32
+	lvx	v24,r11,$sp
+	addi	r11,r11,32
+	lvx	v25,r10,$sp
+	addi	r10,r10,32
+	lvx	v26,r11,$sp
+	addi	r11,r11,32
+	lvx	v27,r10,$sp
+	addi	r10,r10,32
+	lvx	v28,r11,$sp
+	addi	r11,r11,32
+	lvx	v29,r10,$sp
+	addi	r10,r10,32
+	lvx	v30,r11,$sp
+	lvx	v31,r10,$sp
+	$POP	r27,`$VSXFRAME-$SIZE_T*5`($sp)
+	$POP	r28,`$VSXFRAME-$SIZE_T*4`($sp)
+	$POP	r29,`$VSXFRAME-$SIZE_T*3`($sp)
+	$POP	r30,`$VSXFRAME-$SIZE_T*2`($sp)
+	$POP	r31,`$VSXFRAME-$SIZE_T*1`($sp)
+	addi	$sp,$sp,$VSXFRAME
+	blr
+	.long	0
+	.byte	0,12,0x04,1,0x80,5,4,0
+	.long	0
+.size	__poly1305_blocks_vsx,.-__poly1305_blocks_vsx
+
+.align	6
+LPICmeup:
+	mflr	r0
+	bcl	20,31,\$+4
+	mflr	$const      # vvvvvv "distance" between . and 1st data entry
+	addi	$const,$const,`64-8`
+	mtlr	r0
+	blr
+	.long	0
+	.byte	0,12,0x14,0,0,0,0,0
+	.space	`64-9*4`
+
+.quad	0x0000000003ffffff,0x0000000003ffffff	# mask26
+.quad	0x000000000000001a,0x000000000000001a	# _26
+.quad	0x0000000000000028,0x0000000000000028	# _40
+.quad	0x000000000e0f0001,0x000000001e1f1011	# I2perm
+.quad	0x0100000001000000,0x0100000001000000	# padbits
+.quad	0x0706050403020100,0x0f0e0d0c0b0a0908	# byte swap for big-endian
+
+.quad	0x0000000000000000,0x0000000004050607	# magic tail masks
+.quad	0x0405060700000000,0x0000000000000000
+.quad	0x0000000000000000,0x0405060700000000
+
+.quad	0xffffffff00000000,0xffffffffffffffff
+.quad	0xffffffff00000000,0xffffffff00000000
+.quad	0x0000000000000000,0xffffffff00000000
+___
+}}}
+$code.=<<___;
+.asciz	"Poly1305 for PPC, CRYPTOGAMS by \@dot-asm"
+___
+
+foreach (split("\n",$code)) {
+	s/\`([^\`]*)\`/eval($1)/ge;
+
+	# instructions prefixed with '?' are endian-specific and need
+	# to be adjusted accordingly...
+	if ($flavour !~ /le$/) {	# big-endian
+	    s/be\?//		or
+	    s/le\?/#le#/
+	} else {			# little-endian
+	    s/le\?//		or
+	    s/be\?/#be#/
+	}
+
+	print $_,"\n";
+}
+close STDOUT;
diff --git a/src/crypto/zinc/poly1305/poly1305-ppcfp.pl b/src/crypto/zinc/poly1305/poly1305-ppcfp.pl
new file mode 100755
index 0000000..3eb9b88
--- /dev/null
+++ b/src/crypto/zinc/poly1305/poly1305-ppcfp.pl
@@ -0,0 +1,749 @@
+#! /usr/bin/env perl
+# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+#
+# This code is taken from the OpenSSL project but the author, Andy Polyakov,
+# has relicensed it under the licenses specified in the SPDX header above.
+# The original headers, including the original license headers, are
+# included below for completeness.
+#
+# Changes: look in more places for ppc-xlate.pl
+#
+# Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the Apache License 2.0 (the "License").  You may not use
+# this file except in compliance with the License.  You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
+#
+# ====================================================================
+# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
+# project. The module is, however, dual licensed under OpenSSL and
+# CRYPTOGAMS licenses depending on where you obtain it. For further
+# details see http://www.openssl.org/~appro/cryptogams/.
+# ====================================================================
+#
+# This module implements Poly1305 hash for PowerPC FPU.
+#
+# June 2015
+#
+# Numbers are cycles per processed byte with poly1305_blocks alone,
+# and improvement coefficients relative to gcc-generated code.
+#
+# Freescale e300	9.78/+30%
+# PPC74x0		6.92/+50%
+# PPC970		6.03/+80%
+# POWER7		3.50/+30%
+# POWER8		3.75/+10%
+
+$flavour = shift;
+
+if ($flavour =~ /64/) {
+	$SIZE_T	=8;
+	$LRSAVE	=2*$SIZE_T;
+	$UCMP	="cmpld";
+	$STU	="stdu";
+	$POP	="ld";
+	$PUSH	="std";
+} elsif ($flavour =~ /32/) {
+	$SIZE_T	=4;
+	$LRSAVE	=$SIZE_T;
+	$UCMP	="cmplw";
+	$STU	="stwu";
+	$POP	="lwz";
+	$PUSH	="stw";
+} else { die "nonsense $flavour"; }
+
+$LITTLE_ENDIAN = ($flavour=~/le$/) ? 4 : 0;
+
+$LWXLE = $LITTLE_ENDIAN ? "lwzx" : "lwbrx";
+
+$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
+( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
+( $xlate="${dir}../perlasm/ppc-xlate.pl" and -f $xlate) or
+( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
+die "can't locate ppc-xlate.pl";
+
+open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";
+
+$LOCALS=6*$SIZE_T;
+$FRAME=$LOCALS+6*8+18*8;
+
+my $sp="r1";
+
+my ($ctx,$inp,$len,$padbit) = map("r$_",(3..6));
+my ($in0,$in1,$in2,$in3,$i1,$i2,$i3) = map("r$_",(7..12,6));
+
+my ($h0lo,$h0hi,$h1lo,$h1hi,$h2lo,$h2hi,$h3lo,$h3hi,
+    $two0,$two32,$two64,$two96,$two130,$five_two130,
+    $r0lo,$r0hi,$r1lo,$r1hi,$r2lo,$r2hi,
+    $s2lo,$s2hi,$s3lo,$s3hi,
+    $c0lo,$c0hi,$c1lo,$c1hi,$c2lo,$c2hi,$c3lo,$c3hi) = map("f$_",(0..31));
+# borrowings
+my ($r3lo,$r3hi,$s1lo,$s1hi) = ($c0lo,$c0hi,$c1lo,$c1hi);
+my ($x0,$x1,$x2,$x3) = ($c2lo,$c2hi,$c3lo,$c3hi);
+my ($y0,$y1,$y2,$y3) = ($c3lo,$c3hi,$c1lo,$c1hi);
+
+$code.=<<___;
+.machine	"any"
+.text
+
+.globl	.poly1305_init_fpu
+.align	6
+.poly1305_init_fpu:
+	$STU	$sp,-$LOCALS($sp)		# minimal frame
+	mflr	$padbit
+	$PUSH	$padbit,`$LOCALS+$LRSAVE`($sp)
+
+	bl	LPICmeup
+
+	xor	r0,r0,r0
+	mtlr	$padbit				# restore lr
+
+	lfd	$two0,8*0($len)			# load constants
+	lfd	$two32,8*1($len)
+	lfd	$two64,8*2($len)
+	lfd	$two96,8*3($len)
+	lfd	$two130,8*4($len)
+	lfd	$five_two130,8*5($len)
+
+	stfd	$two0,8*0($ctx)			# initial hash value, biased 0
+	stfd	$two32,8*1($ctx)
+	stfd	$two64,8*2($ctx)
+	stfd	$two96,8*3($ctx)
+
+	$UCMP	$inp,r0
+	beq-	Lno_key
+
+	lfd	$h3lo,8*13($len)		# new fpscr
+	mffs	$h3hi				# old fpscr
+
+	stfd	$two0,8*4($ctx)			# key "template"
+	stfd	$two32,8*5($ctx)
+	stfd	$two64,8*6($ctx)
+	stfd	$two96,8*7($ctx)
+
+	li	$in1,4
+	li	$in2,8
+	li	$in3,12
+	$LWXLE	$in0,0,$inp			# load key
+	$LWXLE	$in1,$in1,$inp
+	$LWXLE	$in2,$in2,$inp
+	$LWXLE	$in3,$in3,$inp
+
+	lis	$i1,0xf000			#   0xf0000000
+	ori	$i2,$i1,3			#   0xf0000003
+	andc	$in0,$in0,$i1			# &=0x0fffffff
+	andc	$in1,$in1,$i2			# &=0x0ffffffc
+	andc	$in2,$in2,$i2
+	andc	$in3,$in3,$i2
+
+	stw	$in0,`8*4+(4^$LITTLE_ENDIAN)`($ctx)	# fill "template"
+	stw	$in1,`8*5+(4^$LITTLE_ENDIAN)`($ctx)
+	stw	$in2,`8*6+(4^$LITTLE_ENDIAN)`($ctx)
+	stw	$in3,`8*7+(4^$LITTLE_ENDIAN)`($ctx)
+
+	mtfsf	255,$h3lo			# fpscr
+	stfd	$two0,8*18($ctx)		# copy constants to context
+	stfd	$two32,8*19($ctx)
+	stfd	$two64,8*20($ctx)
+	stfd	$two96,8*21($ctx)
+	stfd	$two130,8*22($ctx)
+	stfd	$five_two130,8*23($ctx)
+
+	lfd	$h0lo,8*4($ctx)			# load [biased] key
+	lfd	$h1lo,8*5($ctx)
+	lfd	$h2lo,8*6($ctx)
+	lfd	$h3lo,8*7($ctx)
+
+	fsub	$h0lo,$h0lo,$two0		# r0
+	fsub	$h1lo,$h1lo,$two32		# r1
+	fsub	$h2lo,$h2lo,$two64		# r2
+	fsub	$h3lo,$h3lo,$two96		# r3
+
+	lfd	$two0,8*6($len)			# more constants
+	lfd	$two32,8*7($len)
+	lfd	$two64,8*8($len)
+	lfd	$two96,8*9($len)
+
+	fmul	$h1hi,$h1lo,$five_two130	# s1
+	fmul	$h2hi,$h2lo,$five_two130	# s2
+	 stfd	$h3hi,8*15($ctx)		# borrow slot for original fpscr
+	fmul	$h3hi,$h3lo,$five_two130	# s3
+
+	fadd	$h0hi,$h0lo,$two0
+	 stfd	$h1hi,8*12($ctx)		# put aside for now
+	fadd	$h1hi,$h1lo,$two32
+	 stfd	$h2hi,8*13($ctx)
+	fadd	$h2hi,$h2lo,$two64
+	 stfd	$h3hi,8*14($ctx)
+	fadd	$h3hi,$h3lo,$two96
+
+	fsub	$h0hi,$h0hi,$two0
+	fsub	$h1hi,$h1hi,$two32
+	fsub	$h2hi,$h2hi,$two64
+	fsub	$h3hi,$h3hi,$two96
+
+	lfd	$two0,8*10($len)		# more constants
+	lfd	$two32,8*11($len)
+	lfd	$two64,8*12($len)
+
+	fsub	$h0lo,$h0lo,$h0hi
+	fsub	$h1lo,$h1lo,$h1hi
+	fsub	$h2lo,$h2lo,$h2hi
+	fsub	$h3lo,$h3lo,$h3hi
+
+	stfd	$h0hi,8*5($ctx)			# r0hi
+	stfd	$h1hi,8*7($ctx)			# r1hi
+	stfd	$h2hi,8*9($ctx)			# r2hi
+	stfd	$h3hi,8*11($ctx)		# r3hi
+
+	stfd	$h0lo,8*4($ctx)			# r0lo
+	stfd	$h1lo,8*6($ctx)			# r1lo
+	stfd	$h2lo,8*8($ctx)			# r2lo
+	stfd	$h3lo,8*10($ctx)		# r3lo
+
+	lfd	$h1lo,8*12($ctx)		# s1
+	lfd	$h2lo,8*13($ctx)		# s2
+	lfd	$h3lo,8*14($ctx)		# s3
+	lfd	$h0lo,8*15($ctx)		# pull original fpscr
+
+	fadd	$h1hi,$h1lo,$two0
+	fadd	$h2hi,$h2lo,$two32
+	fadd	$h3hi,$h3lo,$two64
+
+	fsub	$h1hi,$h1hi,$two0
+	fsub	$h2hi,$h2hi,$two32
+	fsub	$h3hi,$h3hi,$two64
+
+	fsub	$h1lo,$h1lo,$h1hi
+	fsub	$h2lo,$h2lo,$h2hi
+	fsub	$h3lo,$h3lo,$h3hi
+
+	stfd	$h1hi,8*13($ctx)		# s1hi
+	stfd	$h2hi,8*15($ctx)		# s2hi
+	stfd	$h3hi,8*17($ctx)		# s3hi
+
+	stfd	$h1lo,8*12($ctx)		# s1lo
+	stfd	$h2lo,8*14($ctx)		# s2lo
+	stfd	$h3lo,8*16($ctx)		# s3lo
+
+	mtfsf	255,$h0lo			# restore fpscr
+Lno_key:
+	xor	r3,r3,r3
+	addi	$sp,$sp,$LOCALS
+	blr
+	.long	0
+	.byte	0,12,4,1,0x80,0,2,0
+.size	.poly1305_init_fpu,.-.poly1305_init_fpu
+
+.globl	.poly1305_blocks_fpu
+.align	4
+.poly1305_blocks_fpu:
+	srwi.	$len,$len,4
+	beq-	Labort
+
+	$STU	$sp,-$FRAME($sp)
+	mflr	r0
+	stfd	f14,`$FRAME-8*18`($sp)
+	stfd	f15,`$FRAME-8*17`($sp)
+	stfd	f16,`$FRAME-8*16`($sp)
+	stfd	f17,`$FRAME-8*15`($sp)
+	stfd	f18,`$FRAME-8*14`($sp)
+	stfd	f19,`$FRAME-8*13`($sp)
+	stfd	f20,`$FRAME-8*12`($sp)
+	stfd	f21,`$FRAME-8*11`($sp)
+	stfd	f22,`$FRAME-8*10`($sp)
+	stfd	f23,`$FRAME-8*9`($sp)
+	stfd	f24,`$FRAME-8*8`($sp)
+	stfd	f25,`$FRAME-8*7`($sp)
+	stfd	f26,`$FRAME-8*6`($sp)
+	stfd	f27,`$FRAME-8*5`($sp)
+	stfd	f28,`$FRAME-8*4`($sp)
+	stfd	f29,`$FRAME-8*3`($sp)
+	stfd	f30,`$FRAME-8*2`($sp)
+	stfd	f31,`$FRAME-8*1`($sp)
+	$PUSH	r0,`$FRAME+$LRSAVE`($sp)
+
+	xor	r0,r0,r0
+	li	$in3,1
+	mtctr	$len
+	neg	$len,$len
+	stw	r0,`$LOCALS+8*4+(0^$LITTLE_ENDIAN)`($sp)
+	stw	$in3,`$LOCALS+8*4+(4^$LITTLE_ENDIAN)`($sp)
+
+	lfd	$two0,8*18($ctx)		# load constants
+	lfd	$two32,8*19($ctx)
+	lfd	$two64,8*20($ctx)
+	lfd	$two96,8*21($ctx)
+	lfd	$two130,8*22($ctx)
+	lfd	$five_two130,8*23($ctx)
+
+	lfd	$h0lo,8*0($ctx)			# load [biased] hash value
+	lfd	$h1lo,8*1($ctx)
+	lfd	$h2lo,8*2($ctx)
+	lfd	$h3lo,8*3($ctx)
+
+	stfd	$two0,`$LOCALS+8*0`($sp)	# input "template"
+	oris	$in3,$padbit,`(1023+52+96)<<4`
+	stfd	$two32,`$LOCALS+8*1`($sp)
+	stfd	$two64,`$LOCALS+8*2`($sp)
+	stw	$in3,`$LOCALS+8*3+(0^$LITTLE_ENDIAN)`($sp)
+
+	li	$i1,4
+	li	$i2,8
+	li	$i3,12
+	$LWXLE	$in0,0,$inp			# load input
+	$LWXLE	$in1,$i1,$inp
+	$LWXLE	$in2,$i2,$inp
+	$LWXLE	$in3,$i3,$inp
+	addi	$inp,$inp,16
+
+	stw	$in0,`$LOCALS+8*0+(4^$LITTLE_ENDIAN)`($sp)	# fill "template"
+	stw	$in1,`$LOCALS+8*1+(4^$LITTLE_ENDIAN)`($sp)
+	stw	$in2,`$LOCALS+8*2+(4^$LITTLE_ENDIAN)`($sp)
+	stw	$in3,`$LOCALS+8*3+(4^$LITTLE_ENDIAN)`($sp)
+
+	mffs	$x0				# original fpscr
+	lfd	$x1,`$LOCALS+8*4`($sp)		# new fpscr
+	lfd	$r0lo,8*4($ctx)			# load key
+	lfd	$r0hi,8*5($ctx)
+	lfd	$r1lo,8*6($ctx)
+	lfd	$r1hi,8*7($ctx)
+	lfd	$r2lo,8*8($ctx)
+	lfd	$r2hi,8*9($ctx)
+	lfd	$r3lo,8*10($ctx)
+	lfd	$r3hi,8*11($ctx)
+	lfd	$s1lo,8*12($ctx)
+	lfd	$s1hi,8*13($ctx)
+	lfd	$s2lo,8*14($ctx)
+	lfd	$s2hi,8*15($ctx)
+	lfd	$s3lo,8*16($ctx)
+	lfd	$s3hi,8*17($ctx)
+
+	stfd	$x0,`$LOCALS+8*4`($sp)		# save original fpscr
+	mtfsf	255,$x1
+
+	addic	$len,$len,1
+	addze	r0,r0
+	slwi.	r0,r0,4
+	sub	$inp,$inp,r0			# conditional rewind
+
+	lfd	$x0,`$LOCALS+8*0`($sp)
+	lfd	$x1,`$LOCALS+8*1`($sp)
+	lfd	$x2,`$LOCALS+8*2`($sp)
+	lfd	$x3,`$LOCALS+8*3`($sp)
+
+	fsub	$h0lo,$h0lo,$two0		# de-bias hash value
+	 $LWXLE	$in0,0,$inp			# modulo-scheduled input load
+	fsub	$h1lo,$h1lo,$two32
+	 $LWXLE	$in1,$i1,$inp
+	fsub	$h2lo,$h2lo,$two64
+	 $LWXLE	$in2,$i2,$inp
+	fsub	$h3lo,$h3lo,$two96
+	 $LWXLE	$in3,$i3,$inp
+
+	fsub	$x0,$x0,$two0			# de-bias input
+	 addi	$inp,$inp,16
+	fsub	$x1,$x1,$two32
+	fsub	$x2,$x2,$two64
+	fsub	$x3,$x3,$two96
+
+	fadd	$x0,$x0,$h0lo			# accumulate input
+	 stw	$in0,`$LOCALS+8*0+(4^$LITTLE_ENDIAN)`($sp)
+	fadd	$x1,$x1,$h1lo
+	 stw	$in1,`$LOCALS+8*1+(4^$LITTLE_ENDIAN)`($sp)
+	fadd	$x2,$x2,$h2lo
+	 stw	$in2,`$LOCALS+8*2+(4^$LITTLE_ENDIAN)`($sp)
+	fadd	$x3,$x3,$h3lo
+	 stw	$in3,`$LOCALS+8*3+(4^$LITTLE_ENDIAN)`($sp)
+
+	b	Lentry
+
+.align	4
+Loop:
+	fsub	$y0,$y0,$two0			# de-bias input
+	 addic	$len,$len,1
+	fsub	$y1,$y1,$two32
+	 addze	r0,r0
+	fsub	$y2,$y2,$two64
+	 slwi.	r0,r0,4
+	fsub	$y3,$y3,$two96
+	 sub	$inp,$inp,r0			# conditional rewind
+
+	fadd	$h0lo,$h0lo,$y0			# accumulate input
+	fadd	$h0hi,$h0hi,$y1
+	fadd	$h2lo,$h2lo,$y2
+	fadd	$h2hi,$h2hi,$y3
+
+	######################################### base 2^48 -> base 2^32
+	fadd	$c1lo,$h1lo,$two64
+	 $LWXLE	$in0,0,$inp			# modulo-scheduled input load
+	fadd	$c1hi,$h1hi,$two64
+	 $LWXLE	$in1,$i1,$inp
+	fadd	$c3lo,$h3lo,$two130
+	 $LWXLE	$in2,$i2,$inp
+	fadd	$c3hi,$h3hi,$two130
+	 $LWXLE	$in3,$i3,$inp
+	fadd	$c0lo,$h0lo,$two32
+	 addi	$inp,$inp,16
+	fadd	$c0hi,$h0hi,$two32
+	fadd	$c2lo,$h2lo,$two96
+	fadd	$c2hi,$h2hi,$two96
+
+	fsub	$c1lo,$c1lo,$two64
+	 stw	$in0,`$LOCALS+8*0+(4^$LITTLE_ENDIAN)`($sp)	# fill "template"
+	fsub	$c1hi,$c1hi,$two64
+	 stw	$in1,`$LOCALS+8*1+(4^$LITTLE_ENDIAN)`($sp)
+	fsub	$c3lo,$c3lo,$two130
+	 stw	$in2,`$LOCALS+8*2+(4^$LITTLE_ENDIAN)`($sp)
+	fsub	$c3hi,$c3hi,$two130
+	 stw	$in3,`$LOCALS+8*3+(4^$LITTLE_ENDIAN)`($sp)
+	fsub	$c0lo,$c0lo,$two32
+	fsub	$c0hi,$c0hi,$two32
+	fsub	$c2lo,$c2lo,$two96
+	fsub	$c2hi,$c2hi,$two96
+
+	fsub	$h1lo,$h1lo,$c1lo
+	fsub	$h1hi,$h1hi,$c1hi
+	fsub	$h3lo,$h3lo,$c3lo
+	fsub	$h3hi,$h3hi,$c3hi
+	fsub	$h2lo,$h2lo,$c2lo
+	fsub	$h2hi,$h2hi,$c2hi
+	fsub	$h0lo,$h0lo,$c0lo
+	fsub	$h0hi,$h0hi,$c0hi
+
+	fadd	$h1lo,$h1lo,$c0lo
+	fadd	$h1hi,$h1hi,$c0hi
+	fadd	$h3lo,$h3lo,$c2lo
+	fadd	$h3hi,$h3hi,$c2hi
+	fadd	$h2lo,$h2lo,$c1lo
+	fadd	$h2hi,$h2hi,$c1hi
+	fmadd	$h0lo,$c3lo,$five_two130,$h0lo
+	fmadd	$h0hi,$c3hi,$five_two130,$h0hi
+
+	fadd	$x1,$h1lo,$h1hi
+	 lfd	$s1lo,8*12($ctx)		# reload constants
+	fadd	$x3,$h3lo,$h3hi
+	 lfd	$s1hi,8*13($ctx)
+	fadd	$x2,$h2lo,$h2hi
+	 lfd	$r3lo,8*10($ctx)
+	fadd	$x0,$h0lo,$h0hi
+	 lfd	$r3hi,8*11($ctx)
+Lentry:
+	fmul	$h0lo,$s3lo,$x1
+	fmul	$h0hi,$s3hi,$x1
+	fmul	$h2lo,$r1lo,$x1
+	fmul	$h2hi,$r1hi,$x1
+	fmul	$h1lo,$r0lo,$x1
+	fmul	$h1hi,$r0hi,$x1
+	fmul	$h3lo,$r2lo,$x1
+	fmul	$h3hi,$r2hi,$x1
+
+	fmadd	$h0lo,$s1lo,$x3,$h0lo
+	fmadd	$h0hi,$s1hi,$x3,$h0hi
+	fmadd	$h2lo,$s3lo,$x3,$h2lo
+	fmadd	$h2hi,$s3hi,$x3,$h2hi
+	fmadd	$h1lo,$s2lo,$x3,$h1lo
+	fmadd	$h1hi,$s2hi,$x3,$h1hi
+	fmadd	$h3lo,$r0lo,$x3,$h3lo
+	fmadd	$h3hi,$r0hi,$x3,$h3hi
+
+	fmadd	$h0lo,$s2lo,$x2,$h0lo
+	fmadd	$h0hi,$s2hi,$x2,$h0hi
+	fmadd	$h2lo,$r0lo,$x2,$h2lo
+	fmadd	$h2hi,$r0hi,$x2,$h2hi
+	fmadd	$h1lo,$s3lo,$x2,$h1lo
+	fmadd	$h1hi,$s3hi,$x2,$h1hi
+	fmadd	$h3lo,$r1lo,$x2,$h3lo
+	fmadd	$h3hi,$r1hi,$x2,$h3hi
+
+	fmadd	$h0lo,$r0lo,$x0,$h0lo
+	 lfd	$y0,`$LOCALS+8*0`($sp)		# load [biased] input
+	fmadd	$h0hi,$r0hi,$x0,$h0hi
+	 lfd	$y1,`$LOCALS+8*1`($sp)
+	fmadd	$h2lo,$r2lo,$x0,$h2lo
+	 lfd	$y2,`$LOCALS+8*2`($sp)
+	fmadd	$h2hi,$r2hi,$x0,$h2hi
+	 lfd	$y3,`$LOCALS+8*3`($sp)
+	fmadd	$h1lo,$r1lo,$x0,$h1lo
+	fmadd	$h1hi,$r1hi,$x0,$h1hi
+	fmadd	$h3lo,$r3lo,$x0,$h3lo
+	fmadd	$h3hi,$r3hi,$x0,$h3hi
+
+	bdnz	Loop
+
+	######################################### base 2^48 -> base 2^32
+	fadd	$c0lo,$h0lo,$two32
+	fadd	$c0hi,$h0hi,$two32
+	fadd	$c2lo,$h2lo,$two96
+	fadd	$c2hi,$h2hi,$two96
+	fadd	$c1lo,$h1lo,$two64
+	fadd	$c1hi,$h1hi,$two64
+	fadd	$c3lo,$h3lo,$two130
+	fadd	$c3hi,$h3hi,$two130
+
+	fsub	$c0lo,$c0lo,$two32
+	fsub	$c0hi,$c0hi,$two32
+	fsub	$c2lo,$c2lo,$two96
+	fsub	$c2hi,$c2hi,$two96
+	fsub	$c1lo,$c1lo,$two64
+	fsub	$c1hi,$c1hi,$two64
+	fsub	$c3lo,$c3lo,$two130
+	fsub	$c3hi,$c3hi,$two130
+
+	fsub	$h1lo,$h1lo,$c1lo
+	fsub	$h1hi,$h1hi,$c1hi
+	fsub	$h3lo,$h3lo,$c3lo
+	fsub	$h3hi,$h3hi,$c3hi
+	fsub	$h2lo,$h2lo,$c2lo
+	fsub	$h2hi,$h2hi,$c2hi
+	fsub	$h0lo,$h0lo,$c0lo
+	fsub	$h0hi,$h0hi,$c0hi
+
+	fadd	$h1lo,$h1lo,$c0lo
+	fadd	$h1hi,$h1hi,$c0hi
+	fadd	$h3lo,$h3lo,$c2lo
+	fadd	$h3hi,$h3hi,$c2hi
+	fadd	$h2lo,$h2lo,$c1lo
+	fadd	$h2hi,$h2hi,$c1hi
+	fmadd	$h0lo,$c3lo,$five_two130,$h0lo
+	fmadd	$h0hi,$c3hi,$five_two130,$h0hi
+
+	fadd	$x1,$h1lo,$h1hi
+	fadd	$x3,$h3lo,$h3hi
+	fadd	$x2,$h2lo,$h2hi
+	fadd	$x0,$h0lo,$h0hi
+
+	lfd	$h0lo,`$LOCALS+8*4`($sp)	# pull saved fpscr
+	fadd	$x1,$x1,$two32			# bias
+	fadd	$x3,$x3,$two96
+	fadd	$x2,$x2,$two64
+	fadd	$x0,$x0,$two0
+
+	stfd	$x1,8*1($ctx)			# store [biased] hash value
+	stfd	$x3,8*3($ctx)
+	stfd	$x2,8*2($ctx)
+	stfd	$x0,8*0($ctx)
+
+	mtfsf	255,$h0lo			# restore original fpscr
+	lfd	f14,`$FRAME-8*18`($sp)
+	lfd	f15,`$FRAME-8*17`($sp)
+	lfd	f16,`$FRAME-8*16`($sp)
+	lfd	f17,`$FRAME-8*15`($sp)
+	lfd	f18,`$FRAME-8*14`($sp)
+	lfd	f19,`$FRAME-8*13`($sp)
+	lfd	f20,`$FRAME-8*12`($sp)
+	lfd	f21,`$FRAME-8*11`($sp)
+	lfd	f22,`$FRAME-8*10`($sp)
+	lfd	f23,`$FRAME-8*9`($sp)
+	lfd	f24,`$FRAME-8*8`($sp)
+	lfd	f25,`$FRAME-8*7`($sp)
+	lfd	f26,`$FRAME-8*6`($sp)
+	lfd	f27,`$FRAME-8*5`($sp)
+	lfd	f28,`$FRAME-8*4`($sp)
+	lfd	f29,`$FRAME-8*3`($sp)
+	lfd	f30,`$FRAME-8*2`($sp)
+	lfd	f31,`$FRAME-8*1`($sp)
+	addi	$sp,$sp,$FRAME
+Labort:
+	blr
+	.long	0
+	.byte	0,12,4,1,0x80,0,4,0
+.size	.poly1305_blocks_fpu,.-.poly1305_blocks_fpu
+___
+{
+my ($mac,$nonce)=($inp,$len);
+
+my ($h0,$h1,$h2,$h3,$h4, $d0,$d1,$d2,$d3
+   ) = map("r$_",(7..11,28..31));
+my $mask = "r0";
+my $FRAME = (6+4)*$SIZE_T;
+
+$code.=<<___;
+.globl	.poly1305_emit_fpu
+.align	4
+.poly1305_emit_fpu:
+	$STU	$sp,-$FRAME($sp)
+	mflr	r0
+	$PUSH	r28,`$FRAME-$SIZE_T*4`($sp)
+	$PUSH	r29,`$FRAME-$SIZE_T*3`($sp)
+	$PUSH	r30,`$FRAME-$SIZE_T*2`($sp)
+	$PUSH	r31,`$FRAME-$SIZE_T*1`($sp)
+	$PUSH	r0,`$FRAME+$LRSAVE`($sp)
+
+	lwz	$d0,`8*0+(0^$LITTLE_ENDIAN)`($ctx)	# load hash
+	lwz	$h0,`8*0+(4^$LITTLE_ENDIAN)`($ctx)
+	lwz	$d1,`8*1+(0^$LITTLE_ENDIAN)`($ctx)
+	lwz	$h1,`8*1+(4^$LITTLE_ENDIAN)`($ctx)
+	lwz	$d2,`8*2+(0^$LITTLE_ENDIAN)`($ctx)
+	lwz	$h2,`8*2+(4^$LITTLE_ENDIAN)`($ctx)
+	lwz	$d3,`8*3+(0^$LITTLE_ENDIAN)`($ctx)
+	lwz	$h3,`8*3+(4^$LITTLE_ENDIAN)`($ctx)
+
+	lis	$mask,0xfff0
+	andc	$d0,$d0,$mask			# mask exponent
+	andc	$d1,$d1,$mask
+	andc	$d2,$d2,$mask
+	andc	$d3,$d3,$mask			# can be partially reduced...
+	li	$mask,3
+
+	srwi	$padbit,$d3,2			# ... so reduce
+	and	$h4,$d3,$mask
+	andc	$d3,$d3,$mask
+	add	$d3,$d3,$padbit
+___
+						if ($SIZE_T==4) {
+$code.=<<___;
+	addc	$h0,$h0,$d3
+	adde	$h1,$h1,$d0
+	adde	$h2,$h2,$d1
+	adde	$h3,$h3,$d2
+	addze	$h4,$h4
+
+	addic	$d0,$h0,5			# compare to modulus
+	addze	$d1,$h1
+	addze	$d2,$h2
+	addze	$d3,$h3
+	addze	$mask,$h4
+
+	srwi	$mask,$mask,2			# did it carry/borrow?
+	neg	$mask,$mask
+	srawi	$mask,$mask,31			# mask
+
+	andc	$h0,$h0,$mask
+	and	$d0,$d0,$mask
+	andc	$h1,$h1,$mask
+	and	$d1,$d1,$mask
+	or	$h0,$h0,$d0
+	lwz	$d0,0($nonce)			# load nonce
+	andc	$h2,$h2,$mask
+	and	$d2,$d2,$mask
+	or	$h1,$h1,$d1
+	lwz	$d1,4($nonce)
+	andc	$h3,$h3,$mask
+	and	$d3,$d3,$mask
+	or	$h2,$h2,$d2
+	lwz	$d2,8($nonce)
+	or	$h3,$h3,$d3
+	lwz	$d3,12($nonce)
+
+	addc	$h0,$h0,$d0			# accumulate nonce
+	adde	$h1,$h1,$d1
+	adde	$h2,$h2,$d2
+	adde	$h3,$h3,$d3
+___
+						} else {
+$code.=<<___;
+	add	$h0,$h0,$d3
+	add	$h1,$h1,$d0
+	add	$h2,$h2,$d1
+	add	$h3,$h3,$d2
+
+	srdi	$d0,$h0,32
+	add	$h1,$h1,$d0
+	srdi	$d1,$h1,32
+	add	$h2,$h2,$d1
+	srdi	$d2,$h2,32
+	add	$h3,$h3,$d2
+	srdi	$d3,$h3,32
+	add	$h4,$h4,$d3
+
+	insrdi	$h0,$h1,32,0
+	insrdi	$h2,$h3,32,0
+
+	addic	$d0,$h0,5			# compare to modulus
+	addze	$d1,$h2
+	addze	$d2,$h4
+
+	srdi	$mask,$d2,2			# did it carry/borrow?
+	neg	$mask,$mask
+	sradi	$mask,$mask,63			# mask
+	ld	$d2,0($nonce)			# load nonce
+	ld	$d3,8($nonce)
+
+	andc	$h0,$h0,$mask
+	and	$d0,$d0,$mask
+	andc	$h2,$h2,$mask
+	and	$d1,$d1,$mask
+	or	$h0,$h0,$d0
+	or	$h2,$h2,$d1
+___
+$code.=<<___	if (!$LITTLE_ENDIAN);
+	rotldi	$d2,$d2,32			# flip nonce words
+	rotldi	$d3,$d3,32
+___
+$code.=<<___;
+	addc	$h0,$h0,$d2			# accumulate nonce
+	adde	$h2,$h2,$d3
+
+	srdi	$h1,$h0,32
+	srdi	$h3,$h2,32
+___
+						}
+$code.=<<___	if ($LITTLE_ENDIAN);
+	stw	$h0,0($mac)			# write result
+	stw	$h1,4($mac)
+	stw	$h2,8($mac)
+	stw	$h3,12($mac)
+___
+$code.=<<___	if (!$LITTLE_ENDIAN);
+	li	$d1,4
+	stwbrx	$h0,0,$mac			# write result
+	li	$d2,8
+	stwbrx	$h1,$d1,$mac
+	li	$d3,12
+	stwbrx	$h2,$d2,$mac
+	stwbrx	$h3,$d3,$mac
+___
+$code.=<<___;
+	$POP	r28,`$FRAME-$SIZE_T*4`($sp)
+	$POP	r29,`$FRAME-$SIZE_T*3`($sp)
+	$POP	r30,`$FRAME-$SIZE_T*2`($sp)
+	$POP	r31,`$FRAME-$SIZE_T*1`($sp)
+	addi	$sp,$sp,$FRAME
+	blr
+	.long	0
+	.byte	0,12,4,1,0x80,4,3,0
+.size	.poly1305_emit_fpu,.-.poly1305_emit_fpu
+___
+}
+# Ugly hack here, because PPC assembler syntax seem to vary too
+# much from platforms to platform...
+$code.=<<___;
+.align	6
+LPICmeup:
+	mflr	r0
+	bcl	20,31,\$+4
+	mflr	$len	# vvvvvv "distance" between . and 1st data entry
+	addi	$len,$len,`64-8`	# borrow $len
+	mtlr	r0
+	blr
+	.long	0
+	.byte	0,12,0x14,0,0,0,0,0
+	.space	`64-9*4`
+
+.quad	0x4330000000000000		# 2^(52+0)
+.quad	0x4530000000000000		# 2^(52+32)
+.quad	0x4730000000000000		# 2^(52+64)
+.quad	0x4930000000000000		# 2^(52+96)
+.quad	0x4b50000000000000		# 2^(52+130)
+
+.quad	0x37f4000000000000		# 5/2^130
+
+.quad	0x4430000000000000		# 2^(52+16+0)
+.quad	0x4630000000000000		# 2^(52+16+32)
+.quad	0x4830000000000000		# 2^(52+16+64)
+.quad	0x4a30000000000000		# 2^(52+16+96)
+.quad	0x3e30000000000000		# 2^(52+16+0-96)
+.quad	0x4030000000000000		# 2^(52+16+32-96)
+.quad	0x4230000000000000		# 2^(52+16+64-96)
+
+.quad	0x0000000000000001		# fpscr: truncate, no exceptions
+.asciz	"Poly1305 for PPC FPU, CRYPTOGAMS by <appro\@openssl.org>"
+.align	4
+___
+
+$code =~ s/\`([^\`]*)\`/eval $1/gem;
+print $code;
+close STDOUT;
diff --git a/src/crypto/zinc/poly1305/poly1305.c b/src/crypto/zinc/poly1305/poly1305.c
index 7d373b9..dd2e1a3 100644
--- a/src/crypto/zinc/poly1305/poly1305.c
+++ b/src/crypto/zinc/poly1305/poly1305.c
@@ -14,16 +14,85 @@
 #include <linux/kernel.h>
 #include <linux/string.h>
 #include <linux/module.h>
 #include <linux/init.h>
 
+#if defined(CONFIG_ZINC_ARCH_ARM) || defined(CONFIG_ZINC_ARCH_ARM64) || defined(CONFIG_ZINC_ARCH_PPC32) || defined(CONFIG_ZINC_ARCH_PPC64)
+#if defined(CONFIG_ZINC_ARCH_ARM64) || defined(CONFIG_ZINC_ARCH_PPC64)
+struct poly1305_arch_internal {
+	union {
+		u32 h[5];
+		struct {
+			u64 h0, h1, h2;
+		};
+	};
+	u64 is_base2_26;
+	u64 r[2];
+};
+#elif defined(CONFIG_ZINC_ARCH_ARM) || defined(CONFIG_ZINC_ARCH_PPC32)
+struct poly1305_arch_internal {
+	union {
+		u32 h[5];
+		struct {
+			u64 h0, h1;
+			u32 h2;
+		} __packed;
+	};
+	u32 r[4];
+	u32 is_base2_26;
+};
+#endif
+/* The NEON and AVX code uses base 2^26, while the scalar code uses base 2^64 on 64-bit
+ * and base 2^32 on 32-bit. If we hit the unfortunate situation of using NEON or AVX
+ * and then having to go back to scalar -- because the user is silly and has
+ * called the update function from two separate contexts -- then we need to
+ * convert back to the original base before proceeding. The below function is
+ * written for 64-bit integers, and so we have to swap words at the end on
+ * big-endian 32-bit. It is possible to reason that the initial reduction below
+ * is sufficient given the implementation invariants. However, for an avoidance
+ * of doubt and because this is not performance critical, we do the full
+ * reduction anyway.
+ */
+static void convert_to_base2_64(void *ctx)
+{
+	struct poly1305_arch_internal *state = ctx;
+	u32 cy;
+
+	if (!(IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || IS_ENABLED(CONFIG_AVX)) || !state->is_base2_26)
+		return;
+
+	cy = state->h[0] >> 26; state->h[0] &= 0x3ffffff; state->h[1] += cy;
+	cy = state->h[1] >> 26; state->h[1] &= 0x3ffffff; state->h[2] += cy;
+	cy = state->h[2] >> 26; state->h[2] &= 0x3ffffff; state->h[3] += cy;
+	cy = state->h[3] >> 26; state->h[3] &= 0x3ffffff; state->h[4] += cy;
+	state->h0 = ((u64)state->h[2] << 52) | ((u64)state->h[1] << 26) | state->h[0];
+	state->h1 = ((u64)state->h[4] << 40) | ((u64)state->h[3] << 14) | (state->h[2] >> 12);
+	state->h2 = state->h[4] >> 24;
+	if ((IS_ENABLED(CONFIG_ZINC_ARCH_ARM) || IS_ENABLED(CONFIG_ZINC_ARCH_PPC32)) &&
+	    IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) {
+		state->h0 = rol64(state->h0, 32);
+		state->h1 = rol64(state->h1, 32);
+	}
+#define ULT(a, b) ((a ^ ((a ^ b) | ((a - b) ^ b))) >> (sizeof(a) * 8 - 1))
+	cy = (state->h2 >> 2) + (state->h2 & ~3ULL);
+	state->h2 &= 3;
+	state->h0 += cy;
+	state->h1 += (cy = ULT(state->h0, cy));
+	state->h2 += ULT(state->h1, cy);
+#undef ULT
+	state->is_base2_26 = 0;
+}
+#endif
+
 #if defined(CONFIG_ZINC_ARCH_X86_64)
 #include "poly1305-x86_64-glue.c"
 #elif defined(CONFIG_ZINC_ARCH_ARM) || defined(CONFIG_ZINC_ARCH_ARM64)
 #include "poly1305-arm-glue.c"
 #elif defined(CONFIG_ZINC_ARCH_MIPS) || defined(CONFIG_ZINC_ARCH_MIPS64)
 #include "poly1305-mips-glue.c"
+#elif defined(CONFIG_ZINC_ARCH_PPC32) || defined(CONFIG_ZINC_ARCH_PPC64)
+#include "poly1305-ppc-glue.c"
 #else
 static inline bool poly1305_init_arch(void *ctx,
 				      const u8 key[POLY1305_KEY_SIZE])
 {
 	return false;
-- 
2.20.1

_______________________________________________
WireGuard mailing list
WireGuard@lists.zx2c4.com
https://lists.zx2c4.com/mailman/listinfo/wireguard

^ permalink raw reply	[flat|nested] 5+ messages in thread

* [PATCH 1/2 v3] [Zinc] Add PowerPC chacha20 implementation from openssl/cryptograms
  2019-05-11 18:03 ` [PATCH 1/2] " Shawn Landden
  2019-05-11 18:03   ` [PATCH 2/2] [zinc] add accelerated poly1305 " Shawn Landden
@ 2019-05-13 21:31   ` " Shawn Landden
  2019-05-13 21:31     ` [PATCH 2/2 v3] [zinc] Add PowerPC accelerated poly1305 " Shawn Landden
  1 sibling, 1 reply; 5+ messages in thread
From: Shawn Landden @ 2019-05-13 21:31 UTC (permalink / raw)
  To: wireguard

Only runs on the out-bound path, as the in-bound path is in an interrupt,
but that can be fixed in Linux.

Otherwise this is tested (with the VSX code enabled) on Power 9 (ppc64le).
Without this I get 2GiB/s over the loopback (so 4 GiB/s), and with this
I get 2.8 GiB/s (so 5.6 GiB/s), and more time is spent in poly1305 than chacha20.
This is on a 4-thread VPS.

Signed-off-by: Shawn Landden <shawn@git.icu>

v2: more complete simd.h for PPC
    benchmarks
    whitespace issues
v3: honor CONFIG_ALTIVEC (CONFIG_VSX has already honored)
---
 src/compat/simd/include/linux/simd.h         |   29 +-
 src/crypto/Kbuild.include                    |   18 +-
 src/crypto/zinc/chacha20/chacha20-ppc-glue.c |   66 +
 src/crypto/zinc/chacha20/chacha20-ppc.pl     | 1355 ++++++++++++++++++
 src/crypto/zinc/chacha20/chacha20.c          |    2 +
 src/crypto/zinc/chacha20/ppc-xlate.pl        |  353 +++++
 6 files changed, 1820 insertions(+), 3 deletions(-)
 create mode 100644 src/crypto/zinc/chacha20/chacha20-ppc-glue.c
 create mode 100644 src/crypto/zinc/chacha20/chacha20-ppc.pl
 create mode 100644 src/crypto/zinc/chacha20/ppc-xlate.pl

diff --git a/src/compat/simd/include/linux/simd.h b/src/compat/simd/include/linux/simd.h
index c75c724..44060a9 100644
--- a/src/compat/simd/include/linux/simd.h
+++ b/src/compat/simd/include/linux/simd.h
@@ -11,10 +11,13 @@
 #if defined(CONFIG_X86_64)
 #include <linux/version.h>
 #include <asm/fpu/api.h>
 #elif defined(CONFIG_KERNEL_MODE_NEON)
 #include <asm/neon.h>
+#elif defined(CONFIG_ALTIVEC) || defined(CONFIG_VSX)
+#include <asm/switch_to.h>
+#include <asm/cputable.h>
 #endif
 
 typedef enum {
 	HAVE_NO_SIMD = 1 << 0,
 	HAVE_FULL_SIMD = 1 << 1,
@@ -28,17 +31,28 @@ static inline void simd_get(simd_context_t *ctx)
 	*ctx = !IS_ENABLED(CONFIG_PREEMPT_RT_BASE) && may_use_simd() ? HAVE_FULL_SIMD : HAVE_NO_SIMD;
 }
 
 static inline void simd_put(simd_context_t *ctx)
 {
+        if (*ctx & HAVE_SIMD_IN_USE) {
 #if defined(CONFIG_X86_64)
-	if (*ctx & HAVE_SIMD_IN_USE)
 		kernel_fpu_end();
 #elif defined(CONFIG_KERNEL_MODE_NEON)
-	if (*ctx & HAVE_SIMD_IN_USE)
 		kernel_neon_end();
+#elif defined(CONFIG_PPC32) || defined(CONFIG_PPC64)
+		if (cpu_has_feature(CPU_FTR_VSX_COMP)) {
+			disable_kernel_vsx();
+			preempt_enable();
+		} else if (cpu_has_feature(CPU_FTR_ALTIVEC_COMP)) {
+			disable_kernel_altivec();
+			preempt_enable();
+		} else if (!cpu_has_feature(CPU_FTR_FPU_UNAVAILABLE)) {
+			disable_kernel_fp();
+			preempt_enable();
+		}
 #endif
+	}
 	*ctx = HAVE_NO_SIMD;
 }
 
 static inline bool simd_relax(simd_context_t *ctx)
 {
@@ -60,10 +74,21 @@ static __must_check inline bool simd_use(simd_context_t *ctx)
 		return true;
 #if defined(CONFIG_X86_64)
 	kernel_fpu_begin();
 #elif defined(CONFIG_KERNEL_MODE_NEON)
 	kernel_neon_begin();
+#elif defined(CONFIG_PPC32) || defined(CONFIG_PPC64)
+		if (cpu_has_feature(CPU_FTR_VSX_COMP)) {
+			preempt_disable();
+			enable_kernel_vsx();
+		} else if (cpu_has_feature(CPU_FTR_ALTIVEC_COMP)) {
+			preempt_disable();
+			enable_kernel_altivec();
+		} else if (!cpu_has_feature(CPU_FTR_FPU_UNAVAILABLE)) {
+			preempt_disable();
+			enable_kernel_fp();
+		}
 #endif
 	*ctx |= HAVE_SIMD_IN_USE;
 	return true;
 }
 
diff --git a/src/crypto/Kbuild.include b/src/crypto/Kbuild.include
index 460684d..4e05181 100644
--- a/src/crypto/Kbuild.include
+++ b/src/crypto/Kbuild.include
@@ -11,17 +11,25 @@ ifeq ($(CONFIG_MIPS)$(CONFIG_CPU_MIPS32_R2),yy)
 CONFIG_ZINC_ARCH_MIPS := y
 endif
 ifeq ($(CONFIG_MIPS)$(CONFIG_64BIT),yy)
 CONFIG_ZINC_ARCH_MIPS64 := y
 endif
+ifeq ($(CONFIG_PPC32),y)
+CONFIG_ZINC_ARCH_PPC32 := y
+endif
+ifeq ($(CONFIG_PPC64),y)
+CONFIG_ZINC_ARCH_PPC64 := y
+endif
 
 zinc-y += chacha20/chacha20.o
 zinc-$(CONFIG_ZINC_ARCH_X86_64) += chacha20/chacha20-x86_64.o
 zinc-$(CONFIG_ZINC_ARCH_ARM) += chacha20/chacha20-arm.o chacha20/chacha20-unrolled-arm.o
 zinc-$(CONFIG_ZINC_ARCH_ARM64) += chacha20/chacha20-arm64.o
 zinc-$(CONFIG_ZINC_ARCH_MIPS) += chacha20/chacha20-mips.o
 AFLAGS_chacha20-mips.o += -O2 # This is required to fill the branch delay slots
+zinc-$(CONFIG_ZINC_ARCH_PPC32) += chacha20/chacha20-ppc.o
+zinc-$(CONFIG_ZINC_ARCH_PPC64) += chacha20/chacha20-ppc.o
 
 zinc-y += poly1305/poly1305.o
 zinc-$(CONFIG_ZINC_ARCH_X86_64) += poly1305/poly1305-x86_64.o
 zinc-$(CONFIG_ZINC_ARCH_ARM) += poly1305/poly1305-arm.o
 zinc-$(CONFIG_ZINC_ARCH_ARM64) += poly1305/poly1305-arm64.o
@@ -36,22 +44,30 @@ zinc-$(CONFIG_ZINC_ARCH_X86_64) += blake2s/blake2s-x86_64.o
 
 zinc-y += curve25519/curve25519.o
 zinc-$(CONFIG_ZINC_ARCH_ARM) += curve25519/curve25519-arm.o
 
 quiet_cmd_perlasm = PERLASM $@
-      cmd_perlasm = $(PERL) $< > $@
+      cmd_perlasm = $(PERL) $< $(perlflags-y) > $@
 $(obj)/%.S: $(src)/%.pl FORCE
 	$(call if_changed,perlasm)
 kbuild-dir := $(if $(filter /%,$(src)),$(src),$(srctree)/$(src))
 targets := $(patsubst $(kbuild-dir)/%.pl,%.S,$(wildcard $(patsubst %.o,$(kbuild-dir)/crypto/zinc/%.pl,$(zinc-y) $(zinc-m) $(zinc-))))
 
+perlflags-$(CONFIG_ZINC_ARCH_PPC32) += linux32
+ifeq ($(CONFIG_ZINC_ARCH_PPC64),y)
+perlflags-$(CONFIG_CPU_BIG_ENDIAN) += linux64
+perlflags-$(CONFIG_CPU_LITTLE_ENDIAN) += linux64le
+endif
+
 # Old kernels don't set this, which causes trouble.
 .SECONDARY:
 
 wireguard-y += $(addprefix crypto/zinc/,$(zinc-y))
 ccflags-y += -I$(src)/crypto/include
 ccflags-$(CONFIG_ZINC_ARCH_X86_64) += -DCONFIG_ZINC_ARCH_X86_64
 ccflags-$(CONFIG_ZINC_ARCH_ARM) += -DCONFIG_ZINC_ARCH_ARM
 ccflags-$(CONFIG_ZINC_ARCH_ARM64) += -DCONFIG_ZINC_ARCH_ARM64
 ccflags-$(CONFIG_ZINC_ARCH_MIPS) += -DCONFIG_ZINC_ARCH_MIPS
 ccflags-$(CONFIG_ZINC_ARCH_MIPS64) += -DCONFIG_ZINC_ARCH_MIPS64
+ccflags-$(CONFIG_ZINC_ARCH_PPC32) += -DCONFIG_ZINC_ARCH_PPC32
+ccflags-$(CONFIG_ZINC_ARCH_PPC64) += -DCONFIG_ZINC_ARCH_PPC64
 ccflags-$(CONFIG_WIREGUARD_DEBUG) += -DCONFIG_ZINC_SELFTEST
diff --git a/src/crypto/zinc/chacha20/chacha20-ppc-glue.c b/src/crypto/zinc/chacha20/chacha20-ppc-glue.c
new file mode 100644
index 0000000..ebcc26f
--- /dev/null
+++ b/src/crypto/zinc/chacha20/chacha20-ppc-glue.c
@@ -0,0 +1,66 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (C) 2019 Shawn Landden <shawn@git.icu>. All Rights Reserved.
+ */
+
+asmlinkage void ChaCha20_ctr32_int(u8 *out, const u8 *inp,
+				   size_t len, const u32 key[8],
+				   const u32 counter[4]);
+asmlinkage void ChaCha20_ctr32_vmx(u8 *out, const u8 *inp,
+				   size_t len, const u32 key[8],
+				   const u32 counter[4]);
+asmlinkage void ChaCha20_ctr32_vsx(u8 *out, const u8 *inp,
+				   size_t len, const u32 key[8],
+				   const u32 counter[4]);
+static bool *const chacha20_nobs[] __initconst = { };
+static void __init chacha20_fpu_init(void) {}
+
+static inline bool chacha20_arch(struct chacha20_ctx *ctx, u8 *dst,
+				 const u8 *src, size_t len,
+				 simd_context_t *simd_context)
+{
+	void (*ChaCha20SIMD)(u8 *out, const u8 *inp,
+			     size_t len, const u32 key[8],
+			     const u32 counter[4]);
+
+	/* SIMD disables preemption, so relax after processing each page. */
+	BUILD_BUG_ON(PAGE_SIZE < CHACHA20_BLOCK_SIZE ||
+		     PAGE_SIZE % CHACHA20_BLOCK_SIZE);
+
+	if (cpu_has_feature(CPU_FTR_VSX_COMP))
+		ChaCha20SIMD = &ChaCha20_ctr32_vsx;
+	else if (cpu_has_feature(CPU_FTR_ALTIVEC_COMP))
+                ChaCha20SIMD = &ChaCha20_ctr32_vmx;
+	else {
+		ChaCha20_ctr32_int(dst, src, len, ctx->key, ctx->counter);
+		return true;
+	}
+
+	for (;;) {
+		if (len >= CHACHA20_BLOCK_SIZE * 3 && simd_use(simd_context)) {
+			const size_t bytes = min_t(size_t, len, PAGE_SIZE);
+
+			ChaCha20SIMD(dst, src, bytes, ctx->key, ctx->counter);
+			ctx->counter[0] += (bytes + 63) / 64;
+			len -= bytes;
+			if (!len)
+				break;
+			dst += bytes;
+			src += bytes;
+			simd_relax(simd_context);
+		} else {
+			ChaCha20_ctr32_int(dst, src, len, ctx->key, ctx->counter);
+			ctx->counter[0] += (len + 63) / 64;
+			return true;
+		}
+	}
+	return true;
+}
+
+static inline bool hchacha20_arch(u32 derived_key[CHACHA20_KEY_WORDS],
+				  const u8 nonce[HCHACHA20_NONCE_SIZE],
+				  const u8 key[HCHACHA20_KEY_SIZE],
+				  simd_context_t *simd_context)
+{
+	return false;
+}
diff --git a/src/crypto/zinc/chacha20/chacha20-ppc.pl b/src/crypto/zinc/chacha20/chacha20-ppc.pl
new file mode 100644
index 0000000..07468c8
--- /dev/null
+++ b/src/crypto/zinc/chacha20/chacha20-ppc.pl
@@ -0,0 +1,1355 @@
+#! /usr/bin/env perl
+# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+#
+# This code is taken from the OpenSSL project but the author, Andy Polyakov,
+# has relicensed it under the licenses specified in the SPDX header above.
+# The original headers, including the original license headers, are
+# included below for completeness.
+#
+# Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the Apache License 2.0 (the "License").  You may not use
+# this file except in compliance with the License.  You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
+#
+# ====================================================================
+# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
+# project. The module is, however, dual licensed under OpenSSL and
+# CRYPTOGAMS licenses depending on where you obtain it. For further
+# details see http://www.openssl.org/~appro/cryptogams/.
+# ====================================================================
+#
+# October 2015
+#
+# ChaCha20 for PowerPC/AltiVec.
+#
+# June 2018
+#
+# Add VSX 2.07 code path. Original 3xAltiVec+1xIALU is well-suited for
+# processors that can't issue more than one vector instruction per
+# cycle. But POWER8 (and POWER9) can issue a pair, and vector-only 4x
+# interleave would perform better. Incidentally PowerISA 2.07 (first
+# implemented by POWER8) defined new usable instructions, hence 4xVSX
+# code path...
+#
+# Performance in cycles per byte out of large buffer.
+#
+#			IALU/gcc-4.x    3xAltiVec+1xIALU	4xVSX
+#
+# Freescale e300	13.6/+115%	-			-
+# PPC74x0/G4e		6.81/+310%	3.81			-
+# PPC970/G5		9.29/+160%	?			-
+# POWER7		8.62/+61%	3.35			-
+# POWER8		8.70/+51%	2.91			2.09
+# POWER9		8.80/+29%	4.44(*)			2.45(**)
+#
+# (*)	this is trade-off result, it's possible to improve it, but
+#	then it would negatively affect all others;
+# (**)	POWER9 seems to be "allergic" to mixing vector and integer
+#	instructions, which is why switch to vector-only code pays
+#	off that much;
+
+$flavour = shift;
+
+if ($flavour =~ /64/) {
+	$SIZE_T	=8;
+	$LRSAVE	=2*$SIZE_T;
+	$STU	="stdu";
+	$POP	="ld";
+	$PUSH	="std";
+	$UCMP	="cmpld";
+} elsif ($flavour =~ /32/) {
+	$SIZE_T	=4;
+	$LRSAVE	=$SIZE_T;
+	$STU	="stwu";
+	$POP	="lwz";
+	$PUSH	="stw";
+	$UCMP	="cmplw";
+} else { die "nonsense $flavour"; }
+
+$LITTLE_ENDIAN = ($flavour=~/le$/) ? 1 : 0;
+
+$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
+( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
+( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
+die "can't locate ppc-xlate.pl";
+
+open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";
+
+$LOCALS=6*$SIZE_T;
+$FRAME=$LOCALS+64+18*$SIZE_T;	# 64 is for local variables
+
+sub AUTOLOAD()		# thunk [simplified] x86-style perlasm
+{ my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./;
+    $code .= "\t$opcode\t".join(',',@_)."\n";
+}
+
+my $sp = "r1";
+
+my ($out,$inp,$len,$key,$ctr) = map("r$_",(3..7));
+
+my @x=map("r$_",(16..31));
+my @d=map("r$_",(11,12,14,15));
+my @t=map("r$_",(7..10));
+
+sub ROUND {
+my ($a0,$b0,$c0,$d0)=@_;
+my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
+my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
+my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
+
+    (
+	"&add		(@x[$a0],@x[$a0],@x[$b0])",
+	 "&add		(@x[$a1],@x[$a1],@x[$b1])",
+	  "&add		(@x[$a2],@x[$a2],@x[$b2])",
+	   "&add	(@x[$a3],@x[$a3],@x[$b3])",
+	"&xor		(@x[$d0],@x[$d0],@x[$a0])",
+	 "&xor		(@x[$d1],@x[$d1],@x[$a1])",
+	  "&xor		(@x[$d2],@x[$d2],@x[$a2])",
+	   "&xor	(@x[$d3],@x[$d3],@x[$a3])",
+	"&rotlwi	(@x[$d0],@x[$d0],16)",
+	 "&rotlwi	(@x[$d1],@x[$d1],16)",
+	  "&rotlwi	(@x[$d2],@x[$d2],16)",
+	   "&rotlwi	(@x[$d3],@x[$d3],16)",
+
+	"&add		(@x[$c0],@x[$c0],@x[$d0])",
+	 "&add		(@x[$c1],@x[$c1],@x[$d1])",
+	  "&add		(@x[$c2],@x[$c2],@x[$d2])",
+	   "&add	(@x[$c3],@x[$c3],@x[$d3])",
+	"&xor		(@x[$b0],@x[$b0],@x[$c0])",
+	 "&xor		(@x[$b1],@x[$b1],@x[$c1])",
+	  "&xor		(@x[$b2],@x[$b2],@x[$c2])",
+	   "&xor	(@x[$b3],@x[$b3],@x[$c3])",
+	"&rotlwi	(@x[$b0],@x[$b0],12)",
+	 "&rotlwi	(@x[$b1],@x[$b1],12)",
+	  "&rotlwi	(@x[$b2],@x[$b2],12)",
+	   "&rotlwi	(@x[$b3],@x[$b3],12)",
+
+	"&add		(@x[$a0],@x[$a0],@x[$b0])",
+	 "&add		(@x[$a1],@x[$a1],@x[$b1])",
+	  "&add		(@x[$a2],@x[$a2],@x[$b2])",
+	   "&add	(@x[$a3],@x[$a3],@x[$b3])",
+	"&xor		(@x[$d0],@x[$d0],@x[$a0])",
+	 "&xor		(@x[$d1],@x[$d1],@x[$a1])",
+	  "&xor		(@x[$d2],@x[$d2],@x[$a2])",
+	   "&xor	(@x[$d3],@x[$d3],@x[$a3])",
+	"&rotlwi	(@x[$d0],@x[$d0],8)",
+	 "&rotlwi	(@x[$d1],@x[$d1],8)",
+	  "&rotlwi	(@x[$d2],@x[$d2],8)",
+	   "&rotlwi	(@x[$d3],@x[$d3],8)",
+
+	"&add		(@x[$c0],@x[$c0],@x[$d0])",
+	 "&add		(@x[$c1],@x[$c1],@x[$d1])",
+	  "&add		(@x[$c2],@x[$c2],@x[$d2])",
+	   "&add	(@x[$c3],@x[$c3],@x[$d3])",
+	"&xor		(@x[$b0],@x[$b0],@x[$c0])",
+	 "&xor		(@x[$b1],@x[$b1],@x[$c1])",
+	  "&xor		(@x[$b2],@x[$b2],@x[$c2])",
+	   "&xor	(@x[$b3],@x[$b3],@x[$c3])",
+	"&rotlwi	(@x[$b0],@x[$b0],7)",
+	 "&rotlwi	(@x[$b1],@x[$b1],7)",
+	  "&rotlwi	(@x[$b2],@x[$b2],7)",
+	   "&rotlwi	(@x[$b3],@x[$b3],7)"
+    );
+}
+
+$code.=<<___;
+.machine	"any"
+.text
+
+.globl	.ChaCha20_ctr32_int
+.align	5
+.ChaCha20_ctr32_int:
+__ChaCha20_ctr32_int:
+	${UCMP}i $len,0
+	beqlr-
+
+	$STU	$sp,-$FRAME($sp)
+	mflr	r0
+
+	$PUSH	r14,`$FRAME-$SIZE_T*18`($sp)
+	$PUSH	r15,`$FRAME-$SIZE_T*17`($sp)
+	$PUSH	r16,`$FRAME-$SIZE_T*16`($sp)
+	$PUSH	r17,`$FRAME-$SIZE_T*15`($sp)
+	$PUSH	r18,`$FRAME-$SIZE_T*14`($sp)
+	$PUSH	r19,`$FRAME-$SIZE_T*13`($sp)
+	$PUSH	r20,`$FRAME-$SIZE_T*12`($sp)
+	$PUSH	r21,`$FRAME-$SIZE_T*11`($sp)
+	$PUSH	r22,`$FRAME-$SIZE_T*10`($sp)
+	$PUSH	r23,`$FRAME-$SIZE_T*9`($sp)
+	$PUSH	r24,`$FRAME-$SIZE_T*8`($sp)
+	$PUSH	r25,`$FRAME-$SIZE_T*7`($sp)
+	$PUSH	r26,`$FRAME-$SIZE_T*6`($sp)
+	$PUSH	r27,`$FRAME-$SIZE_T*5`($sp)
+	$PUSH	r28,`$FRAME-$SIZE_T*4`($sp)
+	$PUSH	r29,`$FRAME-$SIZE_T*3`($sp)
+	$PUSH	r30,`$FRAME-$SIZE_T*2`($sp)
+	$PUSH	r31,`$FRAME-$SIZE_T*1`($sp)
+	$PUSH	r0,`$FRAME+$LRSAVE`($sp)
+
+	lwz	@d[0],0($ctr)			# load counter
+	lwz	@d[1],4($ctr)
+	lwz	@d[2],8($ctr)
+	lwz	@d[3],12($ctr)
+
+	bl	__ChaCha20_1x
+
+	$POP	r0,`$FRAME+$LRSAVE`($sp)
+	$POP	r14,`$FRAME-$SIZE_T*18`($sp)
+	$POP	r15,`$FRAME-$SIZE_T*17`($sp)
+	$POP	r16,`$FRAME-$SIZE_T*16`($sp)
+	$POP	r17,`$FRAME-$SIZE_T*15`($sp)
+	$POP	r18,`$FRAME-$SIZE_T*14`($sp)
+	$POP	r19,`$FRAME-$SIZE_T*13`($sp)
+	$POP	r20,`$FRAME-$SIZE_T*12`($sp)
+	$POP	r21,`$FRAME-$SIZE_T*11`($sp)
+	$POP	r22,`$FRAME-$SIZE_T*10`($sp)
+	$POP	r23,`$FRAME-$SIZE_T*9`($sp)
+	$POP	r24,`$FRAME-$SIZE_T*8`($sp)
+	$POP	r25,`$FRAME-$SIZE_T*7`($sp)
+	$POP	r26,`$FRAME-$SIZE_T*6`($sp)
+	$POP	r27,`$FRAME-$SIZE_T*5`($sp)
+	$POP	r28,`$FRAME-$SIZE_T*4`($sp)
+	$POP	r29,`$FRAME-$SIZE_T*3`($sp)
+	$POP	r30,`$FRAME-$SIZE_T*2`($sp)
+	$POP	r31,`$FRAME-$SIZE_T*1`($sp)
+	mtlr	r0
+	addi	$sp,$sp,$FRAME
+	blr
+	.long	0
+	.byte	0,12,4,1,0x80,18,5,0
+	.long	0
+.size	.ChaCha20_ctr32_int,.-.ChaCha20_ctr32_int
+
+.align	5
+__ChaCha20_1x:
+Loop_outer:
+	lis	@x[0],0x6170			# synthesize sigma
+	lis	@x[1],0x3320
+	lis	@x[2],0x7962
+	lis	@x[3],0x6b20
+	ori	@x[0],@x[0],0x7865
+	ori	@x[1],@x[1],0x646e
+	ori	@x[2],@x[2],0x2d32
+	ori	@x[3],@x[3],0x6574
+
+	li	r0,10				# inner loop counter
+	lwz	@x[4],0($key)			# load key
+	lwz	@x[5],4($key)
+	lwz	@x[6],8($key)
+	lwz	@x[7],12($key)
+	lwz	@x[8],16($key)
+	mr	@x[12],@d[0]			# copy counter
+	lwz	@x[9],20($key)
+	mr	@x[13],@d[1]
+	lwz	@x[10],24($key)
+	mr	@x[14],@d[2]
+	lwz	@x[11],28($key)
+	mr	@x[15],@d[3]
+
+	mr	@t[0],@x[4]
+	mr	@t[1],@x[5]
+	mr	@t[2],@x[6]
+	mr	@t[3],@x[7]
+
+	mtctr	r0
+Loop:
+___
+	foreach (&ROUND(0, 4, 8,12)) { eval; }
+	foreach (&ROUND(0, 5,10,15)) { eval; }
+$code.=<<___;
+	bdnz	Loop
+
+	subic	$len,$len,64			# $len-=64
+	addi	@x[0],@x[0],0x7865		# accumulate key block
+	addi	@x[1],@x[1],0x646e
+	addi	@x[2],@x[2],0x2d32
+	addi	@x[3],@x[3],0x6574
+	addis	@x[0],@x[0],0x6170
+	addis	@x[1],@x[1],0x3320
+	addis	@x[2],@x[2],0x7962
+	addis	@x[3],@x[3],0x6b20
+
+	subfe.	r0,r0,r0			# borrow?-1:0
+	add	@x[4],@x[4],@t[0]
+	lwz	@t[0],16($key)
+	add	@x[5],@x[5],@t[1]
+	lwz	@t[1],20($key)
+	add	@x[6],@x[6],@t[2]
+	lwz	@t[2],24($key)
+	add	@x[7],@x[7],@t[3]
+	lwz	@t[3],28($key)
+	add	@x[8],@x[8],@t[0]
+	add	@x[9],@x[9],@t[1]
+	add	@x[10],@x[10],@t[2]
+	add	@x[11],@x[11],@t[3]
+
+	add	@x[12],@x[12],@d[0]
+	add	@x[13],@x[13],@d[1]
+	add	@x[14],@x[14],@d[2]
+	add	@x[15],@x[15],@d[3]
+	addi	@d[0],@d[0],1			# increment counter
+___
+if (!$LITTLE_ENDIAN) { for($i=0;$i<16;$i++) {	# flip byte order
+$code.=<<___;
+	mr	@t[$i&3],@x[$i]
+	rotlwi	@x[$i],@x[$i],8
+	rlwimi	@x[$i],@t[$i&3],24,0,7
+	rlwimi	@x[$i],@t[$i&3],24,16,23
+___
+} }
+$code.=<<___;
+	bne	Ltail				# $len-=64 borrowed
+
+	lwz	@t[0],0($inp)			# load input, aligned or not
+	lwz	@t[1],4($inp)
+	${UCMP}i $len,0				# done already?
+	lwz	@t[2],8($inp)
+	lwz	@t[3],12($inp)
+	xor	@x[0],@x[0],@t[0]		# xor with input
+	lwz	@t[0],16($inp)
+	xor	@x[1],@x[1],@t[1]
+	lwz	@t[1],20($inp)
+	xor	@x[2],@x[2],@t[2]
+	lwz	@t[2],24($inp)
+	xor	@x[3],@x[3],@t[3]
+	lwz	@t[3],28($inp)
+	xor	@x[4],@x[4],@t[0]
+	lwz	@t[0],32($inp)
+	xor	@x[5],@x[5],@t[1]
+	lwz	@t[1],36($inp)
+	xor	@x[6],@x[6],@t[2]
+	lwz	@t[2],40($inp)
+	xor	@x[7],@x[7],@t[3]
+	lwz	@t[3],44($inp)
+	xor	@x[8],@x[8],@t[0]
+	lwz	@t[0],48($inp)
+	xor	@x[9],@x[9],@t[1]
+	lwz	@t[1],52($inp)
+	xor	@x[10],@x[10],@t[2]
+	lwz	@t[2],56($inp)
+	xor	@x[11],@x[11],@t[3]
+	lwz	@t[3],60($inp)
+	xor	@x[12],@x[12],@t[0]
+	stw	@x[0],0($out)			# store output, aligned or not
+	xor	@x[13],@x[13],@t[1]
+	stw	@x[1],4($out)
+	xor	@x[14],@x[14],@t[2]
+	stw	@x[2],8($out)
+	xor	@x[15],@x[15],@t[3]
+	stw	@x[3],12($out)
+	stw	@x[4],16($out)
+	stw	@x[5],20($out)
+	stw	@x[6],24($out)
+	stw	@x[7],28($out)
+	stw	@x[8],32($out)
+	stw	@x[9],36($out)
+	stw	@x[10],40($out)
+	stw	@x[11],44($out)
+	stw	@x[12],48($out)
+	stw	@x[13],52($out)
+	stw	@x[14],56($out)
+	addi	$inp,$inp,64
+	stw	@x[15],60($out)
+	addi	$out,$out,64
+
+	bne	Loop_outer
+
+	blr
+
+.align	4
+Ltail:
+	addi	$len,$len,64			# restore tail length
+	subi	$inp,$inp,1			# prepare for *++ptr
+	subi	$out,$out,1
+	addi	@t[0],$sp,$LOCALS-1
+	mtctr	$len
+
+	stw	@x[0],`$LOCALS+0`($sp)		# save whole block to stack
+	stw	@x[1],`$LOCALS+4`($sp)
+	stw	@x[2],`$LOCALS+8`($sp)
+	stw	@x[3],`$LOCALS+12`($sp)
+	stw	@x[4],`$LOCALS+16`($sp)
+	stw	@x[5],`$LOCALS+20`($sp)
+	stw	@x[6],`$LOCALS+24`($sp)
+	stw	@x[7],`$LOCALS+28`($sp)
+	stw	@x[8],`$LOCALS+32`($sp)
+	stw	@x[9],`$LOCALS+36`($sp)
+	stw	@x[10],`$LOCALS+40`($sp)
+	stw	@x[11],`$LOCALS+44`($sp)
+	stw	@x[12],`$LOCALS+48`($sp)
+	stw	@x[13],`$LOCALS+52`($sp)
+	stw	@x[14],`$LOCALS+56`($sp)
+	stw	@x[15],`$LOCALS+60`($sp)
+
+Loop_tail:					# byte-by-byte loop
+	lbzu	@d[0],1($inp)
+	lbzu	@x[0],1(@t[0])
+	xor	@d[1],@d[0],@x[0]
+	stbu	@d[1],1($out)
+	bdnz	Loop_tail
+
+	stw	$sp,`$LOCALS+0`($sp)		# wipe block on stack
+	stw	$sp,`$LOCALS+4`($sp)
+	stw	$sp,`$LOCALS+8`($sp)
+	stw	$sp,`$LOCALS+12`($sp)
+	stw	$sp,`$LOCALS+16`($sp)
+	stw	$sp,`$LOCALS+20`($sp)
+	stw	$sp,`$LOCALS+24`($sp)
+	stw	$sp,`$LOCALS+28`($sp)
+	stw	$sp,`$LOCALS+32`($sp)
+	stw	$sp,`$LOCALS+36`($sp)
+	stw	$sp,`$LOCALS+40`($sp)
+	stw	$sp,`$LOCALS+44`($sp)
+	stw	$sp,`$LOCALS+48`($sp)
+	stw	$sp,`$LOCALS+52`($sp)
+	stw	$sp,`$LOCALS+56`($sp)
+	stw	$sp,`$LOCALS+60`($sp)
+
+	blr
+	.long	0
+	.byte	0,12,0x14,0,0,0,0,0
+___
+
+{{{
+my ($A0,$B0,$C0,$D0,$A1,$B1,$C1,$D1,$A2,$B2,$C2,$D2)
+				= map("v$_",(0..11));
+my @K				= map("v$_",(12..17));
+my ($FOUR,$sixteen,$twenty4)	= map("v$_",(18..19,23));
+my ($inpperm,$outperm,$outmask)	= map("v$_",(24..26));
+my @D				= map("v$_",(27..31));
+my ($twelve,$seven,$T0,$T1) = @D;
+
+my $FRAME=$LOCALS+64+10*16+18*$SIZE_T;	# 10*16 is for v23-v31 offload
+
+sub VMXROUND {
+my $odd = pop;
+my ($a,$b,$c,$d)=@_;
+
+	(
+	"&vadduwm	('$a','$a','$b')",
+	"&vxor		('$d','$d','$a')",
+	"&vperm		('$d','$d','$d','$sixteen')",
+
+	"&vadduwm	('$c','$c','$d')",
+	"&vxor		('$b','$b','$c')",
+	"&vrlw		('$b','$b','$twelve')",
+
+	"&vadduwm	('$a','$a','$b')",
+	"&vxor		('$d','$d','$a')",
+	"&vperm		('$d','$d','$d','$twenty4')",
+
+	"&vadduwm	('$c','$c','$d')",
+	"&vxor		('$b','$b','$c')",
+	"&vrlw		('$b','$b','$seven')",
+
+	"&vrldoi	('$c','$c',8)",
+	"&vrldoi	('$b','$b',$odd?4:12)",
+	"&vrldoi	('$d','$d',$odd?12:4)"
+	);
+}
+
+$code.=<<___;
+
+.globl	.ChaCha20_ctr32_vmx
+.align	5
+.ChaCha20_ctr32_vmx:
+	${UCMP}i $len,256
+	blt	__ChaCha20_ctr32_int
+
+	$STU	$sp,-$FRAME($sp)
+	mflr	r0
+	li	r10,`15+$LOCALS+64`
+	li	r11,`31+$LOCALS+64`
+	mfspr	r12,256
+	stvx	v23,r10,$sp
+	addi	r10,r10,32
+	stvx	v24,r11,$sp
+	addi	r11,r11,32
+	stvx	v25,r10,$sp
+	addi	r10,r10,32
+	stvx	v26,r11,$sp
+	addi	r11,r11,32
+	stvx	v27,r10,$sp
+	addi	r10,r10,32
+	stvx	v28,r11,$sp
+	addi	r11,r11,32
+	stvx	v29,r10,$sp
+	addi	r10,r10,32
+	stvx	v30,r11,$sp
+	stvx	v31,r10,$sp
+	stw	r12,`$FRAME-$SIZE_T*18-4`($sp)	# save vrsave
+	$PUSH	r14,`$FRAME-$SIZE_T*18`($sp)
+	$PUSH	r15,`$FRAME-$SIZE_T*17`($sp)
+	$PUSH	r16,`$FRAME-$SIZE_T*16`($sp)
+	$PUSH	r17,`$FRAME-$SIZE_T*15`($sp)
+	$PUSH	r18,`$FRAME-$SIZE_T*14`($sp)
+	$PUSH	r19,`$FRAME-$SIZE_T*13`($sp)
+	$PUSH	r20,`$FRAME-$SIZE_T*12`($sp)
+	$PUSH	r21,`$FRAME-$SIZE_T*11`($sp)
+	$PUSH	r22,`$FRAME-$SIZE_T*10`($sp)
+	$PUSH	r23,`$FRAME-$SIZE_T*9`($sp)
+	$PUSH	r24,`$FRAME-$SIZE_T*8`($sp)
+	$PUSH	r25,`$FRAME-$SIZE_T*7`($sp)
+	$PUSH	r26,`$FRAME-$SIZE_T*6`($sp)
+	$PUSH	r27,`$FRAME-$SIZE_T*5`($sp)
+	$PUSH	r28,`$FRAME-$SIZE_T*4`($sp)
+	$PUSH	r29,`$FRAME-$SIZE_T*3`($sp)
+	$PUSH	r30,`$FRAME-$SIZE_T*2`($sp)
+	$PUSH	r31,`$FRAME-$SIZE_T*1`($sp)
+	li	r12,-4096+511
+	$PUSH	r0, `$FRAME+$LRSAVE`($sp)
+	mtspr	256,r12				# preserve 29 AltiVec registers
+
+	bl	Lconsts				# returns pointer Lsigma in r12
+	li	@x[0],16
+	li	@x[1],32
+	li	@x[2],48
+	li	@x[3],64
+	li	@x[4],31			# 31 is not a typo
+	li	@x[5],15			# nor is 15
+
+	lvx	@K[1],0,$key			# load key
+	?lvsr	$T0,0,$key			# prepare unaligned load
+	lvx	@K[2],@x[0],$key
+	lvx	@D[0],@x[4],$key
+
+	lvx	@K[3],0,$ctr			# load counter
+	?lvsr	$T1,0,$ctr			# prepare unaligned load
+	lvx	@D[1],@x[5],$ctr
+
+	lvx	@K[0],0,r12			# load constants
+	lvx	@K[5],@x[0],r12			# one
+	lvx	$FOUR,@x[1],r12
+	lvx	$sixteen,@x[2],r12
+	lvx	$twenty4,@x[3],r12
+
+	?vperm	@K[1],@K[2],@K[1],$T0		# align key
+	?vperm	@K[2],@D[0],@K[2],$T0
+	?vperm	@K[3],@D[1],@K[3],$T1		# align counter
+
+	lwz	@d[0],0($ctr)			# load counter to GPR
+	lwz	@d[1],4($ctr)
+	vadduwm	@K[3],@K[3],@K[5]		# adjust AltiVec counter
+	lwz	@d[2],8($ctr)
+	vadduwm	@K[4],@K[3],@K[5]
+	lwz	@d[3],12($ctr)
+	vadduwm	@K[5],@K[4],@K[5]
+
+	vxor	$T0,$T0,$T0			# 0x00..00
+	vspltisw $outmask,-1			# 0xff..ff
+	?lvsr	$inpperm,0,$inp			# prepare for unaligned load
+	?lvsl	$outperm,0,$out			# prepare for unaligned store
+	?vperm	$outmask,$outmask,$T0,$outperm
+
+	be?lvsl	$T0,0,@x[0]			# 0x00..0f
+	be?vspltisb $T1,3			# 0x03..03
+	be?vxor	$T0,$T0,$T1			# swap bytes within words
+	be?vxor	$outperm,$outperm,$T1
+	be?vperm $inpperm,$inpperm,$inpperm,$T0
+
+	li	r0,10				# inner loop counter
+	b	Loop_outer_vmx
+
+.align	4
+Loop_outer_vmx:
+	lis	@x[0],0x6170			# synthesize sigma
+	lis	@x[1],0x3320
+	 vmr	$A0,@K[0]
+	lis	@x[2],0x7962
+	lis	@x[3],0x6b20
+	 vmr	$A1,@K[0]
+	ori	@x[0],@x[0],0x7865
+	ori	@x[1],@x[1],0x646e
+	 vmr	$A2,@K[0]
+	ori	@x[2],@x[2],0x2d32
+	ori	@x[3],@x[3],0x6574
+	 vmr	$B0,@K[1]
+
+	lwz	@x[4],0($key)			# load key to GPR
+	 vmr	$B1,@K[1]
+	lwz	@x[5],4($key)
+	 vmr	$B2,@K[1]
+	lwz	@x[6],8($key)
+	 vmr	$C0,@K[2]
+	lwz	@x[7],12($key)
+	 vmr	$C1,@K[2]
+	lwz	@x[8],16($key)
+	 vmr	$C2,@K[2]
+	mr	@x[12],@d[0]			# copy GPR counter
+	lwz	@x[9],20($key)
+	 vmr	$D0,@K[3]
+	mr	@x[13],@d[1]
+	lwz	@x[10],24($key)
+	 vmr	$D1,@K[4]
+	mr	@x[14],@d[2]
+	lwz	@x[11],28($key)
+	 vmr	$D2,@K[5]
+	mr	@x[15],@d[3]
+
+	mr	@t[0],@x[4]
+	mr	@t[1],@x[5]
+	mr	@t[2],@x[6]
+	mr	@t[3],@x[7]
+
+	vspltisw $twelve,12			# synthesize constants
+	vspltisw $seven,7
+
+	mtctr	r0
+	nop
+Loop_vmx:
+___
+	my @thread0=&VMXROUND($A0,$B0,$C0,$D0,0);
+	my @thread1=&VMXROUND($A1,$B1,$C1,$D1,0);
+	my @thread2=&VMXROUND($A2,$B2,$C2,$D2,0);
+	my @thread3=&ROUND(0,4,8,12);
+
+	foreach (@thread0) {
+		eval;
+		eval(shift(@thread1));
+		eval(shift(@thread2));
+
+		eval(shift(@thread3));
+		eval(shift(@thread3));
+		eval(shift(@thread3));
+	}
+	foreach (@thread3) { eval; }
+
+	@thread0=&VMXROUND($A0,$B0,$C0,$D0,1);
+	@thread1=&VMXROUND($A1,$B1,$C1,$D1,1);
+	@thread2=&VMXROUND($A2,$B2,$C2,$D2,1);
+	@thread3=&ROUND(0,5,10,15);
+
+	foreach (@thread0) {
+		eval;
+		eval(shift(@thread1));
+		eval(shift(@thread2));
+
+		eval(shift(@thread3));
+		eval(shift(@thread3));
+		eval(shift(@thread3));
+	}
+	foreach (@thread3) { eval; }
+$code.=<<___;
+	bdnz	Loop_vmx
+
+	subi	$len,$len,256			# $len-=256
+	addi	@x[0],@x[0],0x7865		# accumulate key block
+	addi	@x[1],@x[1],0x646e
+	addi	@x[2],@x[2],0x2d32
+	addi	@x[3],@x[3],0x6574
+	addis	@x[0],@x[0],0x6170
+	addis	@x[1],@x[1],0x3320
+	addis	@x[2],@x[2],0x7962
+	addis	@x[3],@x[3],0x6b20
+	add	@x[4],@x[4],@t[0]
+	lwz	@t[0],16($key)
+	add	@x[5],@x[5],@t[1]
+	lwz	@t[1],20($key)
+	add	@x[6],@x[6],@t[2]
+	lwz	@t[2],24($key)
+	add	@x[7],@x[7],@t[3]
+	lwz	@t[3],28($key)
+	add	@x[8],@x[8],@t[0]
+	add	@x[9],@x[9],@t[1]
+	add	@x[10],@x[10],@t[2]
+	add	@x[11],@x[11],@t[3]
+	add	@x[12],@x[12],@d[0]
+	add	@x[13],@x[13],@d[1]
+	add	@x[14],@x[14],@d[2]
+	add	@x[15],@x[15],@d[3]
+
+	vadduwm	$A0,$A0,@K[0]			# accumulate key block
+	vadduwm	$A1,$A1,@K[0]
+	vadduwm	$A2,$A2,@K[0]
+	vadduwm	$B0,$B0,@K[1]
+	vadduwm	$B1,$B1,@K[1]
+	vadduwm	$B2,$B2,@K[1]
+	vadduwm	$C0,$C0,@K[2]
+	vadduwm	$C1,$C1,@K[2]
+	vadduwm	$C2,$C2,@K[2]
+	vadduwm	$D0,$D0,@K[3]
+	vadduwm	$D1,$D1,@K[4]
+	vadduwm	$D2,$D2,@K[5]
+
+	addi	@d[0],@d[0],4			# increment counter
+	vadduwm	@K[3],@K[3],$FOUR
+	vadduwm	@K[4],@K[4],$FOUR
+	vadduwm	@K[5],@K[5],$FOUR
+
+___
+if (!$LITTLE_ENDIAN) { for($i=0;$i<16;$i++) {	# flip byte order
+$code.=<<___;
+	mr	@t[$i&3],@x[$i]
+	rotlwi	@x[$i],@x[$i],8
+	rlwimi	@x[$i],@t[$i&3],24,0,7
+	rlwimi	@x[$i],@t[$i&3],24,16,23
+___
+} }
+$code.=<<___;
+	lwz	@t[0],0($inp)			# load input, aligned or not
+	lwz	@t[1],4($inp)
+	lwz	@t[2],8($inp)
+	lwz	@t[3],12($inp)
+	xor	@x[0],@x[0],@t[0]		# xor with input
+	lwz	@t[0],16($inp)
+	xor	@x[1],@x[1],@t[1]
+	lwz	@t[1],20($inp)
+	xor	@x[2],@x[2],@t[2]
+	lwz	@t[2],24($inp)
+	xor	@x[3],@x[3],@t[3]
+	lwz	@t[3],28($inp)
+	xor	@x[4],@x[4],@t[0]
+	lwz	@t[0],32($inp)
+	xor	@x[5],@x[5],@t[1]
+	lwz	@t[1],36($inp)
+	xor	@x[6],@x[6],@t[2]
+	lwz	@t[2],40($inp)
+	xor	@x[7],@x[7],@t[3]
+	lwz	@t[3],44($inp)
+	xor	@x[8],@x[8],@t[0]
+	lwz	@t[0],48($inp)
+	xor	@x[9],@x[9],@t[1]
+	lwz	@t[1],52($inp)
+	xor	@x[10],@x[10],@t[2]
+	lwz	@t[2],56($inp)
+	xor	@x[11],@x[11],@t[3]
+	lwz	@t[3],60($inp)
+	xor	@x[12],@x[12],@t[0]
+	stw	@x[0],0($out)			# store output, aligned or not
+	xor	@x[13],@x[13],@t[1]
+	stw	@x[1],4($out)
+	xor	@x[14],@x[14],@t[2]
+	stw	@x[2],8($out)
+	xor	@x[15],@x[15],@t[3]
+	stw	@x[3],12($out)
+	addi	$inp,$inp,64
+	stw	@x[4],16($out)
+	li	@t[0],16
+	stw	@x[5],20($out)
+	li	@t[1],32
+	stw	@x[6],24($out)
+	li	@t[2],48
+	stw	@x[7],28($out)
+	li	@t[3],64
+	stw	@x[8],32($out)
+	stw	@x[9],36($out)
+	stw	@x[10],40($out)
+	stw	@x[11],44($out)
+	stw	@x[12],48($out)
+	stw	@x[13],52($out)
+	stw	@x[14],56($out)
+	stw	@x[15],60($out)
+	addi	$out,$out,64
+
+	lvx	@D[0],0,$inp			# load input
+	lvx	@D[1],@t[0],$inp
+	lvx	@D[2],@t[1],$inp
+	lvx	@D[3],@t[2],$inp
+	lvx	@D[4],@t[3],$inp
+	addi	$inp,$inp,64
+
+	?vperm	@D[0],@D[1],@D[0],$inpperm	# align input
+	?vperm	@D[1],@D[2],@D[1],$inpperm
+	?vperm	@D[2],@D[3],@D[2],$inpperm
+	?vperm	@D[3],@D[4],@D[3],$inpperm
+	vxor	$A0,$A0,@D[0]			# xor with input
+	vxor	$B0,$B0,@D[1]
+	lvx	@D[1],@t[0],$inp		# keep loading input
+	vxor	$C0,$C0,@D[2]
+	lvx	@D[2],@t[1],$inp
+	vxor	$D0,$D0,@D[3]
+	lvx	@D[3],@t[2],$inp
+	lvx	@D[0],@t[3],$inp
+	addi	$inp,$inp,64
+	li	@t[3],63			# 63 is not a typo
+	vperm	$A0,$A0,$A0,$outperm		# pre-misalign output
+	vperm	$B0,$B0,$B0,$outperm
+	vperm	$C0,$C0,$C0,$outperm
+	vperm	$D0,$D0,$D0,$outperm
+
+	?vperm	@D[4],@D[1],@D[4],$inpperm	# align input
+	?vperm	@D[1],@D[2],@D[1],$inpperm
+	?vperm	@D[2],@D[3],@D[2],$inpperm
+	?vperm	@D[3],@D[0],@D[3],$inpperm
+	vxor	$A1,$A1,@D[4]
+	vxor	$B1,$B1,@D[1]
+	lvx	@D[1],@t[0],$inp		# keep loading input
+	vxor	$C1,$C1,@D[2]
+	lvx	@D[2],@t[1],$inp
+	vxor	$D1,$D1,@D[3]
+	lvx	@D[3],@t[2],$inp
+	lvx	@D[4],@t[3],$inp		# redundant in aligned case
+	addi	$inp,$inp,64
+	vperm	$A1,$A1,$A1,$outperm		# pre-misalign output
+	vperm	$B1,$B1,$B1,$outperm
+	vperm	$C1,$C1,$C1,$outperm
+	vperm	$D1,$D1,$D1,$outperm
+
+	?vperm	@D[0],@D[1],@D[0],$inpperm	# align input
+	?vperm	@D[1],@D[2],@D[1],$inpperm
+	?vperm	@D[2],@D[3],@D[2],$inpperm
+	?vperm	@D[3],@D[4],@D[3],$inpperm
+	vxor	$A2,$A2,@D[0]
+	vxor	$B2,$B2,@D[1]
+	vxor	$C2,$C2,@D[2]
+	vxor	$D2,$D2,@D[3]
+	vperm	$A2,$A2,$A2,$outperm		# pre-misalign output
+	vperm	$B2,$B2,$B2,$outperm
+	vperm	$C2,$C2,$C2,$outperm
+	vperm	$D2,$D2,$D2,$outperm
+
+	andi.	@x[1],$out,15			# is $out aligned?
+	mr	@x[0],$out
+
+	vsel	@D[0],$A0,$B0,$outmask		# collect pre-misaligned output
+	vsel	@D[1],$B0,$C0,$outmask
+	vsel	@D[2],$C0,$D0,$outmask
+	vsel	@D[3],$D0,$A1,$outmask
+	vsel	$B0,$A1,$B1,$outmask
+	vsel	$C0,$B1,$C1,$outmask
+	vsel	$D0,$C1,$D1,$outmask
+	vsel	$A1,$D1,$A2,$outmask
+	vsel	$B1,$A2,$B2,$outmask
+	vsel	$C1,$B2,$C2,$outmask
+	vsel	$D1,$C2,$D2,$outmask
+
+	#stvx	$A0,0,$out			# take it easy on the edges
+	stvx	@D[0],@t[0],$out		# store output
+	stvx	@D[1],@t[1],$out
+	stvx	@D[2],@t[2],$out
+	addi	$out,$out,64
+	stvx	@D[3],0,$out
+	stvx	$B0,@t[0],$out
+	stvx	$C0,@t[1],$out
+	stvx	$D0,@t[2],$out
+	addi	$out,$out,64
+	stvx	$A1,0,$out
+	stvx	$B1,@t[0],$out
+	stvx	$C1,@t[1],$out
+	stvx	$D1,@t[2],$out
+	addi	$out,$out,64
+
+	beq	Laligned_vmx
+
+	sub	@x[2],$out,@x[1]		# in misaligned case edges
+	li	@x[3],0				# are written byte-by-byte
+Lunaligned_tail_vmx:
+	stvebx	$D2,@x[3],@x[2]
+	addi	@x[3],@x[3],1
+	cmpw	@x[3],@x[1]
+	bne	Lunaligned_tail_vmx
+
+	sub	@x[2],@x[0],@x[1]
+Lunaligned_head_vmx:
+	stvebx	$A0,@x[1],@x[2]
+	cmpwi	@x[1],15
+	addi	@x[1],@x[1],1
+	bne	Lunaligned_head_vmx
+
+	${UCMP}i $len,255			# done with 256-byte blocks yet?
+	bgt	Loop_outer_vmx
+
+	b	Ldone_vmx
+
+.align	4
+Laligned_vmx:
+	stvx	$A0,0,@x[0]			# head hexaword was not stored
+
+	${UCMP}i $len,255			# done with 256-byte blocks yet?
+	bgt	Loop_outer_vmx
+	nop
+
+Ldone_vmx:
+	${UCMP}i $len,0				# done yet?
+	bnel	__ChaCha20_1x
+
+	lwz	r12,`$FRAME-$SIZE_T*18-4`($sp)	# pull vrsave
+	li	r10,`15+$LOCALS+64`
+	li	r11,`31+$LOCALS+64`
+	mtspr	256,r12				# restore vrsave
+	lvx	v23,r10,$sp
+	addi	r10,r10,32
+	lvx	v24,r11,$sp
+	addi	r11,r11,32
+	lvx	v25,r10,$sp
+	addi	r10,r10,32
+	lvx	v26,r11,$sp
+	addi	r11,r11,32
+	lvx	v27,r10,$sp
+	addi	r10,r10,32
+	lvx	v28,r11,$sp
+	addi	r11,r11,32
+	lvx	v29,r10,$sp
+	addi	r10,r10,32
+	lvx	v30,r11,$sp
+	lvx	v31,r10,$sp
+	$POP	r0, `$FRAME+$LRSAVE`($sp)
+	$POP	r14,`$FRAME-$SIZE_T*18`($sp)
+	$POP	r15,`$FRAME-$SIZE_T*17`($sp)
+	$POP	r16,`$FRAME-$SIZE_T*16`($sp)
+	$POP	r17,`$FRAME-$SIZE_T*15`($sp)
+	$POP	r18,`$FRAME-$SIZE_T*14`($sp)
+	$POP	r19,`$FRAME-$SIZE_T*13`($sp)
+	$POP	r20,`$FRAME-$SIZE_T*12`($sp)
+	$POP	r21,`$FRAME-$SIZE_T*11`($sp)
+	$POP	r22,`$FRAME-$SIZE_T*10`($sp)
+	$POP	r23,`$FRAME-$SIZE_T*9`($sp)
+	$POP	r24,`$FRAME-$SIZE_T*8`($sp)
+	$POP	r25,`$FRAME-$SIZE_T*7`($sp)
+	$POP	r26,`$FRAME-$SIZE_T*6`($sp)
+	$POP	r27,`$FRAME-$SIZE_T*5`($sp)
+	$POP	r28,`$FRAME-$SIZE_T*4`($sp)
+	$POP	r29,`$FRAME-$SIZE_T*3`($sp)
+	$POP	r30,`$FRAME-$SIZE_T*2`($sp)
+	$POP	r31,`$FRAME-$SIZE_T*1`($sp)
+	mtlr	r0
+	addi	$sp,$sp,$FRAME
+	blr
+	.long	0
+	.byte	0,12,0x04,1,0x80,18,5,0
+	.long	0
+.size	.ChaCha20_ctr32_vmx,.-.ChaCha20_ctr32_vmx
+___
+}}}
+{{{
+my ($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3,
+    $xc0,$xc1,$xc2,$xc3, $xd0,$xd1,$xd2,$xd3) = map("v$_",(0..15));
+my @K = map("v$_",(16..19));
+my $CTR = "v26";
+my ($xt0,$xt1,$xt2,$xt3) = map("v$_",(27..30));
+my ($sixteen,$twelve,$eight,$seven) = ($xt0,$xt1,$xt2,$xt3);
+my $beperm = "v31";
+
+my ($x00,$x10,$x20,$x30) = (0, map("r$_",(8..10)));
+
+my $FRAME=$LOCALS+64+7*16;	# 7*16 is for v26-v31 offload
+
+sub VSX_lane_ROUND {
+my ($a0,$b0,$c0,$d0)=@_;
+my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
+my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
+my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
+my @x=map("\"v$_\"",(0..15));
+
+	(
+	"&vadduwm	(@x[$a0],@x[$a0],@x[$b0])",	# Q1
+	 "&vadduwm	(@x[$a1],@x[$a1],@x[$b1])",	# Q2
+	  "&vadduwm	(@x[$a2],@x[$a2],@x[$b2])",	# Q3
+	   "&vadduwm	(@x[$a3],@x[$a3],@x[$b3])",	# Q4
+	"&vxor		(@x[$d0],@x[$d0],@x[$a0])",
+	 "&vxor		(@x[$d1],@x[$d1],@x[$a1])",
+	  "&vxor	(@x[$d2],@x[$d2],@x[$a2])",
+	   "&vxor	(@x[$d3],@x[$d3],@x[$a3])",
+	"&vrlw		(@x[$d0],@x[$d0],'$sixteen')",
+	 "&vrlw		(@x[$d1],@x[$d1],'$sixteen')",
+	  "&vrlw	(@x[$d2],@x[$d2],'$sixteen')",
+	   "&vrlw	(@x[$d3],@x[$d3],'$sixteen')",
+
+	"&vadduwm	(@x[$c0],@x[$c0],@x[$d0])",
+	 "&vadduwm	(@x[$c1],@x[$c1],@x[$d1])",
+	  "&vadduwm	(@x[$c2],@x[$c2],@x[$d2])",
+	   "&vadduwm	(@x[$c3],@x[$c3],@x[$d3])",
+	"&vxor		(@x[$b0],@x[$b0],@x[$c0])",
+	 "&vxor		(@x[$b1],@x[$b1],@x[$c1])",
+	  "&vxor	(@x[$b2],@x[$b2],@x[$c2])",
+	   "&vxor	(@x[$b3],@x[$b3],@x[$c3])",
+	"&vrlw		(@x[$b0],@x[$b0],'$twelve')",
+	 "&vrlw		(@x[$b1],@x[$b1],'$twelve')",
+	  "&vrlw	(@x[$b2],@x[$b2],'$twelve')",
+	   "&vrlw	(@x[$b3],@x[$b3],'$twelve')",
+
+	"&vadduwm	(@x[$a0],@x[$a0],@x[$b0])",
+	 "&vadduwm	(@x[$a1],@x[$a1],@x[$b1])",
+	  "&vadduwm	(@x[$a2],@x[$a2],@x[$b2])",
+	   "&vadduwm	(@x[$a3],@x[$a3],@x[$b3])",
+	"&vxor		(@x[$d0],@x[$d0],@x[$a0])",
+	 "&vxor		(@x[$d1],@x[$d1],@x[$a1])",
+	  "&vxor	(@x[$d2],@x[$d2],@x[$a2])",
+	   "&vxor	(@x[$d3],@x[$d3],@x[$a3])",
+	"&vrlw		(@x[$d0],@x[$d0],'$eight')",
+	 "&vrlw		(@x[$d1],@x[$d1],'$eight')",
+	  "&vrlw	(@x[$d2],@x[$d2],'$eight')",
+	   "&vrlw	(@x[$d3],@x[$d3],'$eight')",
+
+	"&vadduwm	(@x[$c0],@x[$c0],@x[$d0])",
+	 "&vadduwm	(@x[$c1],@x[$c1],@x[$d1])",
+	  "&vadduwm	(@x[$c2],@x[$c2],@x[$d2])",
+	   "&vadduwm	(@x[$c3],@x[$c3],@x[$d3])",
+	"&vxor		(@x[$b0],@x[$b0],@x[$c0])",
+	 "&vxor		(@x[$b1],@x[$b1],@x[$c1])",
+	  "&vxor	(@x[$b2],@x[$b2],@x[$c2])",
+	   "&vxor	(@x[$b3],@x[$b3],@x[$c3])",
+	"&vrlw		(@x[$b0],@x[$b0],'$seven')",
+	 "&vrlw		(@x[$b1],@x[$b1],'$seven')",
+	  "&vrlw	(@x[$b2],@x[$b2],'$seven')",
+	   "&vrlw	(@x[$b3],@x[$b3],'$seven')"
+	);
+}
+
+$code.=<<___;
+
+.globl	.ChaCha20_ctr32_vsx
+.align	5
+.ChaCha20_ctr32_vsx:
+	$STU	$sp,-$FRAME($sp)
+	mflr	r0
+	li	r10,`15+$LOCALS+64`
+	li	r11,`31+$LOCALS+64`
+	mfspr	r12,256
+	stvx	v26,r10,$sp
+	addi	r10,r10,32
+	stvx	v27,r11,$sp
+	addi	r11,r11,32
+	stvx	v28,r10,$sp
+	addi	r10,r10,32
+	stvx	v29,r11,$sp
+	addi	r11,r11,32
+	stvx	v30,r10,$sp
+	stvx	v31,r11,$sp
+	stw	r12,`$FRAME-4`($sp)		# save vrsave
+	li	r12,-4096+63
+	$PUSH	r0, `$FRAME+$LRSAVE`($sp)
+	mtspr	256,r12				# preserve 29 AltiVec registers
+
+	bl	Lconsts				# returns pointer Lsigma in r12
+	lvx_4w	@K[0],0,r12			# load sigma
+	addi	r12,r12,0x50
+	li	$x10,16
+	li	$x20,32
+	li	$x30,48
+	li	r11,64
+
+	lvx_4w	@K[1],0,$key			# load key
+	lvx_4w	@K[2],$x10,$key
+	lvx_4w	@K[3],0,$ctr			# load counter
+
+	vxor	$xt0,$xt0,$xt0
+	lvx_4w	$xt1,r11,r12
+	vspltw	$CTR,@K[3],0
+	vsldoi	@K[3],@K[3],$xt0,4
+	vsldoi	@K[3],$xt0,@K[3],12		# clear @K[3].word[0]
+	vadduwm	$CTR,$CTR,$xt1
+
+	be?lvsl	$beperm,0,$x10			# 0x00..0f
+	be?vspltisb $xt0,3			# 0x03..03
+	be?vxor	$beperm,$beperm,$xt0		# swap bytes within words
+
+	li	r0,10				# inner loop counter
+	mtctr	r0
+	b	Loop_outer_vsx
+
+.align	5
+Loop_outer_vsx:
+	lvx	$xa0,$x00,r12			# load [smashed] sigma
+	lvx	$xa1,$x10,r12
+	lvx	$xa2,$x20,r12
+	lvx	$xa3,$x30,r12
+
+	vspltw	$xb0,@K[1],0			# smash the key
+	vspltw	$xb1,@K[1],1
+	vspltw	$xb2,@K[1],2
+	vspltw	$xb3,@K[1],3
+
+	vspltw	$xc0,@K[2],0
+	vspltw	$xc1,@K[2],1
+	vspltw	$xc2,@K[2],2
+	vspltw	$xc3,@K[2],3
+
+	vmr	$xd0,$CTR			# smash the counter
+	vspltw	$xd1,@K[3],1
+	vspltw	$xd2,@K[3],2
+	vspltw	$xd3,@K[3],3
+
+	vspltisw $sixteen,-16			# synthesize constants
+	vspltisw $twelve,12
+	vspltisw $eight,8
+	vspltisw $seven,7
+
+Loop_vsx:
+___
+	foreach (&VSX_lane_ROUND(0, 4, 8,12)) { eval; }
+	foreach (&VSX_lane_ROUND(0, 5,10,15)) { eval; }
+$code.=<<___;
+	bdnz	Loop_vsx
+
+	vadduwm	$xd0,$xd0,$CTR
+
+	vmrgew	$xt0,$xa0,$xa1			# transpose data
+	vmrgew	$xt1,$xa2,$xa3
+	vmrgow	$xa0,$xa0,$xa1
+	vmrgow	$xa2,$xa2,$xa3
+	 vmrgew	$xt2,$xb0,$xb1
+	 vmrgew	$xt3,$xb2,$xb3
+	vpermdi	$xa1,$xa0,$xa2,0b00
+	vpermdi	$xa3,$xa0,$xa2,0b11
+	vpermdi	$xa0,$xt0,$xt1,0b00
+	vpermdi	$xa2,$xt0,$xt1,0b11
+
+	vmrgow	$xb0,$xb0,$xb1
+	vmrgow	$xb2,$xb2,$xb3
+	 vmrgew	$xt0,$xc0,$xc1
+	 vmrgew	$xt1,$xc2,$xc3
+	vpermdi	$xb1,$xb0,$xb2,0b00
+	vpermdi	$xb3,$xb0,$xb2,0b11
+	vpermdi	$xb0,$xt2,$xt3,0b00
+	vpermdi	$xb2,$xt2,$xt3,0b11
+
+	vmrgow	$xc0,$xc0,$xc1
+	vmrgow	$xc2,$xc2,$xc3
+	 vmrgew	$xt2,$xd0,$xd1
+	 vmrgew	$xt3,$xd2,$xd3
+	vpermdi	$xc1,$xc0,$xc2,0b00
+	vpermdi	$xc3,$xc0,$xc2,0b11
+	vpermdi	$xc0,$xt0,$xt1,0b00
+	vpermdi	$xc2,$xt0,$xt1,0b11
+
+	vmrgow	$xd0,$xd0,$xd1
+	vmrgow	$xd2,$xd2,$xd3
+	 vspltisw $xt0,4
+	 vadduwm  $CTR,$CTR,$xt0		# next counter value
+	vpermdi	$xd1,$xd0,$xd2,0b00
+	vpermdi	$xd3,$xd0,$xd2,0b11
+	vpermdi	$xd0,$xt2,$xt3,0b00
+	vpermdi	$xd2,$xt2,$xt3,0b11
+
+	vadduwm	$xa0,$xa0,@K[0]
+	vadduwm	$xb0,$xb0,@K[1]
+	vadduwm	$xc0,$xc0,@K[2]
+	vadduwm	$xd0,$xd0,@K[3]
+
+	be?vperm $xa0,$xa0,$xa0,$beperm
+	be?vperm $xb0,$xb0,$xb0,$beperm
+	be?vperm $xc0,$xc0,$xc0,$beperm
+	be?vperm $xd0,$xd0,$xd0,$beperm
+
+	${UCMP}i $len,0x40
+	blt	Ltail_vsx
+
+	lvx_4w	$xt0,$x00,$inp
+	lvx_4w	$xt1,$x10,$inp
+	lvx_4w	$xt2,$x20,$inp
+	lvx_4w	$xt3,$x30,$inp
+
+	vxor	$xt0,$xt0,$xa0
+	vxor	$xt1,$xt1,$xb0
+	vxor	$xt2,$xt2,$xc0
+	vxor	$xt3,$xt3,$xd0
+
+	stvx_4w	$xt0,$x00,$out
+	stvx_4w	$xt1,$x10,$out
+	addi	$inp,$inp,0x40
+	stvx_4w	$xt2,$x20,$out
+	subi	$len,$len,0x40
+	stvx_4w	$xt3,$x30,$out
+	addi	$out,$out,0x40
+	beq	Ldone_vsx
+
+	vadduwm	$xa0,$xa1,@K[0]
+	vadduwm	$xb0,$xb1,@K[1]
+	vadduwm	$xc0,$xc1,@K[2]
+	vadduwm	$xd0,$xd1,@K[3]
+
+	be?vperm $xa0,$xa0,$xa0,$beperm
+	be?vperm $xb0,$xb0,$xb0,$beperm
+	be?vperm $xc0,$xc0,$xc0,$beperm
+	be?vperm $xd0,$xd0,$xd0,$beperm
+
+	${UCMP}i $len,0x40
+	blt	Ltail_vsx
+
+	lvx_4w	$xt0,$x00,$inp
+	lvx_4w	$xt1,$x10,$inp
+	lvx_4w	$xt2,$x20,$inp
+	lvx_4w	$xt3,$x30,$inp
+
+	vxor	$xt0,$xt0,$xa0
+	vxor	$xt1,$xt1,$xb0
+	vxor	$xt2,$xt2,$xc0
+	vxor	$xt3,$xt3,$xd0
+
+	stvx_4w	$xt0,$x00,$out
+	stvx_4w	$xt1,$x10,$out
+	addi	$inp,$inp,0x40
+	stvx_4w	$xt2,$x20,$out
+	subi	$len,$len,0x40
+	stvx_4w	$xt3,$x30,$out
+	addi	$out,$out,0x40
+	beq	Ldone_vsx
+
+	vadduwm	$xa0,$xa2,@K[0]
+	vadduwm	$xb0,$xb2,@K[1]
+	vadduwm	$xc0,$xc2,@K[2]
+	vadduwm	$xd0,$xd2,@K[3]
+
+	be?vperm $xa0,$xa0,$xa0,$beperm
+	be?vperm $xb0,$xb0,$xb0,$beperm
+	be?vperm $xc0,$xc0,$xc0,$beperm
+	be?vperm $xd0,$xd0,$xd0,$beperm
+
+	${UCMP}i $len,0x40
+	blt	Ltail_vsx
+
+	lvx_4w	$xt0,$x00,$inp
+	lvx_4w	$xt1,$x10,$inp
+	lvx_4w	$xt2,$x20,$inp
+	lvx_4w	$xt3,$x30,$inp
+
+	vxor	$xt0,$xt0,$xa0
+	vxor	$xt1,$xt1,$xb0
+	vxor	$xt2,$xt2,$xc0
+	vxor	$xt3,$xt3,$xd0
+
+	stvx_4w	$xt0,$x00,$out
+	stvx_4w	$xt1,$x10,$out
+	addi	$inp,$inp,0x40
+	stvx_4w	$xt2,$x20,$out
+	subi	$len,$len,0x40
+	stvx_4w	$xt3,$x30,$out
+	addi	$out,$out,0x40
+	beq	Ldone_vsx
+
+	vadduwm	$xa0,$xa3,@K[0]
+	vadduwm	$xb0,$xb3,@K[1]
+	vadduwm	$xc0,$xc3,@K[2]
+	vadduwm	$xd0,$xd3,@K[3]
+
+	be?vperm $xa0,$xa0,$xa0,$beperm
+	be?vperm $xb0,$xb0,$xb0,$beperm
+	be?vperm $xc0,$xc0,$xc0,$beperm
+	be?vperm $xd0,$xd0,$xd0,$beperm
+
+	${UCMP}i $len,0x40
+	blt	Ltail_vsx
+
+	lvx_4w	$xt0,$x00,$inp
+	lvx_4w	$xt1,$x10,$inp
+	lvx_4w	$xt2,$x20,$inp
+	lvx_4w	$xt3,$x30,$inp
+
+	vxor	$xt0,$xt0,$xa0
+	vxor	$xt1,$xt1,$xb0
+	vxor	$xt2,$xt2,$xc0
+	vxor	$xt3,$xt3,$xd0
+
+	stvx_4w	$xt0,$x00,$out
+	stvx_4w	$xt1,$x10,$out
+	addi	$inp,$inp,0x40
+	stvx_4w	$xt2,$x20,$out
+	subi	$len,$len,0x40
+	stvx_4w	$xt3,$x30,$out
+	addi	$out,$out,0x40
+	mtctr	r0
+	bne	Loop_outer_vsx
+
+Ldone_vsx:
+	lwz	r12,`$FRAME-4`($sp)		# pull vrsave
+	li	r10,`15+$LOCALS+64`
+	li	r11,`31+$LOCALS+64`
+	$POP	r0, `$FRAME+$LRSAVE`($sp)
+	mtspr	256,r12				# restore vrsave
+	lvx	v26,r10,$sp
+	addi	r10,r10,32
+	lvx	v27,r11,$sp
+	addi	r11,r11,32
+	lvx	v28,r10,$sp
+	addi	r10,r10,32
+	lvx	v29,r11,$sp
+	addi	r11,r11,32
+	lvx	v30,r10,$sp
+	lvx	v31,r11,$sp
+	mtlr	r0
+	addi	$sp,$sp,$FRAME
+	blr
+
+.align	4
+Ltail_vsx:
+	addi	r11,$sp,$LOCALS
+	mtctr	$len
+	stvx_4w	$xa0,$x00,r11			# offload block to stack
+	stvx_4w	$xb0,$x10,r11
+	stvx_4w	$xc0,$x20,r11
+	stvx_4w	$xd0,$x30,r11
+	subi	r12,r11,1			# prepare for *++ptr
+	subi	$inp,$inp,1
+	subi	$out,$out,1
+
+Loop_tail_vsx:
+	lbzu	r6,1(r12)
+	lbzu	r7,1($inp)
+	xor	r6,r6,r7
+	stbu	r6,1($out)
+	bdnz	Loop_tail_vsx
+
+	stvx_4w	$K[0],$x00,r11			# wipe copy of the block
+	stvx_4w	$K[0],$x10,r11
+	stvx_4w	$K[0],$x20,r11
+	stvx_4w	$K[0],$x30,r11
+
+	b	Ldone_vsx
+	.long	0
+	.byte	0,12,0x04,1,0x80,0,5,0
+	.long	0
+.size	.ChaCha20_ctr32_vsx,.-.ChaCha20_ctr32_vsx
+___
+}}}
+$code.=<<___;
+.align	5
+Lconsts:
+	mflr	r0
+	bcl	20,31,\$+4
+	mflr	r12	#vvvvv "distance between . and Lsigma
+	addi	r12,r12,`64-8`
+	mtlr	r0
+	blr
+	.long	0
+	.byte	0,12,0x14,0,0,0,0,0
+	.space	`64-9*4`
+Lsigma:
+	.long   0x61707865,0x3320646e,0x79622d32,0x6b206574
+	.long	1,0,0,0
+	.long	4,0,0,0
+___
+$code.=<<___ 	if ($LITTLE_ENDIAN);
+	.long	0x0e0f0c0d,0x0a0b0809,0x06070405,0x02030001
+	.long	0x0d0e0f0c,0x090a0b08,0x05060704,0x01020300
+___
+$code.=<<___ 	if (!$LITTLE_ENDIAN);	# flipped words
+	.long	0x02030001,0x06070405,0x0a0b0809,0x0e0f0c0d
+	.long	0x01020300,0x05060704,0x090a0b08,0x0d0e0f0c
+___
+$code.=<<___;
+	.long	0x61707865,0x61707865,0x61707865,0x61707865
+	.long	0x3320646e,0x3320646e,0x3320646e,0x3320646e
+	.long	0x79622d32,0x79622d32,0x79622d32,0x79622d32
+	.long	0x6b206574,0x6b206574,0x6b206574,0x6b206574
+	.long	0,1,2,3
+.asciz  "ChaCha20 for PowerPC/AltiVec, CRYPTOGAMS by <appro\@openssl.org>"
+.align	2
+___
+
+foreach (split("\n",$code)) {
+	s/\`([^\`]*)\`/eval $1/ge;
+
+	# instructions prefixed with '?' are endian-specific and need
+	# to be adjusted accordingly...
+	if ($flavour !~ /le$/) {	# big-endian
+	    s/be\?//		or
+	    s/le\?/#le#/	or
+	    s/\?lvsr/lvsl/	or
+	    s/\?lvsl/lvsr/	or
+	    s/\?(vperm\s+v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+)/$1$3$2$4/ or
+	    s/vrldoi(\s+v[0-9]+,\s*)(v[0-9]+,)\s*([0-9]+)/vsldoi$1$2$2 16-$3/;
+	} else {			# little-endian
+	    s/le\?//		or
+	    s/be\?/#be#/	or
+	    s/\?([a-z]+)/$1/	or
+	    s/vrldoi(\s+v[0-9]+,\s*)(v[0-9]+,)\s*([0-9]+)/vsldoi$1$2$2 $3/;
+	}
+
+	print $_,"\n";
+}
+
+close STDOUT;
diff --git a/src/crypto/zinc/chacha20/chacha20.c b/src/crypto/zinc/chacha20/chacha20.c
index b4763c8..42e5360 100644
--- a/src/crypto/zinc/chacha20/chacha20.c
+++ b/src/crypto/zinc/chacha20/chacha20.c
@@ -20,10 +20,12 @@
 #include "chacha20-x86_64-glue.c"
 #elif defined(CONFIG_ZINC_ARCH_ARM) || defined(CONFIG_ZINC_ARCH_ARM64)
 #include "chacha20-arm-glue.c"
 #elif defined(CONFIG_ZINC_ARCH_MIPS)
 #include "chacha20-mips-glue.c"
+#elif defined(CONFIG_ZINC_ARCH_PPC32) || defined(CONFIG_ZINC_ARCH_PPC64)
+#include "chacha20-ppc-glue.c"
 #else
 static bool *const chacha20_nobs[] __initconst = { };
 static void __init chacha20_fpu_init(void)
 {
 }
diff --git a/src/crypto/zinc/chacha20/ppc-xlate.pl b/src/crypto/zinc/chacha20/ppc-xlate.pl
new file mode 100644
index 0000000..2362071
--- /dev/null
+++ b/src/crypto/zinc/chacha20/ppc-xlate.pl
@@ -0,0 +1,353 @@
+#! /usr/bin/env perl
+# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+#
+# This code is taken from the OpenSSL project but the author, Andy Polyakov,
+# has relicensed it under the licenses specified in the SPDX header above.
+# The original headers, including the original license headers, are
+# included below for completeness.
+#
+# Copyright 2006-2018 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the Apache License 2.0 (the "License").  You may not use
+# this file except in compliance with the License.  You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
+my $flavour = shift;
+my $output = shift;
+open STDOUT,">$output" || die "can't open $output: $!";
+
+my %GLOBALS;
+my %TYPES;
+my $dotinlocallabels=($flavour=~/linux/)?1:0;
+
+################################################################
+# directives which need special treatment on different platforms
+################################################################
+my $type = sub {
+    my ($dir,$name,$type) = @_;
+
+    $TYPES{$name} = $type;
+    if ($flavour =~ /linux/) {
+	$name =~ s|^\.||;
+	".type	$name,$type";
+    } else {
+	"";
+    }
+};
+my $globl = sub {
+    my $junk = shift;
+    my $name = shift;
+    my $global = \$GLOBALS{$name};
+    my $type = \$TYPES{$name};
+    my $ret;
+
+    $name =~ s|^\.||;
+
+    SWITCH: for ($flavour) {
+	/aix/		&& do { if (!$$type) {
+				    $$type = "\@function";
+				}
+				if ($$type =~ /function/) {
+				    $name = ".$name";
+				}
+				last;
+			      };
+	/osx/		&& do { $name = "_$name";
+				last;
+			      };
+	/linux.*(32|64le)/
+			&& do {	$ret .= ".globl	$name";
+				if (!$$type) {
+				    $ret .= "\n.type	$name,\@function";
+				    $$type = "\@function";
+				}
+				last;
+			      };
+	/linux.*64/	&& do {	$ret .= ".globl	$name";
+				if (!$$type) {
+				    $ret .= "\n.type	$name,\@function";
+				    $$type = "\@function";
+				}
+				if ($$type =~ /function/) {
+				    $ret .= "\n.section	\".opd\",\"aw\"";
+				    $ret .= "\n.align	3";
+				    $ret .= "\n$name:";
+				    $ret .= "\n.quad	.$name,.TOC.\@tocbase,0";
+				    $ret .= "\n.previous";
+				    $name = ".$name";
+				}
+				last;
+			      };
+    }
+
+    $ret = ".globl	$name" if (!$ret);
+    $$global = $name;
+    $ret;
+};
+my $text = sub {
+    my $ret = ($flavour =~ /aix/) ? ".csect\t.text[PR],7" : ".text";
+    $ret = ".abiversion	2\n".$ret	if ($flavour =~ /linux.*64le/);
+    $ret;
+};
+my $machine = sub {
+    my $junk = shift;
+    my $arch = shift;
+    if ($flavour =~ /osx/)
+    {	$arch =~ s/\"//g;
+	$arch = ($flavour=~/64/) ? "ppc970-64" : "ppc970" if ($arch eq "any");
+    }
+    ".machine	$arch";
+};
+my $size = sub {
+    if ($flavour =~ /linux/)
+    {	shift;
+	my $name = shift;
+	my $real = $GLOBALS{$name} ? \$GLOBALS{$name} : \$name;
+	my $ret  = ".size	$$real,.-$$real";
+	$name =~ s|^\.||;
+	if ($$real ne $name) {
+	    $ret .= "\n.size	$name,.-$$real";
+	}
+	$ret;
+    }
+    else
+    {	"";	}
+};
+my $asciz = sub {
+    shift;
+    my $line = join(",",@_);
+    if ($line =~ /^"(.*)"$/)
+    {	".byte	" . join(",",unpack("C*",$1),0) . "\n.align	2";	}
+    else
+    {	"";	}
+};
+my $quad = sub {
+    shift;
+    my @ret;
+    my ($hi,$lo);
+    for (@_) {
+	if (/^0x([0-9a-f]*?)([0-9a-f]{1,8})$/io)
+	{  $hi=$1?"0x$1":"0"; $lo="0x$2";  }
+	elsif (/^([0-9]+)$/o)
+	{  $hi=$1>>32; $lo=$1&0xffffffff;  } # error-prone with 32-bit perl
+	else
+	{  $hi=undef; $lo=$_; }
+
+	if (defined($hi))
+	{  push(@ret,$flavour=~/le$/o?".long\t$lo,$hi":".long\t$hi,$lo");  }
+	else
+	{  push(@ret,".quad	$lo");  }
+    }
+    join("\n",@ret);
+};
+
+################################################################
+# simplified mnemonics not handled by at least one assembler
+################################################################
+my $cmplw = sub {
+    my $f = shift;
+    my $cr = 0; $cr = shift if ($#_>1);
+    # Some out-of-date 32-bit GNU assembler just can't handle cmplw...
+    ($flavour =~ /linux.*32/) ?
+	"	.long	".sprintf "0x%x",31<<26|$cr<<23|$_[0]<<16|$_[1]<<11|64 :
+	"	cmplw	".join(',',$cr,@_);
+};
+my $bdnz = sub {
+    my $f = shift;
+    my $bo = $f=~/[\+\-]/ ? 16+9 : 16;	# optional "to be taken" hint
+    "	bc	$bo,0,".shift;
+} if ($flavour!~/linux/);
+my $bltlr = sub {
+    my $f = shift;
+    my $bo = $f=~/\-/ ? 12+2 : 12;	# optional "not to be taken" hint
+    ($flavour =~ /linux/) ?		# GNU as doesn't allow most recent hints
+	"	.long	".sprintf "0x%x",19<<26|$bo<<21|16<<1 :
+	"	bclr	$bo,0";
+};
+my $bnelr = sub {
+    my $f = shift;
+    my $bo = $f=~/\-/ ? 4+2 : 4;	# optional "not to be taken" hint
+    ($flavour =~ /linux/) ?		# GNU as doesn't allow most recent hints
+	"	.long	".sprintf "0x%x",19<<26|$bo<<21|2<<16|16<<1 :
+	"	bclr	$bo,2";
+};
+my $beqlr = sub {
+    my $f = shift;
+    my $bo = $f=~/-/ ? 12+2 : 12;	# optional "not to be taken" hint
+    ($flavour =~ /linux/) ?		# GNU as doesn't allow most recent hints
+	"	.long	".sprintf "0x%X",19<<26|$bo<<21|2<<16|16<<1 :
+	"	bclr	$bo,2";
+};
+# GNU assembler can't handle extrdi rA,rS,16,48, or when sum of last two
+# arguments is 64, with "operand out of range" error.
+my $extrdi = sub {
+    my ($f,$ra,$rs,$n,$b) = @_;
+    $b = ($b+$n)&63; $n = 64-$n;
+    "	rldicl	$ra,$rs,$b,$n";
+};
+my $vmr = sub {
+    my ($f,$vx,$vy) = @_;
+    "	vor	$vx,$vy,$vy";
+};
+
+# Some ABIs specify vrsave, special-purpose register #256, as reserved
+# for system use.
+my $no_vrsave = ($flavour =~ /aix|linux64le/);
+my $mtspr = sub {
+    my ($f,$idx,$ra) = @_;
+    if ($idx == 256 && $no_vrsave) {
+	"	or	$ra,$ra,$ra";
+    } else {
+	"	mtspr	$idx,$ra";
+    }
+};
+my $mfspr = sub {
+    my ($f,$rd,$idx) = @_;
+    if ($idx == 256 && $no_vrsave) {
+	"	li	$rd,-1";
+    } else {
+	"	mfspr	$rd,$idx";
+    }
+};
+
+# PowerISA 2.06 stuff
+sub vsxmem_op {
+    my ($f, $vrt, $ra, $rb, $op) = @_;
+    "	.long	".sprintf "0x%X",(31<<26)|($vrt<<21)|($ra<<16)|($rb<<11)|($op*2+1);
+}
+# made-up unaligned memory reference AltiVec/VMX instructions
+my $lvx_u	= sub {	vsxmem_op(@_, 844); };	# lxvd2x
+my $stvx_u	= sub {	vsxmem_op(@_, 972); };	# stxvd2x
+my $lvdx_u	= sub {	vsxmem_op(@_, 588); };	# lxsdx
+my $stvdx_u	= sub {	vsxmem_op(@_, 716); };	# stxsdx
+my $lvx_4w	= sub { vsxmem_op(@_, 780); };	# lxvw4x
+my $stvx_4w	= sub { vsxmem_op(@_, 908); };	# stxvw4x
+my $lvx_splt	= sub { vsxmem_op(@_, 332); };	# lxvdsx
+# VSX instruction[s] masqueraded as made-up AltiVec/VMX
+my $vpermdi	= sub {				# xxpermdi
+    my ($f, $vrt, $vra, $vrb, $dm) = @_;
+    $dm = oct($dm) if ($dm =~ /^0/);
+    "	.long	".sprintf "0x%X",(60<<26)|($vrt<<21)|($vra<<16)|($vrb<<11)|($dm<<8)|(10<<3)|7;
+};
+
+# PowerISA 2.07 stuff
+sub vcrypto_op {
+    my ($f, $vrt, $vra, $vrb, $op) = @_;
+    "	.long	".sprintf "0x%X",(4<<26)|($vrt<<21)|($vra<<16)|($vrb<<11)|$op;
+}
+sub vfour {
+    my ($f, $vrt, $vra, $vrb, $vrc, $op) = @_;
+    "	.long	".sprintf "0x%X",(4<<26)|($vrt<<21)|($vra<<16)|($vrb<<11)|($vrc<<6)|$op;
+};
+my $vcipher	= sub { vcrypto_op(@_, 1288); };
+my $vcipherlast	= sub { vcrypto_op(@_, 1289); };
+my $vncipher	= sub { vcrypto_op(@_, 1352); };
+my $vncipherlast= sub { vcrypto_op(@_, 1353); };
+my $vsbox	= sub { vcrypto_op(@_, 0, 1480); };
+my $vshasigmad	= sub { my ($st,$six)=splice(@_,-2); vcrypto_op(@_, $st<<4|$six, 1730); };
+my $vshasigmaw	= sub { my ($st,$six)=splice(@_,-2); vcrypto_op(@_, $st<<4|$six, 1666); };
+my $vpmsumb	= sub { vcrypto_op(@_, 1032); };
+my $vpmsumd	= sub { vcrypto_op(@_, 1224); };
+my $vpmsubh	= sub { vcrypto_op(@_, 1096); };
+my $vpmsumw	= sub { vcrypto_op(@_, 1160); };
+# These are not really crypto, but vcrypto_op template works
+my $vaddudm	= sub { vcrypto_op(@_, 192);  };
+my $vadduqm	= sub { vcrypto_op(@_, 256);  };
+my $vmuleuw	= sub { vcrypto_op(@_, 648);  };
+my $vmulouw	= sub { vcrypto_op(@_, 136);  };
+my $vrld	= sub { vcrypto_op(@_, 196);  };
+my $vsld	= sub { vcrypto_op(@_, 1476); };
+my $vsrd	= sub { vcrypto_op(@_, 1732); };
+my $vsubudm	= sub { vcrypto_op(@_, 1216); };
+my $vaddcuq	= sub { vcrypto_op(@_, 320);  };
+my $vaddeuqm	= sub { vfour(@_,60); };
+my $vaddecuq	= sub { vfour(@_,61); };
+my $vmrgew	= sub { vfour(@_,0,1932); };
+my $vmrgow	= sub { vfour(@_,0,1676); };
+
+my $mtsle	= sub {
+    my ($f, $arg) = @_;
+    "	.long	".sprintf "0x%X",(31<<26)|($arg<<21)|(147*2);
+};
+
+# VSX instructions masqueraded as AltiVec/VMX
+my $mtvrd	= sub {
+    my ($f, $vrt, $ra) = @_;
+    "	.long	".sprintf "0x%X",(31<<26)|($vrt<<21)|($ra<<16)|(179<<1)|1;
+};
+my $mtvrwz	= sub {
+    my ($f, $vrt, $ra) = @_;
+    "	.long	".sprintf "0x%X",(31<<26)|($vrt<<21)|($ra<<16)|(243<<1)|1;
+};
+my $lvwzx_u	= sub { vsxmem_op(@_, 12); };	# lxsiwzx
+my $stvwx_u	= sub { vsxmem_op(@_, 140); };	# stxsiwx
+
+# PowerISA 3.0 stuff
+my $maddhdu	= sub { vfour(@_,49); };
+my $maddld	= sub { vfour(@_,51); };
+my $darn = sub {
+    my ($f, $rt, $l) = @_;
+    "	.long	".sprintf "0x%X",(31<<26)|($rt<<21)|($l<<16)|(755<<1);
+};
+my $iseleq = sub {
+    my ($f, $rt, $ra, $rb) = @_;
+    "	.long	".sprintf "0x%X",(31<<26)|($rt<<21)|($ra<<16)|($rb<<11)|(2<<6)|30;
+};
+# VSX instruction[s] masqueraded as made-up AltiVec/VMX
+my $vspltib	= sub {				# xxspltib
+    my ($f, $vrt, $imm8) = @_;
+    $imm8 = oct($imm8) if ($imm8 =~ /^0/);
+    $imm8 &= 0xff;
+    "	.long	".sprintf "0x%X",(60<<26)|($vrt<<21)|($imm8<<11)|(360<<1)|1;
+};
+
+# PowerISA 3.0B stuff
+my $addex = sub {
+    my ($f, $rt, $ra, $rb, $cy) = @_;	# only cy==0 is specified in 3.0B
+    "	.long	".sprintf "0x%X",(31<<26)|($rt<<21)|($ra<<16)|($rb<<11)|($cy<<9)|(170<<1);
+};
+my $vmsumudm	= sub { vfour(@_,35); };
+
+while($line=<>) {
+
+    $line =~ s|[#!;].*$||;	# get rid of asm-style comments...
+    $line =~ s|/\*.*\*/||;	# ... and C-style comments...
+    $line =~ s|^\s+||;		# ... and skip white spaces in beginning...
+    $line =~ s|\s+$||;		# ... and at the end
+
+    {
+	$line =~ s|\.L(\w+)|L$1|g;	# common denominator for Locallabel
+	$line =~ s|\bL(\w+)|\.L$1|g	if ($dotinlocallabels);
+    }
+
+    {
+	$line =~ s|(^[\.\w]+)\:\s*||;
+	my $label = $1;
+	if ($label) {
+	    my $xlated = ($GLOBALS{$label} or $label);
+	    print "$xlated:";
+	    if ($flavour =~ /linux.*64le/) {
+		if ($TYPES{$label} =~ /function/) {
+		    printf "\n.localentry	%s,0\n",$xlated;
+		}
+	    }
+	}
+    }
+
+    {
+	$line =~ s|^\s*(\.?)(\w+)([\.\+\-]?)\s*||;
+	my $c = $1; $c = "\t" if ($c eq "");
+	my $mnemonic = $2;
+	my $f = $3;
+	my $opcode = eval("\$$mnemonic");
+	$line =~ s/\b(c?[rf]|v|vs)([0-9]+)\b/$2/g if ($c ne "." and $flavour !~ /osx/);
+	if (ref($opcode) eq 'CODE') { $line = &$opcode($f,split(/,\s*/,$line)); }
+	elsif ($mnemonic)           { $line = $c.$mnemonic.$f."\t".$line; }
+    }
+
+    print $line if ($line);
+    print "\n";
+}
+
+close STDOUT;
-- 
2.20.1

_______________________________________________
WireGuard mailing list
WireGuard@lists.zx2c4.com
https://lists.zx2c4.com/mailman/listinfo/wireguard

^ permalink raw reply	[flat|nested] 5+ messages in thread

* [PATCH 2/2 v3] [zinc] Add PowerPC accelerated poly1305 from openssl/cryptograms
  2019-05-13 21:31   ` [PATCH 1/2 v3] [Zinc] Add PowerPC chacha20 implementation " Shawn Landden
@ 2019-05-13 21:31     ` " Shawn Landden
  0 siblings, 0 replies; 5+ messages in thread
From: Shawn Landden @ 2019-05-13 21:31 UTC (permalink / raw)
  To: wireguard

Unfortunately I am not seeing a speed up with this patch,
but it does decrease CPU usage.

Only (currently) runs on the outbound path, as the in-bound path is in
an interrupt, but that can be fixed in Linux.

v2: - Do not include the FPU version, as +10% performance on POWER8
    (admittedly better on really old CPUs, like old world macs) is not
    worth it, especially when there is a fast VSX version available.
    - Honor CONFIG_VSX.
Signed-off-by: Shawn Landden <shawn@git.icu>
---
 src/crypto/Kbuild.include                     |   10 +-
 src/crypto/zinc/chacha20/chacha20-ppc.pl      |    3 +
 .../zinc/{chacha20 => perlasm}/ppc-xlate.pl   |    0
 src/crypto/zinc/poly1305/poly1305-arm-glue.c  |   65 -
 src/crypto/zinc/poly1305/poly1305-ppc-glue.c  |   60 +
 src/crypto/zinc/poly1305/poly1305-ppc.pl      | 1989 +++++++++++++++++
 src/crypto/zinc/poly1305/poly1305.c           |   69 +
 7 files changed, 2127 insertions(+), 69 deletions(-)
 rename src/crypto/zinc/{chacha20 => perlasm}/ppc-xlate.pl (100%)
 create mode 100644 src/crypto/zinc/poly1305/poly1305-ppc-glue.c
 create mode 100644 src/crypto/zinc/poly1305/poly1305-ppc.pl

diff --git a/src/crypto/Kbuild.include b/src/crypto/Kbuild.include
index 4e05181..ece20c9 100644
--- a/src/crypto/Kbuild.include
+++ b/src/crypto/Kbuild.include
@@ -34,30 +34,32 @@ zinc-$(CONFIG_ZINC_ARCH_X86_64) += poly1305/poly1305-x86_64.o
 zinc-$(CONFIG_ZINC_ARCH_ARM) += poly1305/poly1305-arm.o
 zinc-$(CONFIG_ZINC_ARCH_ARM64) += poly1305/poly1305-arm64.o
 zinc-$(CONFIG_ZINC_ARCH_MIPS) += poly1305/poly1305-mips.o
 AFLAGS_poly1305-mips.o += -O2 # This is required to fill the branch delay slots
 zinc-$(CONFIG_ZINC_ARCH_MIPS64) += poly1305/poly1305-mips64.o
+zinc-$(CONFIG_ZINC_ARCH_PPC32) += poly1305/poly1305-ppc.o
+zinc-$(CONFIG_ZINC_ARCH_PPC64) += poly1305/poly1305-ppc.o
 
 zinc-y += chacha20poly1305.o
 
 zinc-y += blake2s/blake2s.o
 zinc-$(CONFIG_ZINC_ARCH_X86_64) += blake2s/blake2s-x86_64.o
 
 zinc-y += curve25519/curve25519.o
 zinc-$(CONFIG_ZINC_ARCH_ARM) += curve25519/curve25519-arm.o
 
 quiet_cmd_perlasm = PERLASM $@
-      cmd_perlasm = $(PERL) $< $(perlflags-y) > $@
+      cmd_perlasm = $(PERL) $(perlflags-y) $< $(perlargs-y) > $@
 $(obj)/%.S: $(src)/%.pl FORCE
 	$(call if_changed,perlasm)
 kbuild-dir := $(if $(filter /%,$(src)),$(src),$(srctree)/$(src))
 targets := $(patsubst $(kbuild-dir)/%.pl,%.S,$(wildcard $(patsubst %.o,$(kbuild-dir)/crypto/zinc/%.pl,$(zinc-y) $(zinc-m) $(zinc-))))
 
-perlflags-$(CONFIG_ZINC_ARCH_PPC32) += linux32
+perlargs-$(CONFIG_ZINC_ARCH_PPC32) += linux32
 ifeq ($(CONFIG_ZINC_ARCH_PPC64),y)
-perlflags-$(CONFIG_CPU_BIG_ENDIAN) += linux64
-perlflags-$(CONFIG_CPU_LITTLE_ENDIAN) += linux64le
+perlargs-$(CONFIG_CPU_BIG_ENDIAN) += linux64
+perlargs-$(CONFIG_CPU_LITTLE_ENDIAN) += linux64le
 endif
 
 # Old kernels don't set this, which causes trouble.
 .SECONDARY:
 
diff --git a/src/crypto/zinc/chacha20/chacha20-ppc.pl b/src/crypto/zinc/chacha20/chacha20-ppc.pl
index 07468c8..fa8f6bc 100644
--- a/src/crypto/zinc/chacha20/chacha20-ppc.pl
+++ b/src/crypto/zinc/chacha20/chacha20-ppc.pl
@@ -4,10 +4,12 @@
 # This code is taken from the OpenSSL project but the author, Andy Polyakov,
 # has relicensed it under the licenses specified in the SPDX header above.
 # The original headers, including the original license headers, are
 # included below for completeness.
 #
+# Changes: search in more places for ppc-xlate.pl
+#
 # Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved.
 #
 # Licensed under the Apache License 2.0 (the "License").  You may not use
 # this file except in compliance with the License.  You can obtain a copy
 # in the file LICENSE in the source distribution or at
@@ -71,10 +73,11 @@ if ($flavour =~ /64/) {
 
 $LITTLE_ENDIAN = ($flavour=~/le$/) ? 1 : 0;
 
 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
 ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
+( $xlate="${dir}../perlasm/ppc-xlate.pl" and -f $xlate) or
 ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
 die "can't locate ppc-xlate.pl";
 
 open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";
 
diff --git a/src/crypto/zinc/chacha20/ppc-xlate.pl b/src/crypto/zinc/perlasm/ppc-xlate.pl
similarity index 100%
rename from src/crypto/zinc/chacha20/ppc-xlate.pl
rename to src/crypto/zinc/perlasm/ppc-xlate.pl
diff --git a/src/crypto/zinc/poly1305/poly1305-arm-glue.c b/src/crypto/zinc/poly1305/poly1305-arm-glue.c
index a80f046..6100700 100644
--- a/src/crypto/zinc/poly1305/poly1305-arm-glue.c
+++ b/src/crypto/zinc/poly1305/poly1305-arm-glue.c
@@ -24,75 +24,10 @@ static void __init poly1305_fpu_init(void)
 #elif defined(CONFIG_ZINC_ARCH_ARM)
 	poly1305_use_neon = elf_hwcap & HWCAP_NEON;
 #endif
 }
 
-#if defined(CONFIG_ZINC_ARCH_ARM64)
-struct poly1305_arch_internal {
-	union {
-		u32 h[5];
-		struct {
-			u64 h0, h1, h2;
-		};
-	};
-	u64 is_base2_26;
-	u64 r[2];
-};
-#elif defined(CONFIG_ZINC_ARCH_ARM)
-struct poly1305_arch_internal {
-	union {
-		u32 h[5];
-		struct {
-			u64 h0, h1;
-			u32 h2;
-		} __packed;
-	};
-	u32 r[4];
-	u32 is_base2_26;
-};
-#endif
-
-/* The NEON code uses base 2^26, while the scalar code uses base 2^64 on 64-bit
- * and base 2^32 on 32-bit. If we hit the unfortunate situation of using NEON
- * and then having to go back to scalar -- because the user is silly and has
- * called the update function from two separate contexts -- then we need to
- * convert back to the original base before proceeding. The below function is
- * written for 64-bit integers, and so we have to swap words at the end on
- * big-endian 32-bit. It is possible to reason that the initial reduction below
- * is sufficient given the implementation invariants. However, for an avoidance
- * of doubt and because this is not performance critical, we do the full
- * reduction anyway.
- */
-static void convert_to_base2_64(void *ctx)
-{
-	struct poly1305_arch_internal *state = ctx;
-	u32 cy;
-
-	if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !state->is_base2_26)
-		return;
-
-	cy = state->h[0] >> 26; state->h[0] &= 0x3ffffff; state->h[1] += cy;
-	cy = state->h[1] >> 26; state->h[1] &= 0x3ffffff; state->h[2] += cy;
-	cy = state->h[2] >> 26; state->h[2] &= 0x3ffffff; state->h[3] += cy;
-	cy = state->h[3] >> 26; state->h[3] &= 0x3ffffff; state->h[4] += cy;
-	state->h0 = ((u64)state->h[2] << 52) | ((u64)state->h[1] << 26) | state->h[0];
-	state->h1 = ((u64)state->h[4] << 40) | ((u64)state->h[3] << 14) | (state->h[2] >> 12);
-	state->h2 = state->h[4] >> 24;
-	if (IS_ENABLED(CONFIG_ZINC_ARCH_ARM) && IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) {
-		state->h0 = rol64(state->h0, 32);
-		state->h1 = rol64(state->h1, 32);
-	}
-#define ULT(a, b) ((a ^ ((a ^ b) | ((a - b) ^ b))) >> (sizeof(a) * 8 - 1))
-	cy = (state->h2 >> 2) + (state->h2 & ~3ULL);
-	state->h2 &= 3;
-	state->h0 += cy;
-	state->h1 += (cy = ULT(state->h0, cy));
-	state->h2 += ULT(state->h1, cy);
-#undef ULT
-	state->is_base2_26 = 0;
-}
-
 static inline bool poly1305_init_arch(void *ctx,
 				      const u8 key[POLY1305_KEY_SIZE])
 {
 	poly1305_init_arm(ctx, key);
 	return true;
diff --git a/src/crypto/zinc/poly1305/poly1305-ppc-glue.c b/src/crypto/zinc/poly1305/poly1305-ppc-glue.c
new file mode 100644
index 0000000..265467e
--- /dev/null
+++ b/src/crypto/zinc/poly1305/poly1305-ppc-glue.c
@@ -0,0 +1,60 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (C) 2019 Shawn Landden <shawn@git.icu>. All Rights Reserved.
+ */
+
+#include <asm/cpufeature.h>
+
+asmlinkage void poly1305_init_int(void *ctx, const u8 key[16]);
+asmlinkage void poly1305_blocks_int(void *ctx, const u8 *inp, size_t len,
+				    u32 padbit);
+asmlinkage void poly1305_emit_int(void *ctx, u8 mac[16],
+				  const u32 nonce[4]);
+asmlinkage void poly1305_blocks_vsx(void *ctx, const u8 *inp, size_t len,
+				    u32 padbit);
+static bool *const poly1305_nobs[] __initconst = {};
+static void __init poly1305_fpu_init(void) {}
+
+static inline bool poly1305_init_arch(void *ctx,
+				      const u8 key[POLY1305_KEY_SIZE])
+{
+	poly1305_init_int(ctx, key);
+	return true;
+}
+
+static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp,
+					size_t len, const u32 padbit,
+					simd_context_t *simd_context)
+{
+	/* SIMD disables preemption, so relax after processing each page. */
+	BUILD_BUG_ON(PAGE_SIZE < POLY1305_BLOCK_SIZE ||
+		     PAGE_SIZE % POLY1305_BLOCK_SIZE);
+
+	if (!IS_ENABLED(CONFIG_VSX) ||
+	    !cpu_have_feature(PPC_MODULE_FEATURE_VEC_CRYPTO) ||
+	    !simd_use(simd_context)) {
+		convert_to_base2_64(ctx);
+		poly1305_blocks_int(ctx, inp, len, padbit);
+		return true;
+	}
+
+	for (;;) {
+		const size_t bytes = min_t(size_t, len, PAGE_SIZE);
+
+		poly1305_blocks_vsx(ctx, inp, bytes, padbit);
+		len -= bytes;
+		if (!len)
+			break;
+		inp += bytes;
+		simd_relax(simd_context);
+	}
+	return true;
+}
+
+static inline bool poly1305_emit_arch(void *ctx, u8 mac[POLY1305_MAC_SIZE],
+				      const u32 nonce[4],
+				      simd_context_t *simd_context)
+{
+	poly1305_emit_int(ctx, mac, nonce);
+	return true;
+}
diff --git a/src/crypto/zinc/poly1305/poly1305-ppc.pl b/src/crypto/zinc/poly1305/poly1305-ppc.pl
new file mode 100644
index 0000000..dd4e3fb
--- /dev/null
+++ b/src/crypto/zinc/poly1305/poly1305-ppc.pl
@@ -0,0 +1,1989 @@
+#! /usr/bin/env perl
+# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+#
+# This code is taken from the OpenSSL project but the author, Andy Polyakov,
+# has relicensed it under the licenses specified in the SPDX header above.
+# The original headers, including the original license headers, are
+# included below for completeness.
+#
+# Changes: renamed poly1305_emit to poly1305_emit_int,
+#          renamed poly1305_blocks to poly1305_blocks_int
+#          Look in more places for ppc-xlate.pl
+#
+# Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the Apache License 2.0 (the "License").  You may not use
+# this file except in compliance with the License.  You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
+#
+# ====================================================================
+# Written by Andy Polyakov, @dot-asm, initially for use in the OpenSSL
+# project. The module is dual licensed under OpenSSL and CRYPTOGAMS
+# licenses depending on where you obtain it. For further details see
+# https://github.com/dot-asm/cryptogams/.
+# ====================================================================
+#
+# This module implements Poly1305 hash for PowerPC.
+#
+# June 2015
+#
+# Numbers are cycles per processed byte with poly1305_blocks alone,
+# and improvement coefficients relative to gcc-generated code.
+#
+#			-m32		-m64
+#
+# Freescale e300	14.8/+80%	-
+# PPC74x0		7.60/+60%	-
+# PPC970		7.00/+114%	3.51/+205%
+# POWER7		3.75/+260%	1.93/+100%
+# POWER8		-		2.03/+200%
+# POWER9		-		2.00/+150%
+#
+# Do we need floating-point implementation for PPC? Results presented
+# in poly1305_ieee754.c are tricky to compare to, because they are for
+# compiler-generated code. On the other hand it's known that floating-
+# point performance can be dominated by FPU latency, which means that
+# there is limit even for ideally optimized (and even vectorized) code.
+# And this limit is estimated to be higher than above -m64 results. Or
+# in other words floating-point implementation can be meaningful to
+# consider only in 32-bit application context. We probably have to
+# recognize that 32-bit builds are getting less popular on high-end
+# systems and therefore tend to target embedded ones, which might not
+# even have FPU...
+#
+# On side note, Power ISA 2.07 enables vector base 2^26 implementation,
+# and POWER8 might have capacity to break 1.0 cycle per byte barrier...
+#
+# January 2019
+#
+# ... Unfortunately not:-( Estimate was a projection of ARM result,
+# but ARM has vector multiply-n-add instruction, while PowerISA does
+# not, not one usable in the context. Improvement is ~40% over -m64
+# result above and is ~1.43 on little-endian systems.
+
+$flavour = shift;
+
+if ($flavour =~ /64/) {
+	$SIZE_T	=8;
+	$LRSAVE	=2*$SIZE_T;
+	$UCMP	="cmpld";
+	$STU	="stdu";
+	$POP	="ld";
+	$PUSH	="std";
+} elsif ($flavour =~ /32/) {
+	$SIZE_T	=4;
+	$LRSAVE	=$SIZE_T;
+	$UCMP	="cmplw";
+	$STU	="stwu";
+	$POP	="lwz";
+	$PUSH	="stw";
+} else { die "nonsense $flavour"; }
+
+# Define endianness based on flavour
+# i.e.: linux64le
+$LITTLE_ENDIAN = ($flavour=~/le$/) ? $SIZE_T : 0;
+
+$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
+( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
+( $xlate="${dir}../perlasm/ppc-xlate.pl" and -f $xlate) or
+( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
+die "can't locate ppc-xlate.pl";
+
+open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";
+
+$FRAME=24*$SIZE_T;
+
+$sp="r1";
+my ($ctx,$inp,$len,$padbit) = map("r$_",(3..6));
+my ($mac,$nonce)=($inp,$len);
+my $mask = "r0";
+
+$code=<<___;
+.machine	"any"
+.text
+___
+							if ($flavour =~ /64/) {
+###############################################################################
+# base 2^64 implementation
+
+my ($h0,$h1,$h2,$d0,$d1,$d2, $r0,$r1,$s1, $t0,$t1) = map("r$_",(7..12,27..31));
+
+$code.=<<___;
+.globl	.poly1305_init_int
+.align	4
+.poly1305_init_int:
+	xor	r0,r0,r0
+	std	r0,0($ctx)		# zero hash value
+	std	r0,8($ctx)
+	std	r0,16($ctx)
+	stw	r0,24($ctx)		# clear is_base2_26
+
+	$UCMP	$inp,r0
+	beq-	Lno_key
+___
+$code.=<<___	if ($LITTLE_ENDIAN);
+	ld	$d0,0($inp)		# load key material
+	ld	$d1,8($inp)
+___
+$code.=<<___	if (!$LITTLE_ENDIAN);
+	li	$h0,4
+	lwbrx	$d0,0,$inp		# load key material
+	li	$d1,8
+	lwbrx	$h0,$h0,$inp
+	li	$h1,12
+	lwbrx	$d1,$d1,$inp
+	lwbrx	$h1,$h1,$inp
+	insrdi	$d0,$h0,32,0
+	insrdi	$d1,$h1,32,0
+___
+$code.=<<___;
+	lis	$h1,0xfff		# 0x0fff0000
+	ori	$h1,$h1,0xfffc		# 0x0ffffffc
+	insrdi	$h1,$h1,32,0		# 0x0ffffffc0ffffffc
+	ori	$h0,$h1,3		# 0x0ffffffc0fffffff
+
+	and	$d0,$d0,$h0
+	and	$d1,$d1,$h1
+
+	std	$d0,32($ctx)		# store key
+	std	$d1,40($ctx)
+
+Lno_key:
+	xor	r3,r3,r3
+	blr
+	.long	0
+	.byte	0,12,0x14,0,0,0,2,0
+.size	.poly1305_init_int,.-.poly1305_init_int
+
+.globl	.poly1305_blocks_int
+.align	4
+.poly1305_blocks_int:
+Lpoly1305_blocks:
+	srdi.	$len,$len,4
+	beq-	Labort
+
+	$STU	$sp,-$FRAME($sp)
+	mflr	r0
+	$PUSH	r27,`$FRAME-$SIZE_T*5`($sp)
+	$PUSH	r28,`$FRAME-$SIZE_T*4`($sp)
+	$PUSH	r29,`$FRAME-$SIZE_T*3`($sp)
+	$PUSH	r30,`$FRAME-$SIZE_T*2`($sp)
+	$PUSH	r31,`$FRAME-$SIZE_T*1`($sp)
+	$PUSH	r0,`$FRAME+$LRSAVE`($sp)
+
+	ld	$r0,32($ctx)		# load key
+	ld	$r1,40($ctx)
+
+	ld	$h0,0($ctx)		# load hash value
+	ld	$h1,8($ctx)
+	ld	$h2,16($ctx)
+
+	srdi	$s1,$r1,2
+	mtctr	$len
+	add	$s1,$s1,$r1		# s1 = r1 + r1>>2
+	li	$mask,3
+	b	Loop
+
+.align	4
+Loop:
+___
+$code.=<<___	if ($LITTLE_ENDIAN);
+	ld	$t0,0($inp)		# load input
+	ld	$t1,8($inp)
+___
+$code.=<<___	if (!$LITTLE_ENDIAN);
+	li	$d0,4
+	lwbrx	$t0,0,$inp		# load input
+	li	$t1,8
+	lwbrx	$d0,$d0,$inp
+	li	$d1,12
+	lwbrx	$t1,$t1,$inp
+	lwbrx	$d1,$d1,$inp
+	insrdi	$t0,$d0,32,0
+	insrdi	$t1,$d1,32,0
+___
+$code.=<<___;
+	addi	$inp,$inp,16
+
+	addc	$h0,$h0,$t0		# accumulate input
+	adde	$h1,$h1,$t1
+
+	mulld	$d0,$h0,$r0		# h0*r0
+	mulhdu	$d1,$h0,$r0
+	adde	$h2,$h2,$padbit
+
+	mulld	$t0,$h1,$s1		# h1*5*r1
+	mulhdu	$t1,$h1,$s1
+	addc	$d0,$d0,$t0
+	adde	$d1,$d1,$t1
+
+	mulld	$t0,$h0,$r1		# h0*r1
+	mulhdu	$d2,$h0,$r1
+	addc	$d1,$d1,$t0
+	addze	$d2,$d2
+
+	mulld	$t0,$h1,$r0		# h1*r0
+	mulhdu	$t1,$h1,$r0
+	addc	$d1,$d1,$t0
+	adde	$d2,$d2,$t1
+
+	mulld	$t0,$h2,$s1		# h2*5*r1
+	mulld	$t1,$h2,$r0		# h2*r0
+	addc	$d1,$d1,$t0
+	adde	$d2,$d2,$t1
+
+	andc	$t0,$d2,$mask		# final reduction step
+	and	$h2,$d2,$mask
+	srdi	$t1,$t0,2
+	add	$t0,$t0,$t1
+	addc	$h0,$d0,$t0
+	addze	$h1,$d1
+	addze	$h2,$h2
+
+	bdnz	Loop
+
+	std	$h0,0($ctx)		# store hash value
+	std	$h1,8($ctx)
+	std	$h2,16($ctx)
+
+	$POP	r27,`$FRAME-$SIZE_T*5`($sp)
+	$POP	r28,`$FRAME-$SIZE_T*4`($sp)
+	$POP	r29,`$FRAME-$SIZE_T*3`($sp)
+	$POP	r30,`$FRAME-$SIZE_T*2`($sp)
+	$POP	r31,`$FRAME-$SIZE_T*1`($sp)
+	addi	$sp,$sp,$FRAME
+Labort:
+	blr
+	.long	0
+	.byte	0,12,4,1,0x80,5,4,0
+.size	.poly1305_blocks_int,.-.poly1305_blocks_int
+___
+{
+my ($h0,$h1,$h2,$h3,$h4,$t0) = map("r$_",(7..12));
+
+$code.=<<___;
+.globl	.poly1305_emit_int
+.align	5
+.poly1305_emit_int:
+	lwz	$h0,0($ctx)	# load hash value base 2^26
+	lwz	$h1,4($ctx)
+	lwz	$h2,8($ctx)
+	lwz	$h3,12($ctx)
+	lwz	$h4,16($ctx)
+	lwz	r0,24($ctx)	# is_base2_26
+
+	sldi	$h1,$h1,26	# base 2^26 -> base 2^64
+	sldi	$t0,$h2,52
+	srdi	$h2,$h2,12
+	sldi	$h3,$h3,14
+	add	$h0,$h0,$h1
+	addc	$h0,$h0,$t0
+	sldi	$t0,$h4,40
+	srdi	$h4,$h4,24
+	adde	$h1,$h2,$h3
+	addc	$h1,$h1,$t0
+	addze	$h2,$h4
+
+	ld	$h3,0($ctx)	# load hash value base 2^64
+	ld	$h4,8($ctx)
+	ld	$t0,16($ctx)
+
+	neg	r0,r0
+	xor	$h0,$h0,$h3	# choose between radixes
+	xor	$h1,$h1,$h4
+	xor	$h2,$h2,$t0
+	and	$h0,$h0,r0
+	and	$h1,$h1,r0
+	and	$h2,$h2,r0
+	xor	$h0,$h0,$h3
+	xor	$h1,$h1,$h4
+	xor	$h2,$h2,$t0
+
+	addic	$h3,$h0,5	# compare to modulus
+	addze	$h4,$h1
+	addze	$t0,$h2
+
+	srdi	$t0,$t0,2	# see if it carried/borrowed
+	neg	$t0,$t0
+
+	andc	$h0,$h0,$t0
+	and	$h3,$h3,$t0
+	andc	$h1,$h1,$t0
+	and	$h4,$h4,$t0
+	or	$h0,$h0,$h3
+	or	$h1,$h1,$h4
+
+	lwz	$t0,4($nonce)
+	lwz	$h2,12($nonce)
+	lwz	$h3,0($nonce)
+	lwz	$h4,8($nonce)
+
+	insrdi	$h3,$t0,32,0
+	insrdi	$h4,$h2,32,0
+
+	addc	$h0,$h0,$h3	# accumulate nonce
+	adde	$h1,$h1,$h4
+
+	addi	$ctx,$mac,-1
+	addi	$mac,$mac,7
+
+	stbu	$h0,1($ctx)	# write [little-endian] result
+	srdi	$h0,$h0,8
+	stbu	$h1,1($mac)
+	srdi	$h1,$h1,8
+
+	stbu	$h0,1($ctx)
+	srdi	$h0,$h0,8
+	stbu	$h1,1($mac)
+	srdi	$h1,$h1,8
+
+	stbu	$h0,1($ctx)
+	srdi	$h0,$h0,8
+	stbu	$h1,1($mac)
+	srdi	$h1,$h1,8
+
+	stbu	$h0,1($ctx)
+	srdi	$h0,$h0,8
+	stbu	$h1,1($mac)
+	srdi	$h1,$h1,8
+
+	stbu	$h0,1($ctx)
+	srdi	$h0,$h0,8
+	stbu	$h1,1($mac)
+	srdi	$h1,$h1,8
+
+	stbu	$h0,1($ctx)
+	srdi	$h0,$h0,8
+	stbu	$h1,1($mac)
+	srdi	$h1,$h1,8
+
+	stbu	$h0,1($ctx)
+	srdi	$h0,$h0,8
+	stbu	$h1,1($mac)
+	srdi	$h1,$h1,8
+
+	stbu	$h0,1($ctx)
+	stbu	$h1,1($mac)
+
+	blr
+	.long	0
+	.byte	0,12,0x14,0,0,0,3,0
+.size	.poly1305_emit_int,.-.poly1305_emit_int
+___
+}							} else {
+###############################################################################
+# base 2^32 implementation
+
+my ($h0,$h1,$h2,$h3,$h4, $r0,$r1,$r2,$r3, $s1,$s2,$s3,
+    $t0,$t1,$t2,$t3, $D0,$D1,$D2,$D3, $d0,$d1,$d2,$d3
+   ) = map("r$_",(7..12,14..31));
+
+$code.=<<___;
+.globl	.poly1305_init_int
+.align	4
+.poly1305_init_int:
+	xor	r0,r0,r0
+	stw	r0,0($ctx)		# zero hash value
+	stw	r0,4($ctx)
+	stw	r0,8($ctx)
+	stw	r0,12($ctx)
+	stw	r0,16($ctx)
+	stw	r0,24($ctx)		# clear is_base2_26
+
+	$UCMP	$inp,r0
+	beq-	Lno_key
+___
+$code.=<<___	if ($LITTLE_ENDIAN);
+	lw	$h0,0($inp)		# load key material
+	lw	$h1,4($inp)
+	lw	$h2,8($inp)
+	lw	$h3,12($inp)
+___
+$code.=<<___	if (!$LITTLE_ENDIAN);
+	li	$h1,4
+	lwbrx	$h0,0,$inp		# load key material
+	li	$h2,8
+	lwbrx	$h1,$h1,$inp
+	li	$h3,12
+	lwbrx	$h2,$h2,$inp
+	lwbrx	$h3,$h3,$inp
+___
+$code.=<<___;
+	lis	$mask,0xf000		# 0xf0000000
+	li	$r0,-4
+	andc	$r0,$r0,$mask		# 0x0ffffffc
+
+	andc	$h0,$h0,$mask
+	and	$h1,$h1,$r0
+	and	$h2,$h2,$r0
+	and	$h3,$h3,$r0
+
+	stw	$h0,32($ctx)		# store key
+	stw	$h1,36($ctx)
+	stw	$h2,40($ctx)
+	stw	$h3,44($ctx)
+
+Lno_key:
+	xor	r3,r3,r3
+	blr
+	.long	0
+	.byte	0,12,0x14,0,0,0,2,0
+.size	.poly1305_init_int,.-.poly1305_init_int
+
+.globl	.poly1305_blocks_int
+.align	4
+.poly1305_blocks_int:
+Lpoly1305_blocks:
+	srwi.	$len,$len,4
+	beq-	Labort
+
+	$STU	$sp,-$FRAME($sp)
+	mflr	r0
+	$PUSH	r14,`$FRAME-$SIZE_T*18`($sp)
+	$PUSH	r15,`$FRAME-$SIZE_T*17`($sp)
+	$PUSH	r16,`$FRAME-$SIZE_T*16`($sp)
+	$PUSH	r17,`$FRAME-$SIZE_T*15`($sp)
+	$PUSH	r18,`$FRAME-$SIZE_T*14`($sp)
+	$PUSH	r19,`$FRAME-$SIZE_T*13`($sp)
+	$PUSH	r20,`$FRAME-$SIZE_T*12`($sp)
+	$PUSH	r21,`$FRAME-$SIZE_T*11`($sp)
+	$PUSH	r22,`$FRAME-$SIZE_T*10`($sp)
+	$PUSH	r23,`$FRAME-$SIZE_T*9`($sp)
+	$PUSH	r24,`$FRAME-$SIZE_T*8`($sp)
+	$PUSH	r25,`$FRAME-$SIZE_T*7`($sp)
+	$PUSH	r26,`$FRAME-$SIZE_T*6`($sp)
+	$PUSH	r27,`$FRAME-$SIZE_T*5`($sp)
+	$PUSH	r28,`$FRAME-$SIZE_T*4`($sp)
+	$PUSH	r29,`$FRAME-$SIZE_T*3`($sp)
+	$PUSH	r30,`$FRAME-$SIZE_T*2`($sp)
+	$PUSH	r31,`$FRAME-$SIZE_T*1`($sp)
+	$PUSH	r0,`$FRAME+$LRSAVE`($sp)
+
+	lwz	$r0,32($ctx)		# load key
+	lwz	$r1,36($ctx)
+	lwz	$r2,40($ctx)
+	lwz	$r3,44($ctx)
+
+	lwz	$h0,0($ctx)		# load hash value
+	lwz	$h1,4($ctx)
+	lwz	$h2,8($ctx)
+	lwz	$h3,12($ctx)
+	lwz	$h4,16($ctx)
+
+	srwi	$s1,$r1,2
+	srwi	$s2,$r2,2
+	srwi	$s3,$r3,2
+	add	$s1,$s1,$r1		# si = ri + ri>>2
+	add	$s2,$s2,$r2
+	add	$s3,$s3,$r3
+	mtctr	$len
+	li	$mask,3
+	b	Loop
+
+.align	4
+Loop:
+___
+$code.=<<___	if ($LITTLE_ENDIAN);
+	lwz	$d0,0($inp)		# load input
+	lwz	$d1,4($inp)
+	lwz	$d2,8($inp)
+	lwz	$d3,12($inp)
+___
+$code.=<<___	if (!$LITTLE_ENDIAN);
+	li	$d1,4
+	lwbrx	$d0,0,$inp		# load input
+	li	$d2,8
+	lwbrx	$d1,$d1,$inp
+	li	$d3,12
+	lwbrx	$d2,$d2,$inp
+	lwbrx	$d3,$d3,$inp
+___
+$code.=<<___;
+	addi	$inp,$inp,16
+
+	addc	$h0,$h0,$d0		# accumulate input
+	adde	$h1,$h1,$d1
+	adde	$h2,$h2,$d2
+
+	mullw	$d0,$h0,$r0		# h0*r0
+	mulhwu	$D0,$h0,$r0
+
+	mullw	$d1,$h0,$r1		# h0*r1
+	mulhwu	$D1,$h0,$r1
+
+	mullw	$d2,$h0,$r2		# h0*r2
+	mulhwu	$D2,$h0,$r2
+
+	 adde	$h3,$h3,$d3
+	 adde	$h4,$h4,$padbit
+
+	mullw	$d3,$h0,$r3		# h0*r3
+	mulhwu	$D3,$h0,$r3
+
+	mullw	$t0,$h1,$s3		# h1*s3
+	mulhwu	$t1,$h1,$s3
+
+	mullw	$t2,$h1,$r0		# h1*r0
+	mulhwu	$t3,$h1,$r0
+	 addc	$d0,$d0,$t0
+	 adde	$D0,$D0,$t1
+
+	mullw	$t0,$h1,$r1		# h1*r1
+	mulhwu	$t1,$h1,$r1
+	 addc	$d1,$d1,$t2
+	 adde	$D1,$D1,$t3
+
+	mullw	$t2,$h1,$r2		# h1*r2
+	mulhwu	$t3,$h1,$r2
+	 addc	$d2,$d2,$t0
+	 adde	$D2,$D2,$t1
+
+	mullw	$t0,$h2,$s2		# h2*s2
+	mulhwu	$t1,$h2,$s2
+	 addc	$d3,$d3,$t2
+	 adde	$D3,$D3,$t3
+
+	mullw	$t2,$h2,$s3		# h2*s3
+	mulhwu	$t3,$h2,$s3
+	 addc	$d0,$d0,$t0
+	 adde	$D0,$D0,$t1
+
+	mullw	$t0,$h2,$r0		# h2*r0
+	mulhwu	$t1,$h2,$r0
+	 addc	$d1,$d1,$t2
+	 adde	$D1,$D1,$t3
+
+	mullw	$t2,$h2,$r1		# h2*r1
+	mulhwu	$t3,$h2,$r1
+	 addc	$d2,$d2,$t0
+	 adde	$D2,$D2,$t1
+
+	mullw	$t0,$h3,$s1		# h3*s1
+	mulhwu	$t1,$h3,$s1
+	 addc	$d3,$d3,$t2
+	 adde	$D3,$D3,$t3
+
+	mullw	$t2,$h3,$s2		# h3*s2
+	mulhwu	$t3,$h3,$s2
+	 addc	$d0,$d0,$t0
+	 adde	$D0,$D0,$t1
+
+	mullw	$t0,$h3,$s3		# h3*s3
+	mulhwu	$t1,$h3,$s3
+	 addc	$d1,$d1,$t2
+	 adde	$D1,$D1,$t3
+
+	mullw	$t2,$h3,$r0		# h3*r0
+	mulhwu	$t3,$h3,$r0
+	 addc	$d2,$d2,$t0
+	 adde	$D2,$D2,$t1
+
+	mullw	$t0,$h4,$s1		# h4*s1
+	 addc	$d3,$d3,$t2
+	 adde	$D3,$D3,$t3
+	addc	$d1,$d1,$t0
+
+	mullw	$t1,$h4,$s2		# h4*s2
+	 addze	$D1,$D1
+	addc	$d2,$d2,$t1
+	addze	$D2,$D2
+
+	mullw	$t2,$h4,$s3		# h4*s3
+	addc	$d3,$d3,$t2
+	addze	$D3,$D3
+
+	mullw	$h4,$h4,$r0		# h4*r0
+
+	addc	$h1,$d1,$D0
+	adde	$h2,$d2,$D1
+	adde	$h3,$d3,$D2
+	adde	$h4,$h4,$D3
+
+	andc	$D0,$h4,$mask		# final reduction step
+	and	$h4,$h4,$mask
+	srwi	$D1,$D0,2
+	add	$D0,$D0,$D1
+	addc	$h0,$d0,$D0
+	addze	$h1,$h1
+	addze	$h2,$h2
+	addze	$h3,$h3
+	addze	$h4,$h4
+
+	bdnz	Loop
+
+	stw	$h0,0($ctx)		# store hash value
+	stw	$h1,4($ctx)
+	stw	$h2,8($ctx)
+	stw	$h3,12($ctx)
+	stw	$h4,16($ctx)
+
+	$POP	r14,`$FRAME-$SIZE_T*18`($sp)
+	$POP	r15,`$FRAME-$SIZE_T*17`($sp)
+	$POP	r16,`$FRAME-$SIZE_T*16`($sp)
+	$POP	r17,`$FRAME-$SIZE_T*15`($sp)
+	$POP	r18,`$FRAME-$SIZE_T*14`($sp)
+	$POP	r19,`$FRAME-$SIZE_T*13`($sp)
+	$POP	r20,`$FRAME-$SIZE_T*12`($sp)
+	$POP	r21,`$FRAME-$SIZE_T*11`($sp)
+	$POP	r22,`$FRAME-$SIZE_T*10`($sp)
+	$POP	r23,`$FRAME-$SIZE_T*9`($sp)
+	$POP	r24,`$FRAME-$SIZE_T*8`($sp)
+	$POP	r25,`$FRAME-$SIZE_T*7`($sp)
+	$POP	r26,`$FRAME-$SIZE_T*6`($sp)
+	$POP	r27,`$FRAME-$SIZE_T*5`($sp)
+	$POP	r28,`$FRAME-$SIZE_T*4`($sp)
+	$POP	r29,`$FRAME-$SIZE_T*3`($sp)
+	$POP	r30,`$FRAME-$SIZE_T*2`($sp)
+	$POP	r31,`$FRAME-$SIZE_T*1`($sp)
+	addi	$sp,$sp,$FRAME
+Labort:
+	blr
+	.long	0
+	.byte	0,12,4,1,0x80,18,4,0
+.size	.poly1305_blocks_int,.-.poly1305_blocks_int
+___
+{
+my ($h0,$h1,$h2,$h3,$h4,$t0,$t1) = map("r$_",(6..12));
+
+$code.=<<___;
+.globl	.poly1305_emit_int
+.align	5
+.poly1305_emit_int:
+	lwz	r0,24($ctx)	# is_base2_26
+	lwz	$h0,0($ctx)	# load hash value
+	lwz	$h1,4($ctx)
+	lwz	$h2,8($ctx)
+	lwz	$h3,12($ctx)
+	lwz	$h4,16($ctx)
+	cmplwi	r0,0
+	beq	Lemit_base2_32
+
+	slwi	$t0,$h1,26	# base 2^26 -> base 2^32
+	srwi	$h1,$h1,6
+	slwi	$t1,$h2,20
+	srwi	$h2,$h2,12
+	addc	$h0,$h0,$t0
+	slwi	$t0,$h3,14
+	srwi	$h3,$h3,18
+	adde	$h1,$h1,$t1
+	slwi	$t1,$h4,8
+	srwi	$h4,$h4,24
+	adde	$h2,$h2,$t0
+	adde	$h3,$h3,$t1
+	addze	$h4,$h4
+
+Lemit_base2_32:
+	addic	r0,$h0,5	# compare to modulus
+	addze	r0,$h1
+	addze	r0,$h2
+	addze	r0,$h3
+	addze	r0,$h4
+
+	srwi	r0,r0,2		# see if it carried/borrowed
+	neg	r0,r0
+	andi.	r0,r0,5
+
+	addc	$h0,$h0,r0
+	lwz	r0,0($nonce)
+	addze	$h1,$h1
+	lwz	$t0,4($nonce)
+	addze	$h2,$h2
+	lwz	$t1,8($nonce)
+	addze	$h3,$h3
+	lwz	$h4,12($nonce)
+
+	addc	$h0,$h0,r0	# accumulate nonce
+	adde	$h1,$h1,$t0
+	adde	$h2,$h2,$t1
+	adde	$h3,$h3,$h4
+
+	addi	$ctx,$mac,-1
+	addi	$mac,$mac,7
+
+	stbu	$h0,1($ctx)	# write [little-endian] result
+	srwi	$h0,$h0,8
+	stbu	$h2,1($mac)
+	srwi	$h2,$h2,8
+
+	stbu	$h0,1($ctx)
+	srwi	$h0,$h0,8
+	stbu	$h2,1($mac)
+	srwi	$h2,$h2,8
+
+	stbu	$h0,1($ctx)
+	srwi	$h0,$h0,8
+	stbu	$h2,1($mac)
+	srwi	$h2,$h2,8
+
+	stbu	$h0,1($ctx)
+	stbu	$h2,1($mac)
+
+	stbu	$h1,1($ctx)
+	srwi	$h1,$h1,8
+	stbu	$h3,1($mac)
+	srwi	$h3,$h3,8
+
+	stbu	$h1,1($ctx)
+	srwi	$h1,$h1,8
+	stbu	$h3,1($mac)
+	srwi	$h3,$h3,8
+
+	stbu	$h1,1($ctx)
+	srwi	$h1,$h1,8
+	stbu	$h3,1($mac)
+	srwi	$h3,$h3,8
+
+	stbu	$h1,1($ctx)
+	stbu	$h3,1($mac)
+
+	blr
+	.long	0
+	.byte	0,12,0x14,0,0,0,3,0
+.size	.poly1305_emit_int,.-.poly1305_emit_int
+___
+}							}
+{{{
+########################################################################
+# PowerISA 2.07/VSX section                                            #
+########################################################################
+
+my $LOCALS= 6*$SIZE_T;
+my $VSXFRAME = $LOCALS + 6*$SIZE_T;
+   $VSXFRAME += 128;	# local variables
+   $VSXFRAME += 13*16;	# v20-v31 offload
+
+my $BIG_ENDIAN = ($flavour !~ /le/) ? 4 : 0;
+
+########################################################################
+# Layout of opaque area is following:
+#
+#	unsigned __int32 h[5];		# current hash value base 2^26
+#	unsigned __int32 pad;
+#	unsigned __int32 is_base2_26, pad;
+#	unsigned __int64 r[2];		# key value base 2^64
+#	struct { unsigned __int32 r^2, r^4, r^1, r^3; } r[9];
+#
+# where r^n are base 2^26 digits of powers of multiplier key. There are
+# 5 digits, but last four are interleaved with multiples of 5, totalling
+# in 9 elements: r0, r1, 5*r1, r2, 5*r2, r3, 5*r3, r4, 5*r4. Order of
+# powers is as they appear in register, not memory.
+
+my ($H0, $H1, $H2, $H3, $H4) = map("v$_",(0..4));
+my ($I0, $I1, $I2, $I3, $I4) = map("v$_",(5..9));
+my ($R0, $R1, $S1, $R2, $S2) = map("v$_",(10..14));
+my      ($R3, $S3, $R4, $S4) = ($R1, $S1, $R2, $S2);
+my ($ACC0, $ACC1, $ACC2, $ACC3, $ACC4) = map("v$_",(15..19));
+my ($T0, $T1, $T2, $T3, $T4) = map("v$_",(20..24));
+my ($_26,$_4,$_40,$_14,$mask26,$padbits,$I2perm) = map("v$_",(25..31));
+my ($x00,$x60,$x70,$x10,$x20,$x30,$x40,$x50) = (0, map("r$_",(7,8,27..31)));
+my ($ctx_,$_ctx,$const) = map("r$_",(10..12));
+
+							if ($flavour =~ /64/) {
+###############################################################################
+# setup phase of poly1305_blocks_vsx is different on 32- and 64-bit platforms,
+# but the base 2^26 computational part is same...
+
+my ($h0,$h1,$h2,$d0,$d1,$d2, $r0,$r1,$s1, $t0,$t1) = map("r$_",(6..11,27..31));
+my $mask = "r0";
+
+$code.=<<___;
+.globl	.poly1305_blocks_vsx
+.align	5
+.poly1305_blocks_vsx:
+	lwz	r7,24($ctx)		# is_base2_26
+	cmpldi	$len,128
+	bge	__poly1305_blocks_vsx
+
+	neg	r0,r7			# is_base2_26 as mask
+	lwz	r7,0($ctx)		# load hash base 2^26
+	lwz	r8,4($ctx)
+	lwz	r9,8($ctx)
+	lwz	r10,12($ctx)
+	lwz	r11,16($ctx)
+
+	sldi	r8,r8,26		# base 2^26 -> base 2^64
+	sldi	r12,r9,52
+	add	r7,r7,r8
+	srdi	r9,r9,12
+	sldi	r10,r10,14
+	addc	r7,r7,r12
+	sldi	r8,r11,40
+	adde	r9,r9,r10
+	srdi	r11,r11,24
+	addc	r9,r9,r8
+	addze	r11,r11
+
+	ld	r8,0($ctx)		# load hash base 2^64
+	ld	r10,8($ctx)
+	ld	r12,16($ctx)
+
+	xor	r7,r7,r8		# select between radixes
+	xor	r9,r9,r10
+	xor	r11,r11,r12
+	and	r7,r7,r0
+	and	r9,r9,r0
+	and	r11,r11,r0
+	xor	r7,r7,r8
+	xor	r9,r9,r10
+	xor	r11,r11,r12
+
+	li	r0,0
+	std	r7,0($ctx)		# store hash base 2^64
+	std	r9,8($ctx)
+	std	r11,16($ctx)
+	stw	r0,24($ctx)		# clear is_base2_26
+
+	b	Lpoly1305_blocks
+	.long	0
+	.byte	0,12,0x14,0,0,0,4,0
+.size	.poly1305_blocks_vsx,.-.poly1305_blocks_vsx
+
+.align	5
+__poly1305_mul:
+	mulld	$d0,$h0,$r0		# h0*r0
+	mulhdu	$d1,$h0,$r0
+
+	mulld	$t0,$h1,$s1		# h1*5*r1
+	mulhdu	$t1,$h1,$s1
+	addc	$d0,$d0,$t0
+	adde	$d1,$d1,$t1
+
+	mulld	$t0,$h0,$r1		# h0*r1
+	mulhdu	$d2,$h0,$r1
+	addc	$d1,$d1,$t0
+	addze	$d2,$d2
+
+	mulld	$t0,$h1,$r0		# h1*r0
+	mulhdu	$t1,$h1,$r0
+	addc	$d1,$d1,$t0
+	adde	$d2,$d2,$t1
+
+	mulld	$t0,$h2,$s1		# h2*5*r1
+	mulld	$t1,$h2,$r0		# h2*r0
+	addc	$d1,$d1,$t0
+	adde	$d2,$d2,$t1
+
+	andc	$t0,$d2,$mask		# final reduction step
+	and	$h2,$d2,$mask
+	srdi	$t1,$t0,2
+	add	$t0,$t0,$t1
+	addc	$h0,$d0,$t0
+	addze	$h1,$d1
+	addze	$h2,$h2
+
+	blr
+	.long	0
+	.byte	0,12,0x14,0,0,0,0,0
+.size	__poly1305_mul,.-__poly1305_mul
+
+.align	5
+__poly1305_splat:
+	extrdi	$d0,$h0,26,38
+	extrdi	$d1,$h0,26,12
+	stw	$d0,0x00($t1)
+
+	extrdi	$d2,$h0,12,0
+	slwi	$d0,$d1,2
+	stw	$d1,0x10($t1)
+	add	$d0,$d0,$d1		# * 5
+	stw	$d0,0x20($t1)
+
+	insrdi	$d2,$h1,14,38
+	slwi	$d0,$d2,2
+	stw	$d2,0x30($t1)
+	add	$d0,$d0,$d2		# * 5
+	stw	$d0,0x40($t1)
+
+	extrdi	$d1,$h1,26,24
+	extrdi	$d2,$h1,24,0
+	slwi	$d0,$d1,2
+	stw	$d1,0x50($t1)
+	add	$d0,$d0,$d1		# * 5
+	stw	$d0,0x60($t1)
+
+	insrdi	$d2,$h2,3,37
+	slwi	$d0,$d2,2
+	stw	$d2,0x70($t1)
+	add	$d0,$d0,$d2		# * 5
+	stw	$d0,0x80($t1)
+
+	blr
+	.long	0
+	.byte	0,12,0x14,0,0,0,0,0
+.size	__poly1305_splat,.-__poly1305_splat
+
+.align	5
+__poly1305_blocks_vsx:
+	$STU	$sp,-$VSXFRAME($sp)
+	mflr	r0
+	li	r10,`15+$LOCALS+128`
+	li	r11,`31+$LOCALS+128`
+	mfspr	r12,256
+	stvx	v20,r10,$sp
+	addi	r10,r10,32
+	stvx	v21,r11,$sp
+	addi	r11,r11,32
+	stvx	v22,r10,$sp
+	addi	r10,r10,32
+	stvx	v23,r10,$sp
+	addi	r10,r10,32
+	stvx	v24,r11,$sp
+	addi	r11,r11,32
+	stvx	v25,r10,$sp
+	addi	r10,r10,32
+	stvx	v26,r10,$sp
+	addi	r10,r10,32
+	stvx	v27,r11,$sp
+	addi	r11,r11,32
+	stvx	v28,r10,$sp
+	addi	r10,r10,32
+	stvx	v29,r11,$sp
+	addi	r11,r11,32
+	stvx	v30,r10,$sp
+	stvx	v31,r11,$sp
+	stw	r12,`$VSXFRAME-$SIZE_T*5-4`($sp)# save vrsave
+	li	r12,-1
+	mtspr	256,r12			# preserve all AltiVec registers
+	$PUSH	r27,`$VSXFRAME-$SIZE_T*5`($sp)
+	$PUSH	r28,`$VSXFRAME-$SIZE_T*4`($sp)
+	$PUSH	r29,`$VSXFRAME-$SIZE_T*3`($sp)
+	$PUSH	r30,`$VSXFRAME-$SIZE_T*2`($sp)
+	$PUSH	r31,`$VSXFRAME-$SIZE_T*1`($sp)
+	$PUSH	r0,`$VSXFRAME+$LRSAVE`($sp)
+
+	bl	LPICmeup
+
+	li	$x10,0x10
+	li	$x20,0x20
+	li	$x30,0x30
+	li	$x40,0x40
+	li	$x50,0x50
+	lvx_u	$mask26,$x00,$const
+	lvx_u	$_26,$x10,$const
+	lvx_u	$_40,$x20,$const
+	lvx_u	$I2perm,$x30,$const
+	lvx_u	$padbits,$x40,$const
+
+	cmplwi	r7,0			# is_base2_26?
+	bne	Lskip_init_vsx
+
+	ld	$r0,32($ctx)		# load key base 2^64
+	ld	$r1,40($ctx)
+	srdi	$s1,$r1,2
+	li	$mask,3
+	add	$s1,$s1,$r1		# s1 = r1 + r1>>2
+
+	mr	$h0,$r0			# "calculate" r^1
+	mr	$h1,$r1
+	li	$h2,0
+	addi	$t1,$ctx,`48+(12^$BIG_ENDIAN)`
+	bl	__poly1305_splat
+
+	bl	__poly1305_mul		# caclulate r^2
+	addi	$t1,$ctx,`48+(4^$BIG_ENDIAN)`
+	bl	__poly1305_splat
+
+	bl	__poly1305_mul		# caclulate r^3
+	addi	$t1,$ctx,`48+(8^$BIG_ENDIAN)`
+	bl	__poly1305_splat
+
+	bl	__poly1305_mul		# caclulate r^4
+	addi	$t1,$ctx,`48+(0^$BIG_ENDIAN)`
+	bl	__poly1305_splat
+
+	ld	$h0,0($ctx)		# load hash
+	ld	$h1,8($ctx)
+	ld	$h2,16($ctx)
+
+	extrdi	$d0,$h0,26,38		# base 2^64 -> base 2^26
+	extrdi	$d1,$h0,26,12
+	extrdi	$d2,$h0,12,0
+	mtvrwz	$H0,$d0
+	insrdi	$d2,$h1,14,38
+	mtvrwz	$H1,$d1
+	extrdi	$d1,$h1,26,24
+	mtvrwz	$H2,$d2
+	extrdi	$d2,$h1,24,0
+	mtvrwz	$H3,$d1
+	insrdi	$d2,$h2,3,37
+	mtvrwz	$H4,$d2
+___
+							} else {
+###############################################################################
+# 32-bit initialization
+
+my ($h0,$h1,$h2,$h3,$h4,$t0,$t1) = map("r$_",(7..11,0,12));
+my ($R3,$S3,$R4,$S4)=($I1,$I2,$I3,$I4);
+
+$code.=<<___;
+.globl	.poly1305_blocks_vsx
+.align	5
+.poly1305_blocks_vsx:
+	lwz	r7,24($ctx)		# is_base2_26
+	cmplwi	$len,128
+	bge	__poly1305_blocks_vsx
+	cmplwi	r7,0
+	beq	Lpoly1305_blocks
+
+	lwz	$h0,0($ctx)		# load hash
+	lwz	$h1,4($ctx)
+	lwz	$h2,8($ctx)
+	lwz	$h3,12($ctx)
+	lwz	$h4,16($ctx)
+
+	slwi	$t0,$h1,26		# base 2^26 -> base 2^32
+	srwi	$h1,$h1,6
+	slwi	$t1,$h2,20
+	srwi	$h2,$h2,12
+	addc	$h0,$h0,$t0
+	slwi	$t0,$h3,14
+	srwi	$h3,$h3,18
+	adde	$h1,$h1,$t1
+	slwi	$t1,$h4,8
+	srwi	$h4,$h4,24
+	adde	$h2,$h2,$t0
+	li	$t0,0
+	adde	$h3,$h3,$t1
+	addze	$h4,$h4
+
+	stw	$h0,0($ctx)		# store hash base 2^32
+	stw	$h1,4($ctx)
+	stw	$h2,8($ctx)
+	stw	$h3,12($ctx)
+	stw	$h4,16($ctx)
+	stw	$t0,24($ctx)		# clear is_base2_26
+
+	b	Lpoly1305_blocks
+	.long	0
+	.byte	0,12,0x14,0,0,0,4,0
+.size	.poly1305_blocks_vsx,.-.poly1305_blocks_vsx
+
+.align	5
+__poly1305_mul:
+	vmulouw		$ACC0,$H0,$R0
+	vmulouw		$ACC1,$H1,$R0
+	vmulouw		$ACC2,$H2,$R0
+	vmulouw		$ACC3,$H3,$R0
+	vmulouw		$ACC4,$H4,$R0
+
+	vmulouw		$T0,$H4,$S1
+	vaddudm		$ACC0,$ACC0,$T0
+	vmulouw		$T0,$H0,$R1
+	vaddudm		$ACC1,$ACC1,$T0
+	vmulouw		$T0,$H1,$R1
+	vaddudm		$ACC2,$ACC2,$T0
+	vmulouw		$T0,$H2,$R1
+	vaddudm		$ACC3,$ACC3,$T0
+	vmulouw		$T0,$H3,$R1
+	vaddudm		$ACC4,$ACC4,$T0
+
+	vmulouw		$T0,$H3,$S2
+	vaddudm		$ACC0,$ACC0,$T0
+	vmulouw		$T0,$H4,$S2
+	vaddudm		$ACC1,$ACC1,$T0
+	vmulouw		$T0,$H0,$R2
+	vaddudm		$ACC2,$ACC2,$T0
+	vmulouw		$T0,$H1,$R2
+	vaddudm		$ACC3,$ACC3,$T0
+	vmulouw		$T0,$H2,$R2
+	vaddudm		$ACC4,$ACC4,$T0
+
+	vmulouw		$T0,$H2,$S3
+	vaddudm		$ACC0,$ACC0,$T0
+	vmulouw		$T0,$H3,$S3
+	vaddudm		$ACC1,$ACC1,$T0
+	vmulouw		$T0,$H4,$S3
+	vaddudm		$ACC2,$ACC2,$T0
+	vmulouw		$T0,$H0,$R3
+	vaddudm		$ACC3,$ACC3,$T0
+	vmulouw		$T0,$H1,$R3
+	vaddudm		$ACC4,$ACC4,$T0
+
+	vmulouw		$T0,$H1,$S4
+	vaddudm		$ACC0,$ACC0,$T0
+	vmulouw		$T0,$H2,$S4
+	vaddudm		$ACC1,$ACC1,$T0
+	vmulouw		$T0,$H3,$S4
+	vaddudm		$ACC2,$ACC2,$T0
+	vmulouw		$T0,$H4,$S4
+	vaddudm		$ACC3,$ACC3,$T0
+	vmulouw		$T0,$H0,$R4
+	vaddudm		$ACC4,$ACC4,$T0
+
+	################################################################
+	# lazy reduction
+
+	vspltisb	$T0,2
+	vsrd		$H4,$ACC3,$_26
+	vsrd		$H1,$ACC0,$_26
+	vand		$H3,$ACC3,$mask26
+	vand		$H0,$ACC0,$mask26
+	vaddudm		$H4,$H4,$ACC4		# h3 -> h4
+	vaddudm		$H1,$H1,$ACC1		# h0 -> h1
+
+	vsrd		$ACC4,$H4,$_26
+	vsrd		$ACC1,$H1,$_26
+	vand		$H4,$H4,$mask26
+	vand		$H1,$H1,$mask26
+	vaddudm		$H0,$H0,$ACC4
+	vaddudm		$H2,$ACC2,$ACC1		# h1 -> h2
+
+	vsld		$ACC4,$ACC4,$T0		# <<2
+	vsrd		$ACC2,$H2,$_26
+	vand		$H2,$H2,$mask26
+	vaddudm		$H0,$H0,$ACC4		# h4 -> h0
+	vaddudm		$H3,$H3,$ACC2		# h2 -> h3
+
+	vsrd		$ACC0,$H0,$_26
+	vsrd		$ACC3,$H3,$_26
+	vand		$H0,$H0,$mask26
+	vand		$H3,$H3,$mask26
+	vaddudm		$H1,$H1,$ACC0		# h0 -> h1
+	vaddudm		$H4,$H4,$ACC3		# h3 -> h4
+
+	blr
+	.long	0
+	.byte	0,12,0x14,0,0,0,0,0
+.size	__poly1305_mul,.-__poly1305_mul
+
+.align	5
+__poly1305_blocks_vsx:
+	$STU	$sp,-$VSXFRAME($sp)
+	mflr	r0
+	li	r10,`15+$LOCALS+128`
+	li	r11,`31+$LOCALS+128`
+	mfspr	r12,256
+	stvx	v20,r10,$sp
+	addi	r10,r10,32
+	stvx	v21,r11,$sp
+	addi	r11,r11,32
+	stvx	v22,r10,$sp
+	addi	r10,r10,32
+	stvx	v23,r10,$sp
+	addi	r10,r10,32
+	stvx	v24,r11,$sp
+	addi	r11,r11,32
+	stvx	v25,r10,$sp
+	addi	r10,r10,32
+	stvx	v26,r10,$sp
+	addi	r10,r10,32
+	stvx	v27,r11,$sp
+	addi	r11,r11,32
+	stvx	v28,r10,$sp
+	addi	r10,r10,32
+	stvx	v29,r11,$sp
+	addi	r11,r11,32
+	stvx	v30,r10,$sp
+	stvx	v31,r11,$sp
+	stw	r12,`$VSXFRAME-$SIZE_T*5-4`($sp)# save vrsave
+	li	r12,-1
+	mtspr	256,r12			# preserve all AltiVec registers
+	$PUSH	r27,`$VSXFRAME-$SIZE_T*5`($sp)
+	$PUSH	r28,`$VSXFRAME-$SIZE_T*4`($sp)
+	$PUSH	r29,`$VSXFRAME-$SIZE_T*3`($sp)
+	$PUSH	r30,`$VSXFRAME-$SIZE_T*2`($sp)
+	$PUSH	r31,`$VSXFRAME-$SIZE_T*1`($sp)
+	$PUSH	r0,`$VSXFRAME+$LRSAVE`($sp)
+
+	bl	LPICmeup
+
+	li	$x10,0x10
+	li	$x20,0x20
+	li	$x30,0x30
+	li	$x40,0x40
+	li	$x50,0x50
+	lvx_u	$mask26,$x00,$const
+	lvx_u	$_26,$x10,$const
+	lvx_u	$_40,$x20,$const
+	lvx_u	$I2perm,$x30,$const
+	lvx_u	$padbits,$x40,$const
+
+	cmplwi	r7,0			# is_base2_26?
+	bne	Lskip_init_vsx
+
+	lwz	$h1,32($ctx)		# load key base 2^32
+	lwz	$h2,36($ctx)
+	lwz	$h3,40($ctx)
+	lwz	$h4,44($ctx)
+
+	extrwi	$h0,$h1,26,6		# base 2^32 -> base 2^26
+	extrwi	$h1,$h1,6,0
+	insrwi	$h1,$h2,20,6
+	extrwi	$h2,$h2,12,0
+	insrwi	$h2,$h3,14,6
+	extrwi	$h3,$h3,18,0
+	insrwi	$h3,$h4,8,6
+	extrwi	$h4,$h4,24,0
+
+	mtvrwz	$R0,$h0
+	slwi	$h0,$h1,2
+	mtvrwz	$R1,$h1
+	add	$h1,$h1,$h0
+	mtvrwz	$S1,$h1
+	slwi	$h1,$h2,2
+	mtvrwz	$R2,$h2
+	add	$h2,$h2,$h1
+	mtvrwz	$S2,$h2
+	slwi	$h2,$h3,2
+	mtvrwz	$R3,$h3
+	add	$h3,$h3,$h2
+	mtvrwz	$S3,$h3
+	slwi	$h3,$h4,2
+	mtvrwz	$R4,$h4
+	add	$h4,$h4,$h3
+	mtvrwz	$S4,$h4
+
+	vmr	$H0,$R0
+	vmr	$H1,$R1
+	vmr	$H2,$R2
+	vmr	$H3,$R3
+	vmr	$H4,$R4
+
+	bl	__poly1305_mul		# r^1:- * r^1:-
+
+	vpermdi	$R0,$H0,$R0,0b00
+	vpermdi	$R1,$H1,$R1,0b00
+	vpermdi	$R2,$H2,$R2,0b00
+	vpermdi	$R3,$H3,$R3,0b00
+	vpermdi	$R4,$H4,$R4,0b00
+	vpermdi	$H0,$H0,$H0,0b00
+	vpermdi	$H1,$H1,$H1,0b00
+	vpermdi	$H2,$H2,$H2,0b00
+	vpermdi	$H3,$H3,$H3,0b00
+	vpermdi	$H4,$H4,$H4,0b00
+	vsld	$S1,$R1,$T0		# <<2
+	vsld	$S2,$R2,$T0
+	vsld	$S3,$R3,$T0
+	vsld	$S4,$R4,$T0
+	vaddudm	$S1,$S1,$R1
+	vaddudm	$S2,$S2,$R2
+	vaddudm	$S3,$S3,$R3
+	vaddudm	$S4,$S4,$R4
+
+	bl	__poly1305_mul		# r^2:r^2 * r^2:r^1
+
+	addi	$h0,$ctx,0x60
+	lwz	$h1,0($ctx)		# load hash
+	lwz	$h2,4($ctx)
+	lwz	$h3,8($ctx)
+	lwz	$h4,12($ctx)
+	lwz	$t0,16($ctx)
+
+	vmrgow	$R0,$R0,$H0		# r^2:r^4:r^1:r^3
+	vmrgow	$R1,$R1,$H1
+	vmrgow	$R2,$R2,$H2
+	vmrgow	$R3,$R3,$H3
+	vmrgow	$R4,$R4,$H4
+	vslw	$S1,$R1,$T0		# <<2
+	vslw	$S2,$R2,$T0
+	vslw	$S3,$R3,$T0
+	vslw	$S4,$R4,$T0
+	vadduwm	$S1,$S1,$R1
+	vadduwm	$S2,$S2,$R2
+	vadduwm	$S3,$S3,$R3
+	vadduwm	$S4,$S4,$R4
+
+	stvx_u	$R0,$x30,$ctx
+	stvx_u	$R1,$x40,$ctx
+	stvx_u	$S1,$x50,$ctx
+	stvx_u	$R2,$x00,$h0
+	stvx_u	$S2,$x10,$h0
+	stvx_u	$R3,$x20,$h0
+	stvx_u	$S3,$x30,$h0
+	stvx_u	$R4,$x40,$h0
+	stvx_u	$S4,$x50,$h0
+
+	extrwi	$h0,$h1,26,6		# base 2^32 -> base 2^26
+	extrwi	$h1,$h1,6,0
+	mtvrwz	$H0,$h0
+	insrwi	$h1,$h2,20,6
+	extrwi	$h2,$h2,12,0
+	mtvrwz	$H1,$h1
+	insrwi	$h2,$h3,14,6
+	extrwi	$h3,$h3,18,0
+	mtvrwz	$H2,$h2
+	insrwi	$h3,$h4,8,6
+	extrwi	$h4,$h4,24,0
+	mtvrwz	$H3,$h3
+	insrwi	$h4,$t0,3,5
+	mtvrwz	$H4,$h4
+___
+							}
+$code.=<<___;
+	li	r0,1
+	stw	r0,24($ctx)		# set is_base2_26
+	b	Loaded_vsx
+
+.align	4
+Lskip_init_vsx:
+	li		$x10,4
+	li		$x20,8
+	li		$x30,12
+	li		$x40,16
+	lvwzx_u		$H0,$x00,$ctx
+	lvwzx_u		$H1,$x10,$ctx
+	lvwzx_u		$H2,$x20,$ctx
+	lvwzx_u		$H3,$x30,$ctx
+	lvwzx_u		$H4,$x40,$ctx
+
+Loaded_vsx:
+	li		$x10,0x10
+	li		$x20,0x20
+	li		$x30,0x30
+	li		$x40,0x40
+	li		$x50,0x50
+	li		$x60,0x60
+	li		$x70,0x70
+	addi		$ctx_,$ctx,64		# &ctx->r[1]
+	addi		$_ctx,$sp,`$LOCALS+15`	# &ctx->r[1], r^2:r^4 shadow
+
+	vxor		$T0,$T0,$T0		# ensure second half is zero
+	vpermdi		$H0,$H0,$T0,0b00
+	vpermdi		$H1,$H1,$T0,0b00
+	vpermdi		$H2,$H2,$T0,0b00
+	vpermdi		$H3,$H3,$T0,0b00
+	vpermdi		$H4,$H4,$T0,0b00
+
+	be?lvx_u	$_4,$x50,$const		# byte swap mask
+	lvx_u		$T1,$x00,$inp		# load first input block
+	lvx_u		$T2,$x10,$inp
+	lvx_u		$T3,$x20,$inp
+	lvx_u		$T4,$x30,$inp
+	be?vperm	$T1,$T1,$T1,$_4
+	be?vperm	$T2,$T2,$T2,$_4
+	be?vperm	$T3,$T3,$T3,$_4
+	be?vperm	$T4,$T4,$T4,$_4
+
+	vpermdi		$I0,$T1,$T2,0b00	# smash input to base 2^26
+	vspltisb	$_4,4
+	vperm		$I2,$T1,$T2,$I2perm	# 0x...0e0f0001...1e1f1011
+	vspltisb	$_14,14
+	vpermdi		$I3,$T1,$T2,0b11
+
+	vsrd		$I1,$I0,$_26
+	vsrd		$I2,$I2,$_4
+	vsrd		$I4,$I3,$_40
+	vsrd		$I3,$I3,$_14
+	vand		$I0,$I0,$mask26
+	vand		$I1,$I1,$mask26
+	vand		$I2,$I2,$mask26
+	vand		$I3,$I3,$mask26
+
+	vpermdi		$T1,$T3,$T4,0b00
+	vperm		$T2,$T3,$T4,$I2perm	# 0x...0e0f0001...1e1f1011
+	vpermdi		$T3,$T3,$T4,0b11
+
+	vsrd		$T0,$T1,$_26
+	vsrd		$T2,$T2,$_4
+	vsrd		$T4,$T3,$_40
+	vsrd		$T3,$T3,$_14
+	vand		$T1,$T1,$mask26
+	vand		$T0,$T0,$mask26
+	vand		$T2,$T2,$mask26
+	vand		$T3,$T3,$mask26
+
+	# inp[2]:inp[0]:inp[3]:inp[1]
+	vmrgow		$I4,$T4,$I4
+	vmrgow		$I0,$T1,$I0
+	vmrgow		$I1,$T0,$I1
+	vmrgow		$I2,$T2,$I2
+	vmrgow		$I3,$T3,$I3
+	vor		$I4,$I4,$padbits
+
+	lvx_splt	$R0,$x30,$ctx		# taking lvx_vsplt out of loop
+	lvx_splt	$R1,$x00,$ctx_		# gives ~8% improvement
+	lvx_splt	$S1,$x10,$ctx_
+	lvx_splt	$R2,$x20,$ctx_
+	lvx_splt	$S2,$x30,$ctx_
+	lvx_splt	$T1,$x40,$ctx_
+	lvx_splt	$T2,$x50,$ctx_
+	lvx_splt	$T3,$x60,$ctx_
+	lvx_splt	$T4,$x70,$ctx_
+	stvx		$R1,$x00,$_ctx
+	stvx		$S1,$x10,$_ctx
+	stvx		$R2,$x20,$_ctx
+	stvx		$S2,$x30,$_ctx
+	stvx		$T1,$x40,$_ctx
+	stvx		$T2,$x50,$_ctx
+	stvx		$T3,$x60,$_ctx
+	stvx		$T4,$x70,$_ctx
+
+	addi		$inp,$inp,0x40
+	addi		$const,$const,0x50
+	addi		r0,$len,-64
+	srdi		r0,r0,6
+	mtctr		r0
+	b		Loop_vsx
+
+.align	4
+Loop_vsx:
+	################################################################
+	## ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
+	## ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
+	##   \___________________/
+	##
+	## Note that we start with inp[2:3]*r^2. This is because it
+	## doesn't depend on reduction in previous iteration.
+	################################################################
+	## d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
+	## d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
+	## d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
+	## d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
+	## d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
+
+	vmuleuw		$ACC0,$I0,$R0
+	vmuleuw		$ACC1,$I0,$R1
+	vmuleuw		$ACC2,$I0,$R2
+	vmuleuw		$ACC3,$I1,$R2
+
+	vmuleuw		$T0,$I1,$R0
+	vaddudm		$ACC1,$ACC1,$T0
+	vmuleuw		$T0,$I1,$R1
+	vaddudm		$ACC2,$ACC2,$T0
+	 vmuleuw	$ACC4,$I2,$R2
+	vmuleuw		$T0,$I4,$S1
+	vaddudm		$ACC0,$ACC0,$T0
+	vmuleuw		$T0,$I2,$R1
+	vaddudm		$ACC3,$ACC3,$T0
+	lvx		$S3,$x50,$_ctx
+	vmuleuw		$T0,$I3,$R1
+	vaddudm		$ACC4,$ACC4,$T0
+	lvx		$R3,$x40,$_ctx
+
+	 vaddudm	$H2,$H2,$I2
+	 vaddudm	$H0,$H0,$I0
+	 vaddudm	$H3,$H3,$I3
+	 vaddudm	$H1,$H1,$I1
+	 vaddudm	$H4,$H4,$I4
+
+	vmuleuw		$T0,$I3,$S2
+	vaddudm		$ACC0,$ACC0,$T0
+	vmuleuw		$T0,$I4,$S2
+	vaddudm		$ACC1,$ACC1,$T0
+	vmuleuw		$T0,$I2,$R0
+	vaddudm		$ACC2,$ACC2,$T0
+	vmuleuw		$T0,$I3,$R0
+	vaddudm		$ACC3,$ACC3,$T0
+	lvx		$S4,$x70,$_ctx
+	vmuleuw		$T0,$I4,$R0
+	vaddudm		$ACC4,$ACC4,$T0
+	lvx		$R4,$x60,$_ctx
+
+	vmuleuw		$T0,$I2,$S3
+	vaddudm		$ACC0,$ACC0,$T0
+	vmuleuw		$T0,$I3,$S3
+	vaddudm		$ACC1,$ACC1,$T0
+	vmuleuw		$T0,$I4,$S3
+	vaddudm		$ACC2,$ACC2,$T0
+	vmuleuw		$T0,$I0,$R3
+	vaddudm		$ACC3,$ACC3,$T0
+	vmuleuw		$T0,$I1,$R3
+	vaddudm		$ACC4,$ACC4,$T0
+
+	 be?lvx_u	$_4,$x00,$const		# byte swap mask
+	 lvx_u		$T1,$x00,$inp		# load next input block
+	 lvx_u		$T2,$x10,$inp
+	 lvx_u		$T3,$x20,$inp
+	 lvx_u		$T4,$x30,$inp
+	 be?vperm	$T1,$T1,$T1,$_4
+	 be?vperm	$T2,$T2,$T2,$_4
+	 be?vperm	$T3,$T3,$T3,$_4
+	 be?vperm	$T4,$T4,$T4,$_4
+
+	vmuleuw		$T0,$I1,$S4
+	vaddudm		$ACC0,$ACC0,$T0
+	vmuleuw		$T0,$I2,$S4
+	vaddudm		$ACC1,$ACC1,$T0
+	vmuleuw		$T0,$I3,$S4
+	vaddudm		$ACC2,$ACC2,$T0
+	vmuleuw		$T0,$I4,$S4
+	vaddudm		$ACC3,$ACC3,$T0
+	vmuleuw		$T0,$I0,$R4
+	vaddudm		$ACC4,$ACC4,$T0
+
+	 vpermdi	$I0,$T1,$T2,0b00	# smash input to base 2^26
+	 vspltisb	$_4,4
+	 vperm		$I2,$T1,$T2,$I2perm	# 0x...0e0f0001...1e1f1011
+	 vpermdi	$I3,$T1,$T2,0b11
+
+	# (hash + inp[0:1]) * r^4
+	vmulouw		$T0,$H0,$R0
+	vaddudm		$ACC0,$ACC0,$T0
+	vmulouw		$T0,$H1,$R0
+	vaddudm		$ACC1,$ACC1,$T0
+	vmulouw		$T0,$H2,$R0
+	vaddudm		$ACC2,$ACC2,$T0
+	vmulouw		$T0,$H3,$R0
+	vaddudm		$ACC3,$ACC3,$T0
+	vmulouw		$T0,$H4,$R0
+	vaddudm		$ACC4,$ACC4,$T0
+
+	 vpermdi	$T1,$T3,$T4,0b00
+	 vperm		$T2,$T3,$T4,$I2perm	# 0x...0e0f0001...1e1f1011
+	 vpermdi	$T3,$T3,$T4,0b11
+
+	vmulouw		$T0,$H2,$S3
+	vaddudm		$ACC0,$ACC0,$T0
+	vmulouw		$T0,$H3,$S3
+	vaddudm		$ACC1,$ACC1,$T0
+	vmulouw		$T0,$H4,$S3
+	vaddudm		$ACC2,$ACC2,$T0
+	vmulouw		$T0,$H0,$R3
+	vaddudm		$ACC3,$ACC3,$T0
+	lvx		$S1,$x10,$_ctx
+	vmulouw		$T0,$H1,$R3
+	vaddudm		$ACC4,$ACC4,$T0
+	lvx		$R1,$x00,$_ctx
+
+	 vsrd		$I1,$I0,$_26
+	 vsrd		$I2,$I2,$_4
+	 vsrd		$I4,$I3,$_40
+	 vsrd		$I3,$I3,$_14
+
+	vmulouw		$T0,$H1,$S4
+	vaddudm		$ACC0,$ACC0,$T0
+	vmulouw		$T0,$H2,$S4
+	vaddudm		$ACC1,$ACC1,$T0
+	vmulouw		$T0,$H3,$S4
+	vaddudm		$ACC2,$ACC2,$T0
+	vmulouw		$T0,$H4,$S4
+	vaddudm		$ACC3,$ACC3,$T0
+	lvx		$S2,$x30,$_ctx
+	vmulouw		$T0,$H0,$R4
+	vaddudm		$ACC4,$ACC4,$T0
+	lvx		$R2,$x20,$_ctx
+
+	 vand		$I0,$I0,$mask26
+	 vand		$I1,$I1,$mask26
+	 vand		$I2,$I2,$mask26
+	 vand		$I3,$I3,$mask26
+
+	vmulouw		$T0,$H4,$S1
+	vaddudm		$ACC0,$ACC0,$T0
+	vmulouw		$T0,$H0,$R1
+	vaddudm		$ACC1,$ACC1,$T0
+	vmulouw		$T0,$H1,$R1
+	vaddudm		$ACC2,$ACC2,$T0
+	vmulouw		$T0,$H2,$R1
+	vaddudm		$ACC3,$ACC3,$T0
+	vmulouw		$T0,$H3,$R1
+	vaddudm		$ACC4,$ACC4,$T0
+
+	 vsrd		$T2,$T2,$_4
+	 vsrd		$_4,$T1,$_26
+	 vsrd		$T4,$T3,$_40
+	 vsrd		$T3,$T3,$_14
+
+	vmulouw		$T0,$H3,$S2
+	vaddudm		$ACC0,$ACC0,$T0
+	vmulouw		$T0,$H4,$S2
+	vaddudm		$ACC1,$ACC1,$T0
+	vmulouw		$T0,$H0,$R2
+	vaddudm		$ACC2,$ACC2,$T0
+	vmulouw		$T0,$H1,$R2
+	vaddudm		$ACC3,$ACC3,$T0
+	vmulouw		$T0,$H2,$R2
+	vaddudm		$ACC4,$ACC4,$T0
+
+	 vand		$T1,$T1,$mask26
+	 vand		$_4,$_4,$mask26
+	 vand		$T2,$T2,$mask26
+	 vand		$T3,$T3,$mask26
+
+	################################################################
+	# lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
+	# and P. Schwabe
+
+	vspltisb	$T0,2
+	vsrd		$H4,$ACC3,$_26
+	vsrd		$H1,$ACC0,$_26
+	vand		$H3,$ACC3,$mask26
+	vand		$H0,$ACC0,$mask26
+	vaddudm		$H4,$H4,$ACC4		# h3 -> h4
+	vaddudm		$H1,$H1,$ACC1		# h0 -> h1
+
+	 vmrgow		$I4,$T4,$I4
+	 vmrgow		$I0,$T1,$I0
+	 vmrgow		$I1,$_4,$I1
+	 vmrgow		$I2,$T2,$I2
+	 vmrgow		$I3,$T3,$I3
+	 vor		$I4,$I4,$padbits
+
+	vsrd		$ACC4,$H4,$_26
+	vsrd		$ACC1,$H1,$_26
+	vand		$H4,$H4,$mask26
+	vand		$H1,$H1,$mask26
+	vaddudm		$H0,$H0,$ACC4
+	vaddudm		$H2,$ACC2,$ACC1		# h1 -> h2
+
+	vsld		$ACC4,$ACC4,$T0		# <<2
+	vsrd		$ACC2,$H2,$_26
+	vand		$H2,$H2,$mask26
+	vaddudm		$H0,$H0,$ACC4		# h4 -> h0
+	vaddudm		$H3,$H3,$ACC2		# h2 -> h3
+
+	vsrd		$ACC0,$H0,$_26
+	vsrd		$ACC3,$H3,$_26
+	vand		$H0,$H0,$mask26
+	vand		$H3,$H3,$mask26
+	vaddudm		$H1,$H1,$ACC0		# h0 -> h1
+	vaddudm		$H4,$H4,$ACC3		# h3 -> h4
+
+	addi		$inp,$inp,0x40
+	bdnz		Loop_vsx
+
+	neg		$len,$len
+	andi.		$len,$len,0x30
+	sub		$inp,$inp,$len
+
+	lvx_u		$R0,$x30,$ctx		# load all powers
+	lvx_u		$R1,$x00,$ctx_
+	lvx_u		$S1,$x10,$ctx_
+	lvx_u		$R2,$x20,$ctx_
+	lvx_u		$S2,$x30,$ctx_
+
+Last_vsx:
+	vmuleuw		$ACC0,$I0,$R0
+	vmuleuw		$ACC1,$I1,$R0
+	vmuleuw		$ACC2,$I2,$R0
+	vmuleuw		$ACC3,$I3,$R0
+	vmuleuw		$ACC4,$I4,$R0
+
+	vmuleuw		$T0,$I4,$S1
+	vaddudm		$ACC0,$ACC0,$T0
+	vmuleuw		$T0,$I0,$R1
+	vaddudm		$ACC1,$ACC1,$T0
+	vmuleuw		$T0,$I1,$R1
+	vaddudm		$ACC2,$ACC2,$T0
+	vmuleuw		$T0,$I2,$R1
+	vaddudm		$ACC3,$ACC3,$T0
+	lvx_u		$S3,$x50,$ctx_
+	vmuleuw		$T0,$I3,$R1
+	vaddudm		$ACC4,$ACC4,$T0
+	lvx_u		$R3,$x40,$ctx_
+
+	 vaddudm	$H2,$H2,$I2
+	 vaddudm	$H0,$H0,$I0
+	 vaddudm	$H3,$H3,$I3
+	 vaddudm	$H1,$H1,$I1
+	 vaddudm	$H4,$H4,$I4
+
+	vmuleuw		$T0,$I3,$S2
+	vaddudm		$ACC0,$ACC0,$T0
+	vmuleuw		$T0,$I4,$S2
+	vaddudm		$ACC1,$ACC1,$T0
+	vmuleuw		$T0,$I0,$R2
+	vaddudm		$ACC2,$ACC2,$T0
+	vmuleuw		$T0,$I1,$R2
+	vaddudm		$ACC3,$ACC3,$T0
+	lvx_u		$S4,$x70,$ctx_
+	vmuleuw		$T0,$I2,$R2
+	vaddudm		$ACC4,$ACC4,$T0
+	lvx_u		$R4,$x60,$ctx_
+
+	vmuleuw		$T0,$I2,$S3
+	vaddudm		$ACC0,$ACC0,$T0
+	vmuleuw		$T0,$I3,$S3
+	vaddudm		$ACC1,$ACC1,$T0
+	vmuleuw		$T0,$I4,$S3
+	vaddudm		$ACC2,$ACC2,$T0
+	vmuleuw		$T0,$I0,$R3
+	vaddudm		$ACC3,$ACC3,$T0
+	vmuleuw		$T0,$I1,$R3
+	vaddudm		$ACC4,$ACC4,$T0
+
+	vmuleuw		$T0,$I1,$S4
+	vaddudm		$ACC0,$ACC0,$T0
+	vmuleuw		$T0,$I2,$S4
+	vaddudm		$ACC1,$ACC1,$T0
+	vmuleuw		$T0,$I3,$S4
+	vaddudm		$ACC2,$ACC2,$T0
+	vmuleuw		$T0,$I4,$S4
+	vaddudm		$ACC3,$ACC3,$T0
+	vmuleuw		$T0,$I0,$R4
+	vaddudm		$ACC4,$ACC4,$T0
+
+	# (hash + inp[0:1]) * r^4
+	vmulouw		$T0,$H0,$R0
+	vaddudm		$ACC0,$ACC0,$T0
+	vmulouw		$T0,$H1,$R0
+	vaddudm		$ACC1,$ACC1,$T0
+	vmulouw		$T0,$H2,$R0
+	vaddudm		$ACC2,$ACC2,$T0
+	vmulouw		$T0,$H3,$R0
+	vaddudm		$ACC3,$ACC3,$T0
+	vmulouw		$T0,$H4,$R0
+	vaddudm		$ACC4,$ACC4,$T0
+
+	vmulouw		$T0,$H2,$S3
+	vaddudm		$ACC0,$ACC0,$T0
+	vmulouw		$T0,$H3,$S3
+	vaddudm		$ACC1,$ACC1,$T0
+	vmulouw		$T0,$H4,$S3
+	vaddudm		$ACC2,$ACC2,$T0
+	vmulouw		$T0,$H0,$R3
+	vaddudm		$ACC3,$ACC3,$T0
+	lvx_u		$S1,$x10,$ctx_
+	vmulouw		$T0,$H1,$R3
+	vaddudm		$ACC4,$ACC4,$T0
+	lvx_u		$R1,$x00,$ctx_
+
+	vmulouw		$T0,$H1,$S4
+	vaddudm		$ACC0,$ACC0,$T0
+	vmulouw		$T0,$H2,$S4
+	vaddudm		$ACC1,$ACC1,$T0
+	vmulouw		$T0,$H3,$S4
+	vaddudm		$ACC2,$ACC2,$T0
+	vmulouw		$T0,$H4,$S4
+	vaddudm		$ACC3,$ACC3,$T0
+	lvx_u		$S2,$x30,$ctx_
+	vmulouw		$T0,$H0,$R4
+	vaddudm		$ACC4,$ACC4,$T0
+	lvx_u		$R2,$x20,$ctx_
+
+	vmulouw		$T0,$H4,$S1
+	vaddudm		$ACC0,$ACC0,$T0
+	vmulouw		$T0,$H0,$R1
+	vaddudm		$ACC1,$ACC1,$T0
+	vmulouw		$T0,$H1,$R1
+	vaddudm		$ACC2,$ACC2,$T0
+	vmulouw		$T0,$H2,$R1
+	vaddudm		$ACC3,$ACC3,$T0
+	vmulouw		$T0,$H3,$R1
+	vaddudm		$ACC4,$ACC4,$T0
+
+	vmulouw		$T0,$H3,$S2
+	vaddudm		$ACC0,$ACC0,$T0
+	vmulouw		$T0,$H4,$S2
+	vaddudm		$ACC1,$ACC1,$T0
+	vmulouw		$T0,$H0,$R2
+	vaddudm		$ACC2,$ACC2,$T0
+	vmulouw		$T0,$H1,$R2
+	vaddudm		$ACC3,$ACC3,$T0
+	vmulouw		$T0,$H2,$R2
+	vaddudm		$ACC4,$ACC4,$T0
+
+	################################################################
+	# horizontal addition
+
+	vpermdi		$H0,$ACC0,$ACC0,0b10
+	vpermdi		$H1,$ACC1,$ACC1,0b10
+	vpermdi		$H2,$ACC2,$ACC2,0b10
+	vpermdi		$H3,$ACC3,$ACC3,0b10
+	vpermdi		$H4,$ACC4,$ACC4,0b10
+	vaddudm		$ACC0,$ACC0,$H0
+	vaddudm		$ACC1,$ACC1,$H1
+	vaddudm		$ACC2,$ACC2,$H2
+	vaddudm		$ACC3,$ACC3,$H3
+	vaddudm		$ACC4,$ACC4,$H4
+
+	################################################################
+	# lazy reduction
+
+	vspltisb	$T0,2
+	vsrd		$H4,$ACC3,$_26
+	vsrd		$H1,$ACC0,$_26
+	vand		$H3,$ACC3,$mask26
+	vand		$H0,$ACC0,$mask26
+	vaddudm		$H4,$H4,$ACC4		# h3 -> h4
+	vaddudm		$H1,$H1,$ACC1		# h0 -> h1
+
+	vsrd		$ACC4,$H4,$_26
+	vsrd		$ACC1,$H1,$_26
+	vand		$H4,$H4,$mask26
+	vand		$H1,$H1,$mask26
+	vaddudm		$H0,$H0,$ACC4
+	vaddudm		$H2,$ACC2,$ACC1		# h1 -> h2
+
+	vsld		$ACC4,$ACC4,$T0		# <<2
+	vsrd		$ACC2,$H2,$_26
+	vand		$H2,$H2,$mask26
+	vaddudm		$H0,$H0,$ACC4		# h4 -> h0
+	vaddudm		$H3,$H3,$ACC2		# h2 -> h3
+
+	vsrd		$ACC0,$H0,$_26
+	vsrd		$ACC3,$H3,$_26
+	vand		$H0,$H0,$mask26
+	vand		$H3,$H3,$mask26
+	vaddudm		$H1,$H1,$ACC0		# h0 -> h1
+	vaddudm		$H4,$H4,$ACC3		# h3 -> h4
+
+	beq		Ldone_vsx
+
+	add		r6,$const,$len
+
+	be?lvx_u	$_4,$x00,$const		# byte swap mask
+	lvx_u		$T1,$x00,$inp		# load last partial input block
+	lvx_u		$T2,$x10,$inp
+	lvx_u		$T3,$x20,$inp
+	lvx_u		$T4,$x30,$inp
+	be?vperm	$T1,$T1,$T1,$_4
+	be?vperm	$T2,$T2,$T2,$_4
+	be?vperm	$T3,$T3,$T3,$_4
+	be?vperm	$T4,$T4,$T4,$_4
+
+	vpermdi		$I0,$T1,$T2,0b00	# smash input to base 2^26
+	vspltisb	$_4,4
+	vperm		$I2,$T1,$T2,$I2perm	# 0x...0e0f0001...1e1f1011
+	vpermdi		$I3,$T1,$T2,0b11
+
+	vsrd		$I1,$I0,$_26
+	vsrd		$I2,$I2,$_4
+	vsrd		$I4,$I3,$_40
+	vsrd		$I3,$I3,$_14
+	vand		$I0,$I0,$mask26
+	vand		$I1,$I1,$mask26
+	vand		$I2,$I2,$mask26
+	vand		$I3,$I3,$mask26
+
+	vpermdi		$T0,$T3,$T4,0b00
+	vperm		$T1,$T3,$T4,$I2perm	# 0x...0e0f0001...1e1f1011
+	vpermdi		$T2,$T3,$T4,0b11
+
+	lvx_u		$ACC0,$x00,r6
+	lvx_u		$ACC1,$x30,r6
+
+	vsrd		$T3,$T0,$_26
+	vsrd		$T1,$T1,$_4
+	vsrd		$T4,$T2,$_40
+	vsrd		$T2,$T2,$_14
+	vand		$T0,$T0,$mask26
+	vand		$T3,$T3,$mask26
+	vand		$T1,$T1,$mask26
+	vand		$T2,$T2,$mask26
+
+	# inp[2]:inp[0]:inp[3]:inp[1]
+	vmrgow		$I4,$T4,$I4
+	vmrgow		$I0,$T0,$I0
+	vmrgow		$I1,$T3,$I1
+	vmrgow		$I2,$T1,$I2
+	vmrgow		$I3,$T2,$I3
+	vor		$I4,$I4,$padbits
+
+	vperm		$H0,$H0,$H0,$ACC0	# move hash to right lane
+	vand		$I0,$I0,    $ACC1	# mask redundant input lane[s]
+	vperm		$H1,$H1,$H1,$ACC0
+	vand		$I1,$I1,    $ACC1
+	vperm		$H2,$H2,$H2,$ACC0
+	vand		$I2,$I2,    $ACC1
+	vperm		$H3,$H3,$H3,$ACC0
+	vand		$I3,$I3,    $ACC1
+	vperm		$H4,$H4,$H4,$ACC0
+	vand		$I4,$I4,    $ACC1
+
+	vaddudm		$I0,$I0,$H0		# accumulate hash
+	vxor		$H0,$H0,$H0		# wipe hash value
+	vaddudm		$I1,$I1,$H1
+	vxor		$H1,$H1,$H1
+	vaddudm		$I2,$I2,$H2
+	vxor		$H2,$H2,$H2
+	vaddudm		$I3,$I3,$H3
+	vxor		$H3,$H3,$H3
+	vaddudm		$I4,$I4,$H4
+	vxor		$H4,$H4,$H4
+
+	xor.		$len,$len,$len
+	b		Last_vsx
+
+.align	4
+Ldone_vsx:
+	$POP	r0,`$VSXFRAME+$LRSAVE`($sp)
+	li	$x10,4
+	li	$x20,8
+	li	$x30,12
+	li	$x40,16
+	stvwx_u	$H0,$x00,$ctx			# store hash
+	stvwx_u	$H1,$x10,$ctx
+	stvwx_u	$H2,$x20,$ctx
+	stvwx_u	$H3,$x30,$ctx
+	stvwx_u	$H4,$x40,$ctx
+
+	lwz	r12,`$VSXFRAME-$SIZE_T*5-4`($sp)# pull vrsave
+	mtlr	r0
+	li	r10,`15+$LOCALS+128`
+	li	r11,`31+$LOCALS+128`
+	mtspr	256,r12				# restore vrsave
+	lvx	v20,r10,$sp
+	addi	r10,r10,32
+	lvx	v21,r10,$sp
+	addi	r10,r10,32
+	lvx	v22,r11,$sp
+	addi	r11,r11,32
+	lvx	v23,r10,$sp
+	addi	r10,r10,32
+	lvx	v24,r11,$sp
+	addi	r11,r11,32
+	lvx	v25,r10,$sp
+	addi	r10,r10,32
+	lvx	v26,r11,$sp
+	addi	r11,r11,32
+	lvx	v27,r10,$sp
+	addi	r10,r10,32
+	lvx	v28,r11,$sp
+	addi	r11,r11,32
+	lvx	v29,r10,$sp
+	addi	r10,r10,32
+	lvx	v30,r11,$sp
+	lvx	v31,r10,$sp
+	$POP	r27,`$VSXFRAME-$SIZE_T*5`($sp)
+	$POP	r28,`$VSXFRAME-$SIZE_T*4`($sp)
+	$POP	r29,`$VSXFRAME-$SIZE_T*3`($sp)
+	$POP	r30,`$VSXFRAME-$SIZE_T*2`($sp)
+	$POP	r31,`$VSXFRAME-$SIZE_T*1`($sp)
+	addi	$sp,$sp,$VSXFRAME
+	blr
+	.long	0
+	.byte	0,12,0x04,1,0x80,5,4,0
+	.long	0
+.size	__poly1305_blocks_vsx,.-__poly1305_blocks_vsx
+
+.align	6
+LPICmeup:
+	mflr	r0
+	bcl	20,31,\$+4
+	mflr	$const      # vvvvvv "distance" between . and 1st data entry
+	addi	$const,$const,`64-8`
+	mtlr	r0
+	blr
+	.long	0
+	.byte	0,12,0x14,0,0,0,0,0
+	.space	`64-9*4`
+
+.quad	0x0000000003ffffff,0x0000000003ffffff	# mask26
+.quad	0x000000000000001a,0x000000000000001a	# _26
+.quad	0x0000000000000028,0x0000000000000028	# _40
+.quad	0x000000000e0f0001,0x000000001e1f1011	# I2perm
+.quad	0x0100000001000000,0x0100000001000000	# padbits
+.quad	0x0706050403020100,0x0f0e0d0c0b0a0908	# byte swap for big-endian
+
+.quad	0x0000000000000000,0x0000000004050607	# magic tail masks
+.quad	0x0405060700000000,0x0000000000000000
+.quad	0x0000000000000000,0x0405060700000000
+
+.quad	0xffffffff00000000,0xffffffffffffffff
+.quad	0xffffffff00000000,0xffffffff00000000
+.quad	0x0000000000000000,0xffffffff00000000
+___
+}}}
+$code.=<<___;
+.asciz	"Poly1305 for PPC, CRYPTOGAMS by \@dot-asm"
+___
+
+foreach (split("\n",$code)) {
+	s/\`([^\`]*)\`/eval($1)/ge;
+
+	# instructions prefixed with '?' are endian-specific and need
+	# to be adjusted accordingly...
+	if ($flavour !~ /le$/) {	# big-endian
+	    s/be\?//		or
+	    s/le\?/#le#/
+	} else {			# little-endian
+	    s/le\?//		or
+	    s/be\?/#be#/
+	}
+
+	print $_,"\n";
+}
+close STDOUT;
diff --git a/src/crypto/zinc/poly1305/poly1305.c b/src/crypto/zinc/poly1305/poly1305.c
index 7d373b9..dd2e1a3 100644
--- a/src/crypto/zinc/poly1305/poly1305.c
+++ b/src/crypto/zinc/poly1305/poly1305.c
@@ -14,16 +14,85 @@
 #include <linux/kernel.h>
 #include <linux/string.h>
 #include <linux/module.h>
 #include <linux/init.h>
 
+#if defined(CONFIG_ZINC_ARCH_ARM) || defined(CONFIG_ZINC_ARCH_ARM64) || defined(CONFIG_ZINC_ARCH_PPC32) || defined(CONFIG_ZINC_ARCH_PPC64)
+#if defined(CONFIG_ZINC_ARCH_ARM64) || defined(CONFIG_ZINC_ARCH_PPC64)
+struct poly1305_arch_internal {
+	union {
+		u32 h[5];
+		struct {
+			u64 h0, h1, h2;
+		};
+	};
+	u64 is_base2_26;
+	u64 r[2];
+};
+#elif defined(CONFIG_ZINC_ARCH_ARM) || defined(CONFIG_ZINC_ARCH_PPC32)
+struct poly1305_arch_internal {
+	union {
+		u32 h[5];
+		struct {
+			u64 h0, h1;
+			u32 h2;
+		} __packed;
+	};
+	u32 r[4];
+	u32 is_base2_26;
+};
+#endif
+/* The NEON and AVX code uses base 2^26, while the scalar code uses base 2^64 on 64-bit
+ * and base 2^32 on 32-bit. If we hit the unfortunate situation of using NEON or AVX
+ * and then having to go back to scalar -- because the user is silly and has
+ * called the update function from two separate contexts -- then we need to
+ * convert back to the original base before proceeding. The below function is
+ * written for 64-bit integers, and so we have to swap words at the end on
+ * big-endian 32-bit. It is possible to reason that the initial reduction below
+ * is sufficient given the implementation invariants. However, for an avoidance
+ * of doubt and because this is not performance critical, we do the full
+ * reduction anyway.
+ */
+static void convert_to_base2_64(void *ctx)
+{
+	struct poly1305_arch_internal *state = ctx;
+	u32 cy;
+
+	if (!(IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || IS_ENABLED(CONFIG_AVX)) || !state->is_base2_26)
+		return;
+
+	cy = state->h[0] >> 26; state->h[0] &= 0x3ffffff; state->h[1] += cy;
+	cy = state->h[1] >> 26; state->h[1] &= 0x3ffffff; state->h[2] += cy;
+	cy = state->h[2] >> 26; state->h[2] &= 0x3ffffff; state->h[3] += cy;
+	cy = state->h[3] >> 26; state->h[3] &= 0x3ffffff; state->h[4] += cy;
+	state->h0 = ((u64)state->h[2] << 52) | ((u64)state->h[1] << 26) | state->h[0];
+	state->h1 = ((u64)state->h[4] << 40) | ((u64)state->h[3] << 14) | (state->h[2] >> 12);
+	state->h2 = state->h[4] >> 24;
+	if ((IS_ENABLED(CONFIG_ZINC_ARCH_ARM) || IS_ENABLED(CONFIG_ZINC_ARCH_PPC32)) &&
+	    IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) {
+		state->h0 = rol64(state->h0, 32);
+		state->h1 = rol64(state->h1, 32);
+	}
+#define ULT(a, b) ((a ^ ((a ^ b) | ((a - b) ^ b))) >> (sizeof(a) * 8 - 1))
+	cy = (state->h2 >> 2) + (state->h2 & ~3ULL);
+	state->h2 &= 3;
+	state->h0 += cy;
+	state->h1 += (cy = ULT(state->h0, cy));
+	state->h2 += ULT(state->h1, cy);
+#undef ULT
+	state->is_base2_26 = 0;
+}
+#endif
+
 #if defined(CONFIG_ZINC_ARCH_X86_64)
 #include "poly1305-x86_64-glue.c"
 #elif defined(CONFIG_ZINC_ARCH_ARM) || defined(CONFIG_ZINC_ARCH_ARM64)
 #include "poly1305-arm-glue.c"
 #elif defined(CONFIG_ZINC_ARCH_MIPS) || defined(CONFIG_ZINC_ARCH_MIPS64)
 #include "poly1305-mips-glue.c"
+#elif defined(CONFIG_ZINC_ARCH_PPC32) || defined(CONFIG_ZINC_ARCH_PPC64)
+#include "poly1305-ppc-glue.c"
 #else
 static inline bool poly1305_init_arch(void *ctx,
 				      const u8 key[POLY1305_KEY_SIZE])
 {
 	return false;
-- 
2.20.1

_______________________________________________
WireGuard mailing list
WireGuard@lists.zx2c4.com
https://lists.zx2c4.com/mailman/listinfo/wireguard

^ permalink raw reply	[flat|nested] 5+ messages in thread

end of thread, back to index

Thread overview: 5+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2019-05-11 13:10 [PATCH] [Zinc] Add PowerPC chacha20 implementation from openssl/cryptograms Shawn Landden
2019-05-11 18:03 ` [PATCH 1/2] " Shawn Landden
2019-05-11 18:03   ` [PATCH 2/2] [zinc] add accelerated poly1305 " Shawn Landden
2019-05-13 21:31   ` [PATCH 1/2 v3] [Zinc] Add PowerPC chacha20 implementation " Shawn Landden
2019-05-13 21:31     ` [PATCH 2/2 v3] [zinc] Add PowerPC accelerated poly1305 " Shawn Landden

WireGuard Archive on lore.kernel.org

Archives are clonable:
	git clone --mirror https://lore.kernel.org/wireguard/0 wireguard/git/0.git

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V2 wireguard wireguard/ https://lore.kernel.org/wireguard \
		wireguard@lists.zx2c4.com zx2c4-wireguard@archiver.kernel.org
	public-inbox-index wireguard


Newsgroup available over NNTP:
	nntp://nntp.lore.kernel.org/com.zx2c4.lists.wireguard


AGPL code for this site: git clone https://public-inbox.org/ public-inbox