--- /dev/null
+/* $Id: VIScsum.S,v 1.1 1997/07/16 10:12:02 jj Exp $
+ * VIScsum.S: High bandwidth IP checksumming utilizing the UltraSparc
+ * Visual Instruction Set.
+ *
+ * Copyright (C) 1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
+ *
+ * Based on older sparc32/sparc64 checksum.S, which is:
+ *
+ * Copyright(C) 1995 Linus Torvalds
+ * Copyright(C) 1995 Miguel de Icaza
+ * Copyright(C) 1996,1997 David S. Miller
+ * derived from:
+ * Linux/Alpha checksum c-code
+ * Linux/ix86 inline checksum assembly
+ * RFC1071 Computing the Internet Checksum (esp. Jacobsons m68k code)
+ * David Mosberger-Tang for optimized reference c-code
+ * BSD4.4 portable checksum routine
+ */
+
+#ifdef __sparc_v9__
+#define STACKOFF 2175
+#else
+#define STACKOFF 64
+#endif
+
+#ifdef __KERNEL__
+#include <asm/head.h>
+#include <asm/asi.h>
+#else
+#define ASI_BLK_P 0xf0
+#define FRPS_FEF 0x04
+#endif
+
+/* Dobrou noc, SunSoft engineers. Spete sladce.
+ * This has a couple of tricks in and those
+ * tricks are UltraLinux trade secrets :))
+ */
+
+#define START_THE_TRICK(fz,f0,f2,f4,f6,f8,f10) \
+ fcmpgt32 %fz, %f0, %g1 /* FPM Group */; \
+ fcmpgt32 %fz, %f2, %g2 /* FPM Group */; \
+ fcmpgt32 %fz, %f4, %g3 /* FPM Group */; \
+ fcmpgt32 %fz, %f6, %g5 /* FPM Group */; \
+ inc %g1 /* IEU0 */; \
+ fcmpgt32 %fz, %f8, %g7 /* FPM Group */; \
+ srl %g1, 1, %g1 /* IEU0 */; \
+ inc %g2 /* IEU1 */; \
+ fcmpgt32 %fz, %f10, %o3 /* FPM Group */; \
+ srl %g2, 1, %g2 /* IEU0 */; \
+ add %o2, %g1, %o2 /* IEU1 */; \
+ add %g3, 1, %g3 /* IEU0 Group */; \
+ srl %g3, 1, %g3 /* IEU0 Group */; \
+ add %o2, %g2, %o2 /* IEU1 */; \
+ inc %g5 /* IEU0 Group */; \
+ add %o2, %g3, %o2 /* IEU1 */;
+
+#define DO_THE_TRICK(O12,O14,f0,f2,f4,f6,f8,f10,f12,f14,F0,F2,F4,F6,F8,F10,F12,F14) \
+ fcmpgt32 %O12, %f12, %o4 /* FPM Group */; \
+ srl %g5, 1, %g5 /* IEU0 */; \
+ inc %g7 /* IEU1 */; \
+ fpadd32 %F0, %f0, %F0 /* FPA */; \
+ fcmpgt32 %O14, %f14, %o5 /* FPM Group */; \
+ srl %g7, 1, %g7 /* IEU0 */; \
+ add %o2, %g5, %o2 /* IEU1 */; \
+ fpadd32 %F2, %f2, %F2 /* FPA */; \
+ inc %o3 /* IEU0 Group */; \
+ add %o2, %g7, %o2 /* IEU1 */; \
+ fcmpgt32 %f0, %F0, %g1 /* FPM Group */; \
+ srl %o3, 1, %o3 /* IEU0 */; \
+ inc %o4 /* IEU1 */; \
+ fpadd32 %F4, %f4, %F4 /* FPA */; \
+ fcmpgt32 %f2, %F2, %g2 /* FPM Group */; \
+ srl %o4, 1, %o4 /* IEU0 */; \
+ add %o2, %o3, %o2 /* IEU1 */; \
+ fpadd32 %F6, %f6, %F6 /* FPA */; \
+ inc %o5 /* IEU0 Group */; \
+ add %o2, %o4, %o2 /* IEU1 */; \
+ fcmpgt32 %f4, %F4, %g3 /* FPM Group */; \
+ srl %o5, 1, %o5 /* IEU0 */; \
+ inc %g1 /* IEU1 */; \
+ fpadd32 %F8, %f8, %F8 /* FPA */; \
+ fcmpgt32 %f6, %F6, %g5 /* FPM Group */; \
+ srl %g1, 1, %g1 /* IEU0 */; \
+ add %o2, %o5, %o2 /* IEU1 */; \
+ fpadd32 %F10, %f10, %F10 /* FPA */; \
+ inc %g2 /* IEU0 Group */; \
+ add %o2, %g1, %o2 /* IEU1 */; \
+ fcmpgt32 %f8, %F8, %g7 /* FPM Group */; \
+ srl %g2, 1, %g2 /* IEU0 */; \
+ inc %g3 /* IEU1 */; \
+ fpadd32 %F12, %f12, %F12 /* FPA */; \
+ fcmpgt32 %f10, %F10, %o3 /* FPM Group */; \
+ srl %g3, 1, %g3 /* IEU0 */; \
+ add %o2, %g2, %o2 /* IEU1 */; \
+ fpadd32 %F14, %f14, %F14 /* FPA */; \
+ inc %g5 /* IEU0 Group */; \
+ add %o2, %g3, %o2 /* IEU1 */;
+
+#define END_THE_TRICK(O12,O14,f0,f2,f4,f6,f8,f10,f12,f14,S0,S1,S2,S3,T0,T1,U0,fz) \
+ fcmpgt32 %O12, %f12, %o4 /* FPM Group */; \
+ srl %g5, 1, %g5 /* IEU0 */; \
+ inc %g7 /* IEU1 */; \
+ fpadd32 %f2, %f0, %S0 /* FPA */; \
+ fcmpgt32 %O14, %f14, %o5 /* FPM Group */; \
+ srl %g7, 1, %g7 /* IEU0 */; \
+ add %o2, %g5, %o2 /* IEU1 */; \
+ fpadd32 %f6, %f4, %S1 /* FPA */; \
+ inc %o3 /* IEU0 Group */; \
+ add %o2, %g7, %o2 /* IEU1 */; \
+ fcmpgt32 %f0, %S0, %g1 /* FPM Group */; \
+ srl %o3, 1, %o3 /* IEU0 */; \
+ inc %o4 /* IEU1 */; \
+ fpadd32 %f10, %f8, %S2 /* FPA */; \
+ fcmpgt32 %f4, %S1, %g2 /* FPM Group */; \
+ srl %o4, 1, %o4 /* IEU0 */; \
+ add %o2, %o3, %o2 /* IEU1 */; \
+ fpadd32 %f14, %f12, %S3 /* FPA */; \
+ inc %o5 /* IEU0 Group */; \
+ add %o2, %o4, %o2 /* IEU1 */; \
+ fzero %fz /* FPA */; \
+ fcmpgt32 %f8, %S2, %g3 /* FPM Group */; \
+ srl %o5, 1, %o5 /* IEU0 */; \
+ inc %g1 /* IEU1 */; \
+ fpadd32 %S0, %S1, %T0 /* FPA */; \
+ fcmpgt32 %f12, %S3, %g5 /* FPM Group */; \
+ srl %g1, 1, %g1 /* IEU0 */; \
+ add %o2, %o5, %o2 /* IEU1 */; \
+ fpadd32 %S2, %S3, %T1 /* FPA */; \
+ inc %g2 /* IEU0 Group */; \
+ add %o2, %g1, %o2 /* IEU1 */; \
+ fcmpgt32 %S0, %T0, %g7 /* FPM Group */; \
+ srl %g2, 1, %g2 /* IEU0 */; \
+ inc %g3 /* IEU1 */; \
+ fcmpgt32 %S2, %T1, %o3 /* FPM Group */; \
+ srl %g3, 1, %g3 /* IEU0 */; \
+ add %o2, %g2, %o2 /* IEU1 */; \
+ inc %g5 /* IEU0 Group */; \
+ add %o2, %g3, %o2 /* IEU1 */; \
+ fcmpgt32 %fz, %f2, %o4 /* FPM Group */; \
+ srl %g5, 1, %g5 /* IEU0 */; \
+ inc %g7 /* IEU1 */; \
+ fpadd32 %T0, %T1, %U0 /* FPA */; \
+ fcmpgt32 %fz, %f6, %o5 /* FPM Group */; \
+ srl %g7, 1, %g7 /* IEU0 */; \
+ add %o2, %g5, %o2 /* IEU1 */; \
+ inc %o3 /* IEU0 Group */; \
+ add %o2, %g7, %o2 /* IEU1 */; \
+ fcmpgt32 %fz, %f10, %g1 /* FPM Group */; \
+ srl %o3, 1, %o3 /* IEU0 */; \
+ inc %o4 /* IEU1 */; \
+ fcmpgt32 %fz, %f14, %g2 /* FPM Group */; \
+ srl %o4, 1, %o4 /* IEU0 */; \
+ add %o2, %o3, %o2 /* IEU1 */; \
+ std %U0, [%sp + STACKOFF] /* Store Group */; \
+ inc %o5 /* IEU0 */; \
+ sub %o2, %o4, %o2 /* IEU1 */; \
+ fcmpgt32 %fz, %S1, %g3 /* FPM Group */; \
+ srl %o5, 1, %o5 /* IEU0 */; \
+ inc %g1 /* IEU1 */; \
+ fcmpgt32 %fz, %S3, %g5 /* FPM Group */; \
+ srl %g1, 1, %g1 /* IEU0 */; \
+ sub %o2, %o5, %o2 /* IEU1 */; \
+ ldx [%sp + STACKOFF], %o5 /* Load Group */; \
+ inc %g2 /* IEU0 */; \
+ sub %o2, %g1, %o2 /* IEU1 */; \
+ fcmpgt32 %fz, %T1, %g7 /* FPM Group */; \
+ srl %g2, 1, %g2 /* IEU0 */; \
+ inc %g3 /* IEU1 */; \
+ fcmpgt32 %T0, %U0, %o3 /* FPM Group */; \
+ srl %g3, 1, %g3 /* IEU0 */; \
+ sub %o2, %g2, %o2 /* IEU1 */; \
+ inc %g5 /* IEU0 Group */; \
+ sub %o2, %g3, %o2 /* IEU1 */; \
+ fcmpgt32 %fz, %U0, %o4 /* FPM Group */; \
+ srl %g5, 1, %g5 /* IEU0 */; \
+ inc %g7 /* IEU1 */; \
+ srl %g7, 1, %g7 /* IEU0 Group */; \
+ sub %o2, %g5, %o2 /* IEU1 */; \
+ inc %o3 /* IEU0 Group */; \
+ sub %o2, %g7, %o2 /* IEU1 */; \
+ srl %o3, 1, %o3 /* IEU0 Group */; \
+ inc %o4 /* IEU1 */; \
+ srl %o4, 1, %o4 /* IEU0 Group */; \
+ add %o2, %o3, %o2 /* IEU1 */; \
+ sub %o2, %o4, %o2 /* IEU0 Group */; \
+ addcc %o2, %o5, %o2 /* IEU1 Group */; \
+ bcs,a,pn %xcc, 33f /* CTI */; \
+ add %o2, 1, %o2 /* IEU0 */; \
+33: /* That's it */;
+
+#define CSUM_LASTCHUNK(offset) \
+ ldx [%o0 - offset - 0x10], %g2; \
+ ldx [%o0 - offset - 0x08], %g3; \
+ addcc %g2, %o2, %o2; \
+ bcs,a,pn %xcc, 31f; \
+ add %o2, 1, %o2; \
+31: addcc %g3, %o2, %o2; \
+ bcs,a,pn %xcc, 32f; \
+ add %o2, 1, %o2; \
+32:
+
+ .text
+ .globl csum_partial
+ .align 32
+csum_partial:
+ andcc %o0, 7, %g0 /* IEU1 Group */
+ be,pt %icc, 4f /* CTI */
+ andcc %o0, 0x38, %g3 /* IEU1 */
+ mov 1, %g5 /* IEU0 Group */
+ cmp %o1, 6 /* IEU1 */
+ bl,pn %icc, 21f /* CTI */
+ andcc %o0, 2, %g0 /* IEU1 Group */
+ be,pt %icc, 1f /* CTI */
+ and %o0, 4, %g7 /* IEU0 */
+ lduh [%o0], %g2 /* Load */
+ sub %o1, 2, %o1 /* IEU0 Group */
+ add %o0, 2, %o0 /* IEU1 */
+ andcc %o0, 4, %g7 /* IEU1 Group */
+ sll %g5, 16, %g5 /* IEU0 */
+ sll %g2, 16, %g2 /* IEU0 Group */
+ addcc %g2, %o2, %o2 /* IEU1 Group (regdep) */
+ bcs,a,pn %icc, 1f /* CTI */
+ add %o2, %g5, %o2 /* IEU0 */
+1: ld [%o0], %g2 /* Load */
+ brz,a,pn %g7, 4f /* CTI+IEU1 Group */
+ and %o0, 0x38, %g3 /* IEU0 */
+ add %o0, 4, %o0 /* IEU0 Group */
+ sub %o1, 4, %o1 /* IEU1 */
+ addcc %g2, %o2, %o2 /* IEU1 Group */
+ bcs,a,pn %icc, 1f /* CTI */
+ add %o2, 1, %o2 /* IEU0 */
+1: and %o0, 0x38, %g3 /* IEU1 Group */
+4: srl %o2, 0, %o2 /* IEU0 Group */
+ mov 0x40, %g1 /* IEU1 */
+ brz,pn %g3, 3f /* CTI+IEU1 Group */
+ sub %g1, %g3, %g1 /* IEU0 */
+ cmp %o1, 56 /* IEU1 Group */
+ blu,pn %icc, 20f /* CTI */
+ andcc %o0, 8, %g0 /* IEU1 Group */
+ be,pn %icc, 1f /* CTI */
+ ldx [%o0], %g2 /* Load */
+ add %o0, 8, %o0 /* IEU0 Group */
+ sub %o1, 8, %o1 /* IEU1 */
+ addcc %g2, %o2, %o2 /* IEU1 Group */
+ bcs,a,pn %xcc, 1f /* CTI */
+ add %o2, 1, %o2 /* IEU0 */
+1: andcc %g1, 0x10, %g0 /* IEU1 Group */
+ be,pn %icc, 2f /* CTI */
+ and %g1, 0x20, %g1 /* IEU0 */
+ ldx [%o0], %g2 /* Load */
+ ldx [%o0+8], %g3 /* Load Group */
+ add %o0, 16, %o0 /* IEU0 */
+ sub %o1, 16, %o1 /* IEU1 */
+ addcc %g2, %o2, %o2 /* IEU1 Group */
+ bcs,a,pn %xcc, 1f /* CTI */
+ add %o2, 1, %o2 /* IEU0 */
+1: addcc %g3, %o2, %o2 /* IEU1 Group */
+ bcs,a,pn %xcc, 2f /* CTI */
+ add %o2, 1, %o2 /* IEU0 */
+2: brz,pn %g1, 3f /* CTI+IEU1 Group */
+ ldx [%o0], %g2 /* Load */
+ ldx [%o0+8], %g3 /* Load Group */
+ ldx [%o0+16], %g5 /* Load Group */
+ ldx [%o0+24], %g7 /* Load Group */
+ add %o0, 32, %o0 /* IEU0 */
+ sub %o1, 32, %o1 /* IEU1 */
+ addcc %g2, %o2, %o2 /* IEU1 Group */
+ bcs,a,pn %xcc, 1f /* CTI */
+ add %o2, 1, %o2 /* IEU0 */
+1: addcc %g3, %o2, %o2 /* IEU1 Group */
+ bcs,a,pn %xcc, 1f /* CTI */
+ add %o2, 1, %o2 /* IEU0 */
+1: addcc %g5, %o2, %o2 /* IEU1 Group */
+ bcs,a,pn %xcc, 1f /* CTI */
+ add %o2, 1, %o2 /* IEU0 */
+1: addcc %g7, %o2, %o2 /* IEU1 Group */
+ bcs,a,pn %xcc, 3f /* CTI */
+ add %o2, 1, %o2 /* IEU0 */
+3: cmp %o1, 0xc0 /* IEU1 Group */
+ blu,pn %icc, 20f /* CTI */
+ sllx %o2, 32, %g1 /* IEU0 */
+ addcc %o2, %g1, %o2 /* IEU1 Group */
+ sub %o1, 0xc0, %o1 /* IEU0 */
+ wr %g0, ASI_BLK_P, %asi /* LSU Group */
+#ifdef __KERNEL__
+ wr %g0, FPRS_FEF, %fprs /* LSU Group */
+#endif
+ membar #StoreLoad /* LSU Group */
+ srlx %o2, 32, %o2 /* IEU0 Group */
+ bcs,a,pn %xcc, 1f /* CTI */
+ add %o2, 1, %o2 /* IEU1 */
+1: andcc %o1, 0x80, %g0 /* IEU1 Group */
+ bne,pn %icc, 7f /* CTI */
+ andcc %o1, 0x40, %g0 /* IEU1 Group */
+ be,pn %icc, 6f /* CTI */
+ fzero %f12 /* FPA */
+ fzero %f14 /* FPA Group */
+ ldda [%o0 + 0x000] %asi, %f16
+ ldda [%o0 + 0x040] %asi, %f32
+ ldda [%o0 + 0x080] %asi, %f48
+ START_THE_TRICK(f12,f16,f18,f20,f22,f24,f26)
+ ba,a,pt %xcc, 3f
+6: sub %o0, 0x40, %o0 /* IEU0 Group */
+ fzero %f28 /* FPA */
+ fzero %f30 /* FPA Group */
+ ldda [%o0 + 0x040] %asi, %f32
+ ldda [%o0 + 0x080] %asi, %f48
+ ldda [%o0 + 0x0c0] %asi, %f0
+ START_THE_TRICK(f28,f32,f34,f36,f38,f40,f42)
+ ba,a,pt %xcc, 4f
+7: bne,pt %icc, 8f /* CTI */
+ fzero %f44 /* FPA */
+ add %o0, 0x40, %o0 /* IEU0 Group */
+ fzero %f60 /* FPA */
+ fzero %f62 /* FPA Group */
+ ldda [%o0 - 0x040] %asi, %f0
+ ldda [%o0 + 0x000] %asi, %f16
+ ldda [%o0 + 0x040] %asi, %f32
+ START_THE_TRICK(f60,f0,f2,f4,f6,f8,f10)
+ ba,a,pt %xcc, 2f
+8: add %o0, 0x80, %o0 /* IEU0 Group */
+ fzero %f46 /* FPA */
+ ldda [%o0 - 0x080] %asi, %f48
+ ldda [%o0 - 0x040] %asi, %f0
+ ldda [%o0 + 0x000] %asi, %f16
+ START_THE_TRICK(f44,f48,f50,f52,f54,f56,f58)
+1: DO_THE_TRICK(f44,f46,f48,f50,f52,f54,f56,f58,f60,f62,f0,f2,f4,f6,f8,f10,f12,f14)
+ ldda [%o0 + 0x040] %asi, %f32
+2: DO_THE_TRICK(f60,f62,f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30)
+ ldda [%o0 + 0x080] %asi, %f48
+3: DO_THE_TRICK(f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46)
+ ldda [%o0 + 0x0c0] %asi, %f0
+4: DO_THE_TRICK(f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,f48,f50,f52,f54,f56,f58,f60,f62)
+ add %o0, 0x100, %o0 /* IEU0 Group */
+ subcc %o1, 0x100, %o1 /* IEU1 */
+ bgeu,a,pt %icc, 1b /* CTI */
+ ldda [%o0 + 0x000] %asi, %f16
+ membar #Sync /* LSU Group */
+ DO_THE_TRICK(f44,f46,f48,f50,f52,f54,f56,f58,f60,f62,f0,f2,f4,f6,f8,f10,f12,f14)
+ END_THE_TRICK(f60,f62,f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30)
+ and %o1, 0x3f, %o1 /* IEU0 Group */
+#ifdef __KERNEL__
+ wr %g0, 0, %fprs /* LSU Group */
+#endif
+20: andcc %o1, 0xf0, %g1 /* IEU1 Group */
+ be,pn %icc, 23f /* CTI */
+ and %o1, 0xf, %o3 /* IEU0 */
+22: rd %pc, %g7 /* LSU Group */
+ sll %g1, 1, %o4 /* IEU0 Group */
+ sub %g7, %o4, %g7 /* IEU0 Group (regdep) */
+ jmpl %g7 + (23f - 22b), %g0 /* CTI Group brk forced */
+ add %o0, %g1, %o0 /* IEU0 */
+ CSUM_LASTCHUNK(0xe0)
+ CSUM_LASTCHUNK(0xd0)
+ CSUM_LASTCHUNK(0xc0)
+ CSUM_LASTCHUNK(0xb0)
+ CSUM_LASTCHUNK(0xa0)
+ CSUM_LASTCHUNK(0x90)
+ CSUM_LASTCHUNK(0x80)
+ CSUM_LASTCHUNK(0x70)
+ CSUM_LASTCHUNK(0x60)
+ CSUM_LASTCHUNK(0x50)
+ CSUM_LASTCHUNK(0x40)
+ CSUM_LASTCHUNK(0x30)
+ CSUM_LASTCHUNK(0x20)
+ CSUM_LASTCHUNK(0x10)
+ CSUM_LASTCHUNK(0x00)
+23: brnz,pn %o3, 26f /* CTI+IEU1 Group */
+24: sllx %o2, 32, %g1 /* IEU0 */
+25: addcc %o2, %g1, %o0 /* IEU1 Group */
+ srlx %o0, 32, %o0 /* IEU0 Group (regdep) */
+ bcs,a,pn %xcc, 1f /* CTI */
+ add %o0, 1, %o0 /* IEU1 */
+1: retl /* CTI Group brk forced */
+ srl %o0, 0, %o0 /* IEU0 */
+26: andcc %o1, 8, %g0 /* IEU1 Group */
+ be,pn %icc, 1f /* CTI */
+ ldx [%o0], %g3 /* Load */
+ add %o0, 8, %o0 /* IEU0 Group */
+ addcc %g3, %o2, %o2 /* IEU1 Group */
+ bcs,a,pn %xcc, 1f /* CTI */
+ add %o2, 1, %o2 /* IEU0 */
+1: andcc %o1, 4, %g0 /* IEU1 Group */
+ be,a,pn %icc, 1f /* CTI */
+ clr %g2 /* IEU0 */
+ ld [%o0], %g2 /* Load */
+ add %o0, 4, %o0 /* IEU0 Group */
+ sllx %g2, 32, %g2 /* IEU0 Group */
+1: andcc %o1, 2, %g0 /* IEU1 */
+ be,a,pn %icc, 1f /* CTI */
+ clr %o4 /* IEU0 Group */
+ lduh [%o0], %o4 /* Load */
+ add %o0, 2, %o0 /* IEU1 */
+ sll %o4, 16, %o4 /* IEU0 Group */
+1: andcc %o1, 1, %g0 /* IEU1 */
+ be,a,pn %icc, 1f /* CTI */
+ clr %o5 /* IEU0 Group */
+ ldub [%o0], %o5 /* Load */
+ sll %o5, 8, %o5 /* IEU0 Group */
+1: or %g2, %o4, %o4 /* IEU1 */
+ or %o5, %o4, %o4 /* IEU0 Group (regdep) */
+ addcc %o4, %o2, %o2 /* IEU1 Group (regdep) */
+ bcs,a,pn %xcc, 1f /* CTI */
+ add %o2, 1, %o2 /* IEU0 */
+1: ba,pt %xcc, 25b /* CTI Group */
+ sllx %o2, 32, %g1 /* IEU0 */
+21: srl %o2, 0, %o2 /* IEU0 Group */
+ cmp %o1, 0 /* IEU1 */
+ be,pn %icc, 24b /* CTI */
+ andcc %o1, 4, %g0 /* IEU1 Group */
+ be,a,pn %icc, 1f /* CTI */
+ clr %g2 /* IEU0 */
+ lduh [%o0], %g3 /* Load */
+ lduh [%o0+2], %g2 /* Load Group */
+ add %o0, 4, %o0 /* IEU0 Group */
+ sllx %g3, 48, %g3 /* IEU0 Group */
+ sllx %g2, 32, %g2 /* IEU0 Group */
+ or %g3, %g2, %g2 /* IEU0 Group */
+1: andcc %o1, 2, %g0 /* IEU1 */
+ be,a,pn %icc, 1f /* CTI */
+ clr %o4 /* IEU0 Group */
+ lduh [%o0], %o4 /* Load */
+ add %o0, 2, %o0 /* IEU1 */
+ sll %o4, 16, %o4 /* IEU0 Group */
+1: andcc %o1, 1, %g0 /* IEU1 */
+ be,a,pn %icc, 1f /* CTI */
+ clr %o5 /* IEU0 Group */
+ ldub [%o0], %o5 /* Load */
+ sll %o5, 8, %o5 /* IEU0 Group */
+1: or %g2, %o4, %o4 /* IEU1 */
+ or %o5, %o4, %o4 /* IEU0 Group (regdep) */
+ addcc %o4, %o2, %o2 /* IEU1 Group (regdep) */
+ bcs,a,pn %xcc, 1f /* CTI */
+ add %o2, 1, %o2 /* IEU0 */
+1: ba,pt %xcc, 25b /* CTI Group */
+ sllx %o2, 32, %g1 /* IEU0 */
* into the accumulated sum. The following is much better.
*
* This should run at max bandwidth for ecache hits, a better
- * technique is to use VIS and fpu operations somehow, but
- * this requires more reasoning on my part...
- *
- * Assuming ecache hits and branches predicted well, this
- * can be expected to run at a rate of 16 cycles per 64-bytes
- * of data summed. (the old code summed 32 bytes in 20
- * cycles, with numerous bubbles and unnecessary stalls)
+ * technique is to use VIS and fpu operations. This is already
+ * done for csum_partial, needs to be written for the copy stuff
+ * still.
*/
-#define CSUM_ECACHE_LOAD(buf, offset, t0, t1, t2, t3, t4, t5, t6, t7) \
- ldx [buf + offset + 0x00], t0; \
- ldx [buf + offset + 0x08], t1; \
- ldx [buf + offset + 0x10], t2; \
- ldx [buf + offset + 0x18], t3; \
- ldx [buf + offset + 0x20], t4; \
- ldx [buf + offset + 0x28], t5; \
- ldx [buf + offset + 0x30], t6; \
- ldx [buf + offset + 0x38], t7; \
- nop; nop; /* THIS IS CRITICAL!!!!!!!!! */
-
-#define CSUM_ECACHE_BLOCK_LDNEXT(buf, offset, sum, t0, t1, t2, t3, t4, t5, t6, t7) \
- addcc sum, t0, sum; \
- bcc,pt %xcc, 11f; \
- ldx [buf + offset + 0x00], t0; \
- add sum, 1, sum; \
-11: addcc sum, t1, sum; \
- bcc,pt %xcc, 12f; \
- ldx [buf + offset + 0x08], t1; \
- add sum, 1, sum; \
-12: addcc sum, t2, sum; \
- bcc,pt %xcc, 13f; \
- ldx [buf + offset + 0x10], t2; \
- add sum, 1, sum; \
-13: addcc sum, t3, sum; \
- bcc,pt %xcc, 14f; \
- ldx [buf + offset + 0x18], t3; \
- add sum, 1, sum; \
-14: addcc sum, t4, sum; \
- bcc,pt %xcc, 15f; \
- ldx [buf + offset + 0x20], t4; \
- add sum, 1, sum; \
-15: addcc sum, t5, sum; \
- bcc,pt %xcc, 16f; \
- ldx [buf + offset + 0x28], t5; \
- add sum, 1, sum; \
-16: addcc sum, t6, sum; \
- bcc,pt %xcc, 17f; \
- ldx [buf + offset + 0x30], t6; \
- add sum, 1, sum; \
-17: addcc sum, t7, sum; \
- bcc,pt %xcc, 18f; \
- ldx [buf + offset + 0x38], t7; \
- add sum, 1, sum; \
-18: nop; nop; /* DO NOT TOUCH! */
-
-#define CSUM_ECACHE_BLOCK(sum, t0, t1, t2, t3, t4, t5, t6, t7) \
- addcc sum, t0, sum; \
- bcs,a,pn %xcc, 21f; \
- add sum, 1, sum; \
-21: addcc sum, t1, sum; \
- bcs,a,pn %xcc, 22f; \
- add sum, 1, sum; \
-22: addcc sum, t2, sum; \
- bcs,a,pn %xcc, 23f; \
- add sum, 1, sum; \
-23: addcc sum, t3, sum; \
- bcs,a,pn %xcc, 24f; \
- add sum, 1, sum; \
-24: addcc sum, t4, sum; \
- bcs,a,pn %xcc, 25f; \
- add sum, 1, sum; \
-25: addcc sum, t5, sum; \
- bcs,a,pn %xcc, 26f; \
- add sum, 1, sum; \
-26: addcc sum, t6, sum; \
- bcs,a,pn %xcc, 27f; \
- add sum, 1, sum; \
-27: addcc sum, t7, sum; \
- bcs,a,pn %xcc, 28f; \
- add sum, 1, sum; \
-28:
-
-#define CSUM_LASTCHUNK(buf, offset, sum, t0, t1) \
- ldx [buf - offset - 0x08], t0; \
- ldx [buf - offset - 0x00], t1; \
- addcc t0, sum, sum; \
- bcs,a,pn %xcc, 31f; \
- add sum, 1, sum; \
-31: addcc t1, sum, sum; \
- bcs,a,pn %xcc, 32f; \
- add sum, 1, sum; \
-32:
.text
- /* Keep this garbage from swiping the icache. */
-csum_partial_end_cruft:
- andcc %o1, 8, %g0 ! IEU1 Group
- be,pn %icc, 1f ! CTI
- and %o1, 4, %g5 ! IEU0
- ldx [%o0 + 0x00], %g2 ! Load Group
- add %o0, 0x8, %o0 ! IEU0
- addcc %g2, %o2, %o2 ! IEU1 Group + 2 bubbles
- bcs,a,pn %xcc, 1f ! CTI
- add %o2, 1, %o2 ! IEU0 4 clocks (mispredict)
-1: andcc %o1, 2, %g0 ! IEU1 Group
- brz,pn %g5, 1f ! CTI Group (needs IEU1)
- clr %g2 ! IEU0
- ld [%o0], %g2 ! Load
- add %o0, 4, %o0 ! IEU0 Group
- sllx %g2, 32, %g2 ! IEU0 Group + 2 bubbles
-1: and %o1, 1, %o1 ! IEU1
- be,pn %icc, 1f ! CTI
- clr %o4 ! IEU0 Group
- lduh [%o0], %o4 ! Load
- add %o0, 2, %o0 ! IEU1
- sll %o4, 16, %o4 ! IEU0 Group + 2 bubbles
-1: brz,pn %o1, 1f ! CTI
- clr %o5 ! IEU1
- ldub [%o0], %o5 ! Load Group
- sll %o5, 8, %o5 ! IEU0 Group + 2 bubbles
-1: or %g2, %o4, %o4 ! IEU1
- or %o5, %o4, %o4 ! IEU0 Group
- addcc %o4, %o2, %o2 ! IEU1 Group (regdep)
- bcc,pt %xcc, cfold ! CTI
- sethi %uhi(PAGE_OFFSET), %g4 ! IEU0
-1: b,pt %xcc, cfold ! CTI Group
- add %o2, 1, %o2 ! IEU0
-
-csum_partial_fixit:
- bl,pn %icc, cpte ! CTI Group
- and %o1, 0xf, %o3 ! IEU0
- andcc %o0, 0x2, %g0 ! IEU1
- be,pn %icc, 1f ! CTI Group
- and %o0, 0x4, %g7 ! IEU0
- lduh [%o0 + 0x00], %g2 ! Load
- sub %o1, 2, %o1 ! IEU0 Group
- addcc %o0, 2, %o0 ! IEU1
- and %o0, 0x4, %g7 ! IEU0 Group
- sll %g2, 16, %g2 ! IEU0 Group (no load stall)
- addcc %g2, %o2, %o2 ! IEU1 Group (regdep)
- bcc,pt %icc, 0f ! CTI
- andn %o1, 0xff, %o3 ! IEU0
- srl %o2, 16, %g2 ! IEU0 Group
- b,pt %xcc, 9f ! CTI
- add %g2, 1, %g2 ! IEU1
-0: srl %o2, 16, %g2 ! IEU0 Group 8-(
-9: sll %o2, 16, %o2 ! IEU0 Group 8-(
- sll %g2, 16, %g3 ! IEU0 Group 8-(
- srl %o2, 16, %o2 ! IEU0 Group 8-(
- or %g3, %o2, %o2 ! IEU1
-1: brnz,pt %g7, 2f ! CTI Group
- sub %o1, 4, %o1 ! IEU0
- b,pt %xcc, csum_partial_aligned ! CTI Group
- add %o1, 4, %o1 ! IEU0
-2: ld [%o0 + 0x00], %g2 ! Load Group
- add %o0, 4, %o0 ! IEU0
- andn %o1, 0xff, %o3 ! IEU1
- addcc %g2, %o2, %o2 ! IEU1 Group + 2 bubbles
- bcc,pt %xcc, csum_partial_aligned ! CTI
- nop ! IEU0
- b,pt %xcc, csum_partial_aligned ! CTI Group
- add %o2, 1, %o2 ! IEU0
-
- .align 32
- .globl csum_partial
-csum_partial: /* %o0=buf, %o1=len, %o2=sum */
- andcc %o0, 0x7, %g0 ! IEU1 Group
- srl %o1, 0, %o1 ! IEU0
- srl %o2, 0, %o2 ! IEU0 Group
- be,pt %icc, csum_partial_aligned ! CTI
- andn %o1, 0xff, %o3 ! IEU1
- b,pt %xcc, csum_partial_fixit ! CTI Group
- cmp %o1, 6 ! IEU0
- nop
-csum_partial_aligned:
- brz,pt %o3, 3f ! CTI Group
- and %o1, 0xf0, %g1 ! IEU0
-5: CSUM_ECACHE_LOAD( %o0, 0x000, %o4, %o5, %g2, %g3, %g4, %g5, %g1, %g7)
- CSUM_ECACHE_BLOCK_LDNEXT(%o0, 0x040, %o2, %o4, %o5, %g2, %g3, %g4, %g5, %g1, %g7)
- CSUM_ECACHE_BLOCK_LDNEXT(%o0, 0x080, %o2, %o4, %o5, %g2, %g3, %g4, %g5, %g1, %g7)
- CSUM_ECACHE_BLOCK_LDNEXT(%o0, 0x0c0, %o2, %o4, %o5, %g2, %g3, %g4, %g5, %g1, %g7)
- CSUM_ECACHE_BLOCK( %o2, %o4, %o5, %g2, %g3, %g4, %g5, %g1, %g7)
- subcc %o3, 256, %o3 ! IEU1 Group
- bne,pt %icc, 5b ! CTI
- add %o0, 256, %o0 ! IEU0
- and %o1, 0xf0, %g1 ! IEU0 Group
-3: brz,pn %g1, cpte ! CTI
- and %o1, 0xf, %o3 ! IEU1 Group
-10: rd %pc, %g7 ! LSU Group + 4 clocks
- sll %g1, 1, %o4 ! IEU0 Group
- sub %g7, %o4, %g7 ! IEU1
- jmp %g7 + %lo(cpte - 10b) ! CTI Group brk forced
- add %o0, %g1, %o0 ! IEU0
-cptbl: CSUM_LASTCHUNK(%o0, 0xe8, %o2, %g2, %g3)
- CSUM_LASTCHUNK(%o0, 0xd8, %o2, %g2, %g3)
- CSUM_LASTCHUNK(%o0, 0xc8, %o2, %g2, %g3)
- CSUM_LASTCHUNK(%o0, 0xb8, %o2, %g2, %g3)
- CSUM_LASTCHUNK(%o0, 0xa8, %o2, %g2, %g3)
- CSUM_LASTCHUNK(%o0, 0x98, %o2, %g2, %g3)
- CSUM_LASTCHUNK(%o0, 0x88, %o2, %g2, %g3)
- CSUM_LASTCHUNK(%o0, 0x78, %o2, %g2, %g3)
- CSUM_LASTCHUNK(%o0, 0x68, %o2, %g2, %g3)
- CSUM_LASTCHUNK(%o0, 0x58, %o2, %g2, %g3)
- CSUM_LASTCHUNK(%o0, 0x48, %o2, %g2, %g3)
- CSUM_LASTCHUNK(%o0, 0x38, %o2, %g2, %g3)
- CSUM_LASTCHUNK(%o0, 0x28, %o2, %g2, %g3)
- CSUM_LASTCHUNK(%o0, 0x18, %o2, %g2, %g3)
- CSUM_LASTCHUNK(%o0, 0x08, %o2, %g2, %g3)
-cpte: brnz,pn %o3, csum_partial_end_cruft ! CTI Group
- sethi %uhi(PAGE_OFFSET), %g4 ! IEU0
-cfold: sllx %o2, 32, %o0 ! IEU0 Group
- addcc %o2, %o0, %o0 ! IEU1 Group (regdep)
- srlx %o0, 32, %o0 ! IEU0 Group (regdep)
- bcs,a,pn %xcc, 1f ! CTI
- add %o0, 1, %o0 ! IEU1 4 clocks (mispredict)
-1: retl ! CTI Group brk forced
- sllx %g4, 32, %g4 ! IEU0 Group
-
.globl __csum_partial_copy_start, __csum_partial_copy_end
__csum_partial_copy_start:
projects / history.git / commitdiff