Index: memset.S
===================================================================
RCS file: /cvs/glibc/libc/sysdeps/x86_64/memset.S,v
retrieving revision 1.5
diff -u -r1.5 memset.S
--- memset.S	31 Mar 2005 10:00:13 -0000	1.5
+++ memset.S	14 Dec 2007 00:51:12 -0000
@@ -27,10 +27,6 @@
 /* BEWARE: `#ifdef memset' means that memset is redefined as `bzero' */
 #define BZERO_P (defined memset)
 
-/* This is somehow experimental and could made dependend on the cache
-   size.  */
-#define LARGE $120000
-
         .text
 #if !BZERO_P && defined PIC && !defined NOT_IN_libc
 ENTRY (__memset_chk)
@@ -38,101 +34,284 @@
 	jb	HIDDEN_JUMPTARGET (__chk_fail)
 END (__memset_chk)
 #endif
-ENTRY (memset)
-#if BZERO_P
-	mov	%rsi,%rdx	/* Adjust parameter.  */
-	xorl	%esi,%esi	/* Fill with 0s.  */
-#endif
-	cmp	$0x7,%rdx	/* Check for small length.  */
-	mov	%rdi,%rcx	/* Save ptr as return value.  */
-	jbe	7f
+ENTRY (memset)                		/* (void *, const void*, size_t*/
 
-#if BZERO_P
-	mov	%rsi,%r8	/* Just copy 0.  */
+#ifdef USE_AS_BZERO
+        movq	%rsi, %rdx		/* memset doubles as bzero */
+        xorl	%esi, %esi
 #else
-	/* Populate 8 bit data to full 64-bit.  */
-	movabs	$0x0101010101010101,%r8
-	movzbl	%sil,%eax
-	imul	%rax,%r8
+	movq	$0x0101010101010101, %rcx /* memset proper */
+        movzbq	%sil, %rsi
+        imulq	%rcx, %rsi		/* replicate 8 times */
 #endif
-	test	$0x7,%edi	/* Check for alignment.  */
-	je	2f
 
-	.p2align 4
-1:	/* Align ptr to 8 byte.  */
-	mov	%sil,(%rcx)
-	dec	%rdx
-	inc	%rcx
-	test	$0x7,%ecx
-	jne	1b
-
-2:	/* Check for really large regions.  */
-	mov	%rdx,%rax
-	shr	$0x6,%rax
-	je	4f
-	cmp	LARGE, %rdx
-	jae	11f
+/* Handle tiny blocks. */
 
-	.p2align 4
-3:	/* Copy 64 bytes.  */
-	mov	%r8,(%rcx)
-	mov	%r8,0x8(%rcx)
-	mov	%r8,0x10(%rcx)
-	mov	%r8,0x18(%rcx)
-	mov	%r8,0x20(%rcx)
-	mov	%r8,0x28(%rcx)
-	mov	%r8,0x30(%rcx)
-	mov	%r8,0x38(%rcx)
-	add	$0x40,%rcx
-	dec	%rax
-	jne	3b
-
-4:	/* Copy final bytes.  */
-	and	$0x3f,%edx
-	mov	%rdx,%rax
-	shr	$0x3,%rax
-	je	6f
-
-5:	/* First in chunks of 8 bytes.  */
-	mov	%r8,(%rcx)
-	add	$0x8,%rcx
-	dec	%rax
-	jne	5b
-6:
-	and	$0x7,%edx
-7:
-	test	%rdx,%rdx
-	je	9f
-8:	/* And finally as bytes (up to 7).  */
-	mov	%sil,(%rcx)
-	inc	%rcx
-	dec	%rdx
-	jne	8b
-9:
-#if BZERO_P
-	nop
-#else
-	/* Load result (only if used as memset).  */
-	mov	%rdi,%rax	/* start address of destination is result */
-#endif
-	retq
+L(try1):
+        cmpq	$64, %rdx
+        movq	%rdi, %rax		/* return memory block address (even for bzero ()) */
+        jae	L(1after)
+
+L(1):                                	/* 1-byte */
+        testb	$1, %dl
+        jz      L(1a)
+
+        movb	%sil, (%rdi)
+        incq	%rdi
+
+L(1a):
+        testb	$2, %dl
+        jz      L(1b)
+
+        movw	%si, (%rdi)
+        addq	$2, %rdi
+
+L(1b):
+        testb	$4, %dl
+        jz      L(1c)
+
+        movl	%esi, (%rdi)
+	addq	$4, %rdi
+
+L(1c):
+        testb	$8, %dl
+        jz      L(1d)
+
+        movq	%rsi, (%rdi)
+	addq	$8, %rdi
+
+L(1d):
+        testb	$16, %dl
+        jz      L(1e)
+
+        movq	%rsi,   (%rdi)
+        movq	%rsi, 8 (%rdi)
+	addq	$16, %rdi
+
+L(1e):
+        testb	$32, %dl
+        jz      L(exit)
+
+        movq	%rsi,    (%rdi)
+        movq	%rsi,  8 (%rdi)
+        movq	%rsi, 16 (%rdi)
+        movq	%rsi, 24 (%rdi)
+
+L(exit):
+        rep
+        ret
+
+        .p2align 4
+
+L(1after):
+
+/* Handle small blocks. */
+
+L(32try):
+        cmpq	$512, %rdx
+        jae	L(32after)
+
+L(32):                               	/* 32-byte */
+        movl	%edx, %ecx
+        shrl	$5, %ecx
+        jz      L(32skip)
+
+        .p2align 4
+
+L(32loop):
+        decl	%ecx
+
+        movq	%rsi,    (%rdi)
+        movq	%rsi,  8 (%rdi)
+        movq	%rsi, 16 (%rdi)
+        movq	%rsi, 24 (%rdi)
+
+        leaq	32 (%rdi), %rdi
+
+        jz      L(32skip)
+
+        decl	%ecx
+
+        movq	%rsi,    (%rdi)
+        movq	%rsi,  8 (%rdi)
+        movq	%rsi, 16 (%rdi)
+        movq	%rsi, 24 (%rdi)
+
+        leaq	32 (%rdi), %rdi
+
+        jnz     L(32loop)
+
+        .p2align 4
+
+L(32skip):
+        andl	$31, %edx
+        jnz     L(1)
+
+        rep
+        ret
+
+        .p2align 4
+
+L(32after):
+
+/* Align to natural word alignment. */
+
+L(aligntry):
+        movl	%edi, %ecx              /* align by destination */
+
+        andl	$7, %ecx                /* skip if already aligned */
+        jz      L(alignafter)
+
+L(align):				/* align */
+        leaq	-8 (%rcx, %rdx), %rdx
+        subl	$8, %ecx
+
+        .p2align 4
+
+L(alignloop):
+        incl	%ecx
+
+        movb	%sil, (%rdi)
+        leaq	1 (%rdi), %rdi
+
+        jnz     L(alignloop)
+
+        .p2align 4
+
+L(alignafter):
+
+L(64try):
+	cmpq	$8192, %rdx
+	jae	L(64after)
+
+        .p2align 4
+/* Handle mid-size blocks. */
+L(64):                               	/* 64-byte */
+        movq	%rdx, %rcx
+        shrq	$7, %rcx
+	jz	L(64skip)
+
+        .p2align 4
+
+L(64loop):
+        decq	%rcx
+
+        movq	%rsi,    (%rdi)
+        movq	%rsi,  8 (%rdi)
+        movq	%rsi, 16 (%rdi)
+        movq	%rsi, 24 (%rdi)
+        movq	%rsi, 32 (%rdi)
+        movq	%rsi, 40 (%rdi)
+        movq	%rsi, 48 (%rdi)
+        movq	%rsi, 56 (%rdi)
+        movq	%rsi, 64 (%rdi)
+        movq	%rsi, 72 (%rdi)
+        movq	%rsi, 80 (%rdi)
+        movq	%rsi, 88 (%rdi)
+        movq	%rsi, 96 (%rdi)
+        movq	%rsi, 104 (%rdi)
+        movq	%rsi, 112 (%rdi)
+        movq	%rsi, 120 (%rdi)
+        
+        leaq	128 (%rdi), %rdi
+
+        jnz     L(64loop)
+
+L(64skip):
+	andl	$127, %edx
+	jnz     L(32)
+
+        rep
+        ret
+
+
+        .p2align 4
+
+L(64after):
+
+/* Handle large blocks up to L2 or L3 size. */
+L(fasttry):
+	movq	__x86_64_shared_cache_size_half (%rip), %r8
+        cmpq	%rdx, %r8		/* calculate the lesser of */
+        cmovaq	%rdx, %r8		/* remaining bytes and L2 or L3 */
+	jbe	L(fast)
+
+L(64Ktry):
+	cmp	$65536, %rdx
+	jae	L(64)
+
+        .p2align 4		
+L(fast):				/* microcode */
+	movq	%r8, %rcx
+	andq	$-8, %r8
+
+	shrq	$3, %rcx
+	jz	L(fastskip)
+
+	xchgq	%rax, %rsi
+
+	rep
+	stosq
+
+	xchgq	%rax, %rsi
+
+L(fastskip):
+	subq	%r8, %rdx
+	ja	L(fastafter)
+
+	andl	$7, %edx
+	jnz	L(1)
+
+	rep
+	ret
 
 	.p2align 4
-11:	/* Copy 64 bytes without polluting the cache.  */
-	/* We could use	movntdq    %xmm0,(%rcx) here to further
-	   speed up for large cases but let's not use XMM registers.  */
-	movnti	%r8,(%rcx)
-	movnti  %r8,0x8(%rcx)
-	movnti  %r8,0x10(%rcx)
-	movnti  %r8,0x18(%rcx)
-	movnti  %r8,0x20(%rcx)
-	movnti  %r8,0x28(%rcx)
-	movnti  %r8,0x30(%rcx)
-	movnti  %r8,0x38(%rcx)
-	add	$0x40,%rcx
-	dec	%rax
-	jne	11b
-	jmp	4b
+
+L(fastafter):
+
+/* Handle huge blocks. */
+
+L(NTtry):
+
+L(NT):                               	/* 128-byte */
+        movq	%rdx, %rcx
+        shrq	$7, %rcx
+        jz      L(NTskip)
+
+        .p2align 4
+
+L(NTloop):
+        decq	%rcx
+
+        movntiq	%rsi,     (%rdi)
+        movntiq	%rsi,   8 (%rdi)
+        movntiq	%rsi,  16 (%rdi)
+        movntiq	%rsi,  24 (%rdi)
+        movntiq	%rsi,  32 (%rdi)
+        movntiq	%rsi,  40 (%rdi)
+        movntiq	%rsi,  48 (%rdi)
+        movntiq	%rsi,  56 (%rdi)
+        movntiq	%rsi,  64 (%rdi)
+        movntiq	%rsi,  72 (%rdi)
+        movntiq	%rsi,  80 (%rdi)
+        movntiq	%rsi,  88 (%rdi)
+        movntiq	%rsi,  96 (%rdi)
+        movntiq	%rsi, 104 (%rdi)
+        movntiq	%rsi, 112 (%rdi)
+        movntiq	%rsi, 120 (%rdi)
+
+        leaq	128 (%rdi), %rdi
+
+        jnz     L(NTloop)
+
+        sfence
+
+L(NTskip):
+        andl	$127, %edx
+        jnz     L(32)
+
+        rep
+        ret
 
 END (memset)
 #if !BZERO_P
@@ -144,3 +323,4 @@
 	.section .gnu.warning.__memset_zero_constant_len_parameter
 	.string "memset used with constant zero length parameter; this could be due to transposed parameters"
 #endif
+