performance regress in masked `u128` field updates on x86_64

[camc314]

Code

I tried this code:

const LOW_MASK: u128 = 0xFFFF_FFFF;

#[repr(transparent)]
pub struct Token(u128);

impl Token {
    #[inline(always)]
    fn set_start_mask(&mut self, start: u32) {
        self.0 &= !LOW_MASK;
        self.0 |= start as u128;
    }

    #[inline(always)]
    fn set_end_mask(&mut self, end: u32) {
        self.0 &= !(LOW_MASK << 32);
        self.0 |= (end as u128) << 32;
    }
    
    #[inline(always)]
    fn set_start_ptr(&mut self, start: u32) {
        #[cfg(target_endian = "little")]
        unsafe {
            core::ptr::from_mut(&mut self.0).cast::<u32>().write(start);
        }
    
        #[cfg(not(target_endian = "little"))]
        {
            self.0 &= !LOW_MASK;
            self.0 |= start as u128;
        }
    }


    #[inline(always)]
    fn set_end_ptr(&mut self, end: u32) {
        #[cfg(target_endian = "little")]
        unsafe {
            core::ptr::from_mut(&mut self.0).cast::<u32>().add(1).write(end);
        }
    
        #[cfg(not(target_endian = "little"))]
        {
            token.0 &= !(LOW_MASK << 32);
            token.0 |= (end as u128) << 32;
        }
    }
}

#[unsafe(no_mangle)]
pub extern "C" fn hot_start_mask(token: &mut Token, start: u32) -> u128 {
    token.set_start_mask(start);
    token.0
}

#[unsafe(no_mangle)]
pub extern "C" fn hot_end_mask(token: &mut Token, end: u32) -> u128 {
    token.set_end_mask(end);
    token.0
}

#[unsafe(no_mangle)]
pub extern "C" fn hot_start_ptr(token: &mut Token, start: u32) -> u128 {
    token.set_start_ptr(start);
    token.0
}

#[unsafe(no_mangle)]
pub extern "C" fn hot_end_ptr(token: &mut Token, end: u32) -> u128 {
    token.set_end_ptr(end);
    token.0
}

I expected to see this happen:

hot_end_mask:
        mov     rdx, qword ptr [rdi + 8]
        mov     eax, dword ptr [rdi]
        shl     rsi, 32
        or      rax, rsi
        mov     qword ptr [rdi], rax
        ret

hot_end_ptr:
        mov     dword ptr [rdi + 4], esi
        mov     rdx, qword ptr [rdi + 8]
        mov     rax, qword ptr [rdi]
        ret

hot_start_mask:
        mov     rdx, qword ptr [rdi + 8]
        mov     ecx, dword ptr [rdi + 4]
        shl     rcx, 32
        mov     eax, esi
        or      rax, rcx
        mov     qword ptr [rdi], rax
        ret

hot_start_ptr:
        mov     dword ptr [rdi], esi
        mov     rdx, qword ptr [rdi + 8]
        mov     rax, qword ptr [rdi]
        ret

Instead, this happened:

hot_end_mask:
        pshufd  xmm0, xmmword ptr [rdi], 238
        mov     eax, dword ptr [rdi]
        movq    rdx, xmm0
        shl     rsi, 32
        or      rax, rsi
        movq    qword ptr [rdi + 8], xmm0
        mov     qword ptr [rdi], rax
        ret

hot_end_ptr:
        mov     dword ptr [rdi + 4], esi
        mov     rdx, qword ptr [rdi + 8]
        mov     rax, qword ptr [rdi]
        ret

.LCPI2_0:
        .long   0
        .long   4294967295
        .long   4294967295
        .long   4294967295
hot_start_mask:
        movdqa  xmm0, xmmword ptr [rdi]
        pshufd  xmm1, xmm0, 238
        pand    xmm0, xmmword ptr [rip + .LCPI2_0]
        movq    rdx, xmm1
        movq    rcx, xmm0
        mov     eax, esi
        or      rax, rcx
        movq    qword ptr [rdi + 8], xmm1
        mov     qword ptr [rdi], rax
        ret

hot_start_ptr:
        mov     dword ptr [rdi], esi
        mov     rdx, qword ptr [rdi + 8]
        mov     rax, qword ptr [rdi]
        ret

Version it worked on

It most recently worked on: 1.94

Version with regression

rustc --version --verbose:

1.95

References: oxc-project/oxc#21509

[camc314]

Appears to have regressed in #150722

the llvm IR (from godbolt) it the same in both versions:

define noundef i128 @hot_end_mask(ptr noalias noundef align 16 captures(none) dereferenceable(16) %token, i32 noundef %end) unnamed_addr {
start:
  %0 = load i128, ptr %token, align 16
  %1 = and i128 %0, -18446744069414584321
  %_4 = zext i32 %end to i128
  %_3 = shl nuw nsw i128 %_4, 32
  %2 = or disjoint i128 %1, %_3
  store i128 %2, ptr %token, align 16
  ret i128 %2
}

define noundef i128 @hot_end_ptr(ptr noalias noundef align 16 captures(none) dereferenceable(16) initializes((4, 8)) %token, i32 noundef %end) unnamed_addr {
start:
  %self = getelementptr inbounds nuw i8, ptr %token, i64 4
  store i32 %end, ptr %self, align 4
  %_0 = load i128, ptr %token, align 16
  ret i128 %_0
}

define noundef i128 @hot_start_mask(ptr noalias noundef align 16 captures(none) dereferenceable(16) %token, i32 noundef %start1) unnamed_addr {
start:
  %0 = load i128, ptr %token, align 16
  %1 = and i128 %0, -4294967296
  %_3 = zext i32 %start1 to i128
  %2 = or disjoint i128 %1, %_3
  store i128 %2, ptr %token, align 16
  ret i128 %2
}

define noundef i128 @hot_start_ptr(ptr noalias noundef align 16 captures(none) dereferenceable(16) initializes((0, 4)) %token, i32 noundef %start1) unnamed_addr {
start:
  store i32 %start1, ptr %token, align 16
  %_0 = load i128, ptr %token, align 16
  ret i128 %_0
}

[apiraino]

here an estimate of the perf regression

@rustbot label A-LLVM

[folkertdev]

Yeah it's clearly an LLVM issue https://godbolt.org/z/PM89vMb9d