diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index 1b96be12f71da..87a8f8a253a95 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -1240,59 +1240,49 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } } - match kind { - CallKind::Normal => { - // The callee needs to own the argument memory if we pass it - // by-ref, so make a local copy of non-immediate constants. - if let &mir::Operand::Copy(_) | &mir::Operand::Constant(_) = &arg.node - && let Ref(PlaceValue { llextra: None, .. }) = op.val - { - let tmp = PlaceRef::alloca(bx, op.layout); - bx.lifetime_start(tmp.val.llval, tmp.layout.size); - op.store_with_annotation(bx, tmp); - op.val = Ref(tmp.val); - lifetime_ends_after_call.push((tmp.val.llval, tmp.layout.size)); - } - } - CallKind::Tail => { - if let PassMode::Indirect { on_stack: false, .. } = fn_abi.args[i].mode { - let Some(tmp) = tail_call_temporaries[i].take() else { - span_bug!( - fn_span, - "missing temporary for indirect tail call argument #{i}" - ) - }; + let by_move = if let PassMode::Indirect { on_stack: false, .. } = fn_abi.args[i].mode + && kind == CallKind::Tail + { + // Special logic for tail calls with `PassMode::Indirect { on_stack: false, .. }` arguments. + let Some(tmp) = tail_call_temporaries[i].take() else { + span_bug!(fn_span, "missing temporary for indirect tail call argument #{i}") + }; - let local = self.mir.args_iter().nth(i).unwrap(); + let local = self.mir.args_iter().nth(i).unwrap(); - match &self.locals[local] { - LocalRef::Place(arg) => { - bx.typed_place_copy(arg.val, tmp.val, fn_abi.args[i].layout); - op.val = Ref(arg.val); - } - LocalRef::Operand(arg) => { - let Ref(place_value) = arg.val else { - bug!("only `Ref` should use `PassMode::Indirect`"); - }; - bx.typed_place_copy(place_value, tmp.val, fn_abi.args[i].layout); - op.val = arg.val; - } - LocalRef::UnsizedPlace(_) => { - span_bug!(fn_span, "unsized types are not supported") - } - LocalRef::PendingOperand => { - span_bug!(fn_span, "argument local should not be pending") - } + match &self.locals[local] { + LocalRef::Place(arg) => { + bx.typed_place_copy(arg.val, tmp.val, fn_abi.args[i].layout); + op.val = Ref(arg.val); + } + LocalRef::Operand(arg) => { + let Ref(place_value) = arg.val else { + bug!("only `Ref` should use `PassMode::Indirect`"); }; - - bx.lifetime_end(tmp.val.llval, tmp.layout.size); + bx.typed_place_copy(place_value, tmp.val, fn_abi.args[i].layout); + op.val = arg.val; } - } - } + LocalRef::UnsizedPlace(_) => { + span_bug!(fn_span, "unsized types are not supported") + } + LocalRef::PendingOperand => { + span_bug!(fn_span, "argument local should not be pending") + } + }; + + bx.lifetime_end(tmp.val.llval, tmp.layout.size); + // We have stored the argument for the callee in the corresponding caller's slot. + // Because guaranteed tail calls demand that the caller's signature matches the callee's, + // the argument must be a by-move argument. + true + } else { + matches!(arg.node, mir::Operand::Move(_)) + }; self.codegen_argument( bx, op, + by_move, &mut llargs, &fn_abi.args[i], &mut lifetime_ends_after_call, @@ -1331,6 +1321,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { self.codegen_argument( bx, location, + false, &mut llargs, last_arg, &mut lifetime_ends_after_call, @@ -1649,6 +1640,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { &mut self, bx: &mut Bx, op: OperandRef<'tcx, Bx::Value>, + by_move: bool, llargs: &mut Vec, arg: &ArgAbi<'tcx, Ty<'tcx>>, lifetime_ends_after_call: &mut Vec<(Bx::Value, Size)>, @@ -1703,18 +1695,19 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { _ => (op.immediate_or_packed_pair(bx), arg.layout.align.abi, false), }, Ref(op_place_val) => match arg.mode { - PassMode::Indirect { attrs, .. } => { + PassMode::Indirect { attrs, on_stack, .. } => { + // For `foo(packed.large_field)`, and types with <4 byte alignment on x86, + // alignment requirements may be higher than the type's alignment, so copy + // to a higher-aligned alloca. let required_align = match attrs.pointee_align { Some(pointee_align) => cmp::max(pointee_align, arg.layout.align.abi), None => arg.layout.align.abi, }; - if op_place_val.align < required_align { - // For `foo(packed.large_field)`, and types with <4 byte alignment on x86, - // alignment requirements may be higher than the type's alignment, so copy - // to a higher-aligned alloca. + // Copy to an alloca when the argument is neither by-val nor by-move. + if op_place_val.align < required_align || (!on_stack && !by_move) { let scratch = PlaceValue::alloca(bx, arg.layout.size, required_align); bx.lifetime_start(scratch.llval, arg.layout.size); - bx.typed_place_copy(scratch, op_place_val, op.layout); + op.store_with_annotation(bx, scratch.with_type(arg.layout)); lifetime_ends_after_call.push((scratch.llval, arg.layout.size)); (scratch.llval, scratch.align, true) } else { @@ -1800,6 +1793,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { lifetime_ends_after_call: &mut Vec<(Bx::Value, Size)>, ) -> usize { let tuple = self.codegen_operand(bx, operand); + let by_move = matches!(operand, mir::Operand::Move(_)); // Handle both by-ref and immediate tuples. if let Ref(place_val) = tuple.val { @@ -1810,13 +1804,20 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { for i in 0..tuple.layout.fields.count() { let field_ptr = tuple_ptr.project_field(bx, i); let field = bx.load_operand(field_ptr); - self.codegen_argument(bx, field, llargs, &args[i], lifetime_ends_after_call); + self.codegen_argument( + bx, + field, + by_move, + llargs, + &args[i], + lifetime_ends_after_call, + ); } } else { // If the tuple is immediate, the elements are as well. for i in 0..tuple.layout.fields.count() { let op = tuple.extract_field(self, bx, i); - self.codegen_argument(bx, op, llargs, &args[i], lifetime_ends_after_call); + self.codegen_argument(bx, op, by_move, llargs, &args[i], lifetime_ends_after_call); } } tuple.layout.fields.count() diff --git a/tests/codegen-llvm/call-tmps-lifetime.rs b/tests/codegen-llvm/call-tmps-lifetime.rs index 14c3f0c1e131c..0e62cfc1e604e 100644 --- a/tests/codegen-llvm/call-tmps-lifetime.rs +++ b/tests/codegen-llvm/call-tmps-lifetime.rs @@ -11,19 +11,12 @@ extern crate minicore; use minicore::*; // Const operand. Regression test for #98156. +// Temporary allocas are not required when passing as byval arguments. // // CHECK-LABEL: define void @const_indirect( // CHECK-NEXT: start: -// CHECK-NEXT: [[B:%.*]] = alloca -// CHECK-NEXT: [[A:%.*]] = alloca -// CHECK-NEXT: call void @llvm.lifetime.start.p0({{(i64 4096, )?}}ptr [[A]]) -// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[A]], ptr align 4 {{.*}}, i32 4096, i1 false) -// CHECK-NEXT: call void %h(ptr {{.*}} [[A]]) -// CHECK-NEXT: call void @llvm.lifetime.end.p0({{(i64 4096, )?}}ptr [[A]]) -// CHECK-NEXT: call void @llvm.lifetime.start.p0({{(i64 4096, )?}}ptr [[B]]) -// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[B]], ptr align 4 {{.*}}, i32 4096, i1 false) -// CHECK-NEXT: call void %h(ptr {{.*}} [[B]]) -// CHECK-NEXT: call void @llvm.lifetime.end.p0({{(i64 4096, )?}}ptr [[B]]) +// CHECK-NEXT: call void %h(ptr {{.*}}byval([4096 x i8]){{.*}} [[C:@anon.*]]) +// CHECK-NEXT: call void %h(ptr {{.*}}byval([4096 x i8]){{.*}} [[C]]) #[no_mangle] pub fn const_indirect(h: extern "C" fn([u32; 1024])) { const C: [u32; 1024] = [0; 1024]; diff --git a/tests/codegen-llvm/const-vector.rs b/tests/codegen-llvm/const-vector.rs index f430749234111..25d25d3e8ff35 100644 --- a/tests/codegen-llvm/const-vector.rs +++ b/tests/codegen-llvm/const-vector.rs @@ -74,7 +74,8 @@ pub fn do_call() { // CHECK: call void @test_simd(<4 x i32> test_simd(const { Simd::([2, 4, 6, 8]) }); - // CHECK: call void @test_simd_unaligned(%"minisimd::PackedSimd" %1 + // CHECK: [[UNALIGNED_ARG:%.*]] = load %"minisimd::PackedSimd", ptr @anon{{.*}} + // CHECK-NEXT: call void @test_simd_unaligned(%"minisimd::PackedSimd" [[UNALIGNED_ARG]] test_simd_unaligned(const { Simd::([2, 4, 6]) }); } } diff --git a/tests/codegen-llvm/indirect-bycopy-bymove-byval.rs b/tests/codegen-llvm/indirect-bycopy-bymove-byval.rs new file mode 100644 index 0000000000000..c6a5ec0d162d5 --- /dev/null +++ b/tests/codegen-llvm/indirect-bycopy-bymove-byval.rs @@ -0,0 +1,89 @@ +//! Regression test for issue . +//! Arguments passed indirectly via a hidden pointer must be copied to an alloca, +//! except for by-val or by-move. +//@ compile-flags: -Cno-prepopulate-passes -Copt-level=3 +//@ only-x86_64-unknown-linux-gnu + +#![crate_type = "lib"] +#![feature(fn_traits, stmt_expr_attributes)] +#![expect(unused)] + +#[derive(Copy, Clone)] +struct Thing(usize, usize, usize); + +// The argument of the second call is a by-move argument. + +// CHECK-LABEL: @normal +// CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[normal_V1:%.*]], ptr align 8 %value, i64 24, i1 false) +// CHECK: call void @opaque(ptr{{.*}} [[normal_V1]]) +// CHECK: call void @opaque(ptr{{.*}} %value) +// CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[normal_V3:%.*]], ptr align 8 @anon{{.*}}, i64 24, i1 false) +// CHECK: call void @opaque(ptr{{.*}} [[normal_V3]]) +#[unsafe(no_mangle)] +pub fn normal() { + #[inline(never)] + #[unsafe(no_mangle)] + fn opaque(mut thing: Thing) { + thing.0 = 1; + } + let value = Thing(0, 0, 0); + opaque(value); + opaque(value); + const VALUE: Thing = Thing(0, 0, 0); + opaque(VALUE); +} + +// The argument of the second call is a by-move argument. + +// CHECK-LABEL: @untupled +// CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[untupled_V1:%.*]], ptr align 8 %value, i64 24, i1 false) +// CHECK: call indirect_bycopy_bymove_byval::untupled::{closure#0} +// CHECK-NEXT: call void @{{.*}}(ptr {{.*}}, ptr{{.*}} [[untupled_V1]]) +// CHECK: call indirect_bycopy_bymove_byval::untupled::{closure#1} +// CHECK-NEXT: call void @{{.*}}(ptr {{.*}}, ptr{{.*}} %value) +// CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[untupled_V3:%.*]], ptr align 8 @anon{{.*}}, i64 24, i1 false) +// CHECK: call indirect_bycopy_bymove_byval::untupled::{closure#2} +// CHECK-NEXT: call void @{{.*}}(ptr {{.*}}, ptr{{.*}} [[untupled_V3]]) +#[unsafe(no_mangle)] +pub fn untupled() { + let value = (Thing(0, 0, 0),); + (#[inline(never)] + |mut thing: Thing| { + thing.0 = 1; + }) + .call(value); + (#[inline(never)] + |mut thing: Thing| { + thing.0 = 2; + }) + .call(value); + const VALUE: (Thing,) = (Thing(0, 0, 0),); + (#[inline(never)] + |mut thing: Thing| { + thing.0 = 3; + }) + .call(VALUE); +} + +// All memcpy calls are redundant for byval. + +// CHECK-LABEL: @byval +// CHECK: call void @opaque_byval(ptr{{.*}} byval([24 x i8]){{.*}} %value) +// CHECK: call void @opaque_byval(ptr{{.*}} byval([24 x i8]){{.*}} %value) +// CHECK: call void @opaque_byval(ptr{{.*}} byval([24 x i8]){{.*}} @anon{{.*}}) +#[unsafe(no_mangle)] +pub fn byval() { + #[derive(Copy, Clone)] + #[repr(C)] + struct Thing(usize, usize, usize); + #[inline(never)] + #[unsafe(no_mangle)] + extern "C" fn opaque_byval(mut thing: Thing) { + thing.0 = 1; + } + let value = Thing(0, 0, 0); + opaque_byval(value); + opaque_byval(value); + const VALUE: Thing = Thing(0, 0, 0); + opaque_byval(VALUE); +} diff --git a/tests/ui/moves/bycopy_untupled.rs b/tests/ui/moves/bycopy_untupled.rs new file mode 100644 index 0000000000000..c46c13bf7793e --- /dev/null +++ b/tests/ui/moves/bycopy_untupled.rs @@ -0,0 +1,57 @@ +//! Regression test for issue . +//@ run-pass +//@ revisions: noopt opt +//@[noopt] compile-flags: -C opt-level=0 +//@[opt] compile-flags: -C opt-level=3 + +#![feature(fn_traits, stmt_expr_attributes)] +#![expect(unused)] + +#[derive(Copy, Clone)] +struct Thing { + x: usize, + y: usize, + z: usize, +} + +#[inline(never)] +fn opt_0() { + let value = (Thing { x: 0, y: 0, z: 0 },); + (|mut thing: Thing| { + thing.z = 1; + }) + .call(value); + assert_eq!(value.0.z, 0); +} + +#[inline(never)] +fn opt_3() { + fn with(f: impl FnOnce(Vec)) { + f(Vec::new()) + } + with(|mut v| v.resize(2, 1)); + with(|v| { + if v.len() != 0 { + unreachable!(); + } + }); +} + +#[inline(never)] +fn const_() { + const VALUE: (Thing,) = (Thing { x: 0, y: 0, z: 0 },); + + (#[inline(never)] + |mut thing: Thing| { + thing.z = 1; + std::hint::black_box(&mut thing.z); + assert_eq!(thing.z, 1); + }) + .call(VALUE); +} + +fn main() { + opt_0(); + opt_3(); + const_(); +}