From 2afc9052c0f716aa517112e93bcbf1e2cc70f25d Mon Sep 17 00:00:00 2001 From: ltdk Date: Sat, 29 Nov 2025 16:17:12 -0500 Subject: [PATCH 1/4] Replace RawTable inside HashMap with HashTable --- src/external_trait_impls/rayon/map.rs | 14 ++-- src/map.rs | 107 +++++++++++++------------- src/raw_entry.rs | 8 +- src/rustc_entry.rs | 6 +- src/set.rs | 4 +- src/table.rs | 2 + 6 files changed, 73 insertions(+), 68 deletions(-) diff --git a/src/external_trait_impls/rayon/map.rs b/src/external_trait_impls/rayon/map.rs index 1a33c5ff30..333e8be169 100644 --- a/src/external_trait_impls/rayon/map.rs +++ b/src/external_trait_impls/rayon/map.rs @@ -288,7 +288,7 @@ impl HashMap { #[cfg_attr(feature = "inline-more", inline)] pub fn par_keys(&self) -> ParKeys<'_, K, V> { ParKeys { - inner: unsafe { self.table.par_iter() }, + inner: unsafe { self.table.raw.par_iter() }, marker: PhantomData, } } @@ -297,7 +297,7 @@ impl HashMap { #[cfg_attr(feature = "inline-more", inline)] pub fn par_values(&self) -> ParValues<'_, K, V> { ParValues { - inner: unsafe { self.table.par_iter() }, + inner: unsafe { self.table.raw.par_iter() }, marker: PhantomData, } } @@ -308,7 +308,7 @@ impl HashMap { #[cfg_attr(feature = "inline-more", inline)] pub fn par_values_mut(&mut self) -> ParValuesMut<'_, K, V> { ParValuesMut { - inner: unsafe { self.table.par_iter() }, + inner: unsafe { self.table.raw.par_iter() }, marker: PhantomData, } } @@ -318,7 +318,7 @@ impl HashMap { #[cfg_attr(feature = "inline-more", inline)] pub fn par_drain(&mut self) -> ParDrain<'_, K, V, A> { ParDrain { - inner: self.table.par_drain(), + inner: self.table.raw.par_drain(), } } } @@ -349,7 +349,7 @@ impl IntoParallelIterator for HashMap< #[cfg_attr(feature = "inline-more", inline)] fn into_par_iter(self) -> Self::Iter { IntoParIter { - inner: self.table.into_par_iter(), + inner: self.table.raw.into_par_iter(), } } } @@ -361,7 +361,7 @@ impl<'a, K: Sync, V: Sync, S, A: Allocator> IntoParallelIterator for &'a HashMap #[cfg_attr(feature = "inline-more", inline)] fn into_par_iter(self) -> Self::Iter { ParIter { - inner: unsafe { self.table.par_iter() }, + inner: unsafe { self.table.raw.par_iter() }, marker: PhantomData, } } @@ -374,7 +374,7 @@ impl<'a, K: Sync, V: Send, S, A: Allocator> IntoParallelIterator for &'a mut Has #[cfg_attr(feature = "inline-more", inline)] fn into_par_iter(self) -> Self::Iter { ParIterMut { - inner: unsafe { self.table.par_iter() }, + inner: unsafe { self.table.raw.par_iter() }, marker: PhantomData, } } diff --git a/src/map.rs b/src/map.rs index 2154ec3e18..ef4303d13a 100644 --- a/src/map.rs +++ b/src/map.rs @@ -1,6 +1,6 @@ use crate::alloc::{Allocator, Global}; -use crate::raw::{Bucket, RawDrain, RawExtractIf, RawIntoIter, RawIter, RawTable}; -use crate::{DefaultHashBuilder, Equivalent, TryReserveError}; +use crate::raw::{Bucket, RawDrain, RawExtractIf, RawIntoIter, RawIter}; +use crate::{DefaultHashBuilder, Equivalent, HashTable, TryReserveError}; use core::borrow::Borrow; use core::fmt::{self, Debug}; use core::hash::{BuildHasher, Hash}; @@ -181,7 +181,7 @@ pub use crate::raw_entry::*; /// ``` pub struct HashMap { pub(crate) hash_builder: S, - pub(crate) table: RawTable<(K, V), A>, + pub(crate) table: HashTable<(K, V), A>, } impl Clone for HashMap { @@ -201,7 +201,7 @@ impl Clone for HashMap(hash_builder: &S) -> impl Fn(&(Q, V)) -> u64 + '_ where @@ -212,7 +212,7 @@ where } /// Ensures that a single closure type across uses of this which, in turn prevents multiple -/// instances of any functions like `RawTable::reserve` from being generated +/// instances of any functions like `HashTable::reserve` from being generated #[cfg_attr(feature = "inline-more", inline)] pub(crate) fn equivalent_key(k: &Q) -> impl Fn(&(K, V)) -> bool + '_ where @@ -222,7 +222,7 @@ where } /// Ensures that a single closure type across uses of this which, in turn prevents multiple -/// instances of any functions like `RawTable::reserve` from being generated +/// instances of any functions like `HashTable::reserve` from being generated #[cfg_attr(feature = "inline-more", inline)] #[cfg(feature = "raw-entry")] pub(crate) fn equivalent(k: &Q) -> impl Fn(&K) -> bool + '_ @@ -434,7 +434,7 @@ impl HashMap { pub const fn with_hasher(hash_builder: S) -> Self { Self { hash_builder, - table: RawTable::new(), + table: HashTable::new(), } } @@ -474,7 +474,7 @@ impl HashMap { pub fn with_capacity_and_hasher(capacity: usize, hash_builder: S) -> Self { Self { hash_builder, - table: RawTable::with_capacity(capacity), + table: HashTable::with_capacity(capacity), } } } @@ -483,7 +483,7 @@ impl HashMap { /// Returns a reference to the underlying allocator. #[inline] pub fn allocator(&self) -> &A { - self.table.allocator() + self.table.raw.allocator() } /// Creates an empty `HashMap` which will use the given hash builder to hash @@ -517,7 +517,7 @@ impl HashMap { pub const fn with_hasher_in(hash_builder: S, alloc: A) -> Self { Self { hash_builder, - table: RawTable::new_in(alloc), + table: HashTable::new_in(alloc), } } @@ -551,7 +551,7 @@ impl HashMap { pub fn with_capacity_and_hasher_in(capacity: usize, hash_builder: S, alloc: A) -> Self { Self { hash_builder, - table: RawTable::with_capacity_in(capacity, alloc), + table: HashTable::with_capacity_in(capacity, alloc), } } @@ -587,7 +587,7 @@ impl HashMap { /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn capacity(&self) -> usize { - self.table.capacity() + self.table.raw.capacity() } /// An iterator visiting all keys in arbitrary order. @@ -726,7 +726,7 @@ impl HashMap { // Here we tie the lifetime of self to the iter. unsafe { Iter { - inner: self.table.iter(), + inner: self.table.raw.iter(), marker: PhantomData, } } @@ -771,7 +771,7 @@ impl HashMap { // Here we tie the lifetime of self to the iter. unsafe { IterMut { - inner: self.table.iter(), + inner: self.table.raw.iter(), marker: PhantomData, } } @@ -780,7 +780,7 @@ impl HashMap { #[cfg(test)] #[cfg_attr(feature = "inline-more", inline)] fn raw_capacity(&self) -> usize { - self.table.num_buckets() + self.table.raw.num_buckets() } /// Returns the number of elements in the map. @@ -797,7 +797,7 @@ impl HashMap { /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn len(&self) -> usize { - self.table.len() + self.table.raw.len() } /// Returns `true` if the map contains no elements. @@ -858,7 +858,7 @@ impl HashMap { #[cfg_attr(feature = "inline-more", inline)] pub fn drain(&mut self) -> Drain<'_, K, V, A> { Drain { - inner: self.table.drain(), + inner: self.table.raw.drain(), } } @@ -891,10 +891,10 @@ impl HashMap { { // Here we only use `iter` as a temporary, preventing use-after-free unsafe { - for item in self.table.iter() { + for item in self.table.raw.iter() { let &mut (ref key, ref mut value) = item.as_mut(); if !f(key, value) { - self.table.erase(item); + self.table.raw.erase(item); } } } @@ -951,8 +951,8 @@ impl HashMap { ExtractIf { f, inner: RawExtractIf { - iter: unsafe { self.table.iter() }, - table: &mut self.table, + iter: unsafe { self.table.raw.iter() }, + table: &mut self.table.raw, }, } } @@ -978,7 +978,7 @@ impl HashMap { /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn clear(&mut self) { - self.table.clear(); + self.table.raw.clear(); } /// Creates a consuming iterator visiting all the keys in arbitrary order. @@ -1198,7 +1198,7 @@ where #[cfg_attr(feature = "inline-more", inline)] pub fn entry(&mut self, key: K) -> Entry<'_, K, V, S, A> { let hash = make_hash::(&self.hash_builder, &key); - if let Some(elem) = self.table.find(hash, equivalent_key(&key)) { + if let Some(elem) = self.table.raw.find(hash, equivalent_key(&key)) { Entry::Occupied(OccupiedEntry { hash, elem, @@ -1236,7 +1236,7 @@ where Q: Hash + Equivalent + ?Sized, { let hash = make_hash::(&self.hash_builder, key); - if let Some(elem) = self.table.find(hash, equivalent_key(key)) { + if let Some(elem) = self.table.raw.find(hash, equivalent_key(key)) { EntryRef::Occupied(OccupiedEntry { hash, elem, @@ -1273,9 +1273,9 @@ where Q: Hash + Equivalent + ?Sized, { // Avoid `Option::map` because it bloats LLVM IR. - if !self.table.is_empty() { + if !self.table.raw.is_empty() { let hash = make_hash::(&self.hash_builder, k); - match self.table.get(hash, equivalent_key(k)) { + match self.table.raw.get(hash, equivalent_key(k)) { Some((_, v)) => Some(v), None => None, } @@ -1306,9 +1306,9 @@ where Q: Hash + Equivalent + ?Sized, { // Avoid `Option::map` because it bloats LLVM IR. - if !self.table.is_empty() { + if !self.table.raw.is_empty() { let hash = make_hash::(&self.hash_builder, k); - match self.table.get(hash, equivalent_key(k)) { + match self.table.raw.get(hash, equivalent_key(k)) { Some((key, value)) => Some((key, value)), None => None, } @@ -1343,9 +1343,9 @@ where Q: Hash + Equivalent + ?Sized, { // Avoid `Option::map` because it bloats LLVM IR. - if !self.table.is_empty() { + if !self.table.raw.is_empty() { let hash = make_hash::(&self.hash_builder, k); - match self.table.get_mut(hash, equivalent_key(k)) { + match self.table.raw.get_mut(hash, equivalent_key(k)) { Some(&mut (ref key, ref mut value)) => Some((key, value)), None => None, } @@ -1375,9 +1375,9 @@ where where Q: Hash + Equivalent + ?Sized, { - if !self.table.is_empty() { + if !self.table.raw.is_empty() { let hash = make_hash::(&self.hash_builder, k); - self.table.get(hash, equivalent_key(k)).is_some() + self.table.raw.get(hash, equivalent_key(k)).is_some() } else { false } @@ -1409,9 +1409,9 @@ where Q: Hash + Equivalent + ?Sized, { // Avoid `Option::map` because it bloats LLVM IR. - if !self.table.is_empty() { + if !self.table.raw.is_empty() { let hash = make_hash::(&self.hash_builder, k); - match self.table.get_mut(hash, equivalent_key(k)) { + match self.table.raw.get_mut(hash, equivalent_key(k)) { Some(&mut (_, ref mut v)) => Some(v), None => None, } @@ -1801,12 +1801,13 @@ where let hasher = make_hasher(&self.hash_builder); match self .table + .raw .find_or_find_insert_index(hash, equivalent, hasher) { Ok(bucket) => Some(mem::replace(unsafe { &mut bucket.as_mut().1 }, v)), Err(index) => { unsafe { - self.table.insert_at_index(hash, index, (k, v)); + self.table.raw.insert_at_index(hash, index, (k, v)); } None } @@ -1873,9 +1874,10 @@ where #[cfg_attr(feature = "inline-more", inline)] pub unsafe fn insert_unique_unchecked(&mut self, k: K, v: V) -> (&K, &mut V) { let hash = make_hash::(&self.hash_builder, &k); - let bucket = self - .table - .insert(hash, (k, v), make_hasher::<_, V, S>(&self.hash_builder)); + let bucket = + self.table + .raw + .insert(hash, (k, v), make_hasher::<_, V, S>(&self.hash_builder)); let (k_ref, v_ref) = unsafe { bucket.as_mut() }; (k_ref, v_ref) } @@ -1986,7 +1988,7 @@ where Q: Hash + Equivalent + ?Sized, { let hash = make_hash::(&self.hash_builder, k); - self.table.remove_entry(hash, equivalent_key(k)) + self.table.raw.remove_entry(hash, equivalent_key(k)) } /// Returns the total amount of memory allocated internally by the hash @@ -1996,7 +1998,7 @@ where /// primarily used for memory profiling. #[inline] pub fn allocation_size(&self) -> usize { - self.table.allocation_size() + self.table.raw.allocation_size() } } @@ -3116,7 +3118,7 @@ impl IntoIterator for HashMap { #[cfg_attr(feature = "inline-more", inline)] fn into_iter(self) -> IntoIter { IntoIter { - inner: self.table.into_iter(), + inner: self.table.raw.into_iter(), } } } @@ -3885,7 +3887,7 @@ impl<'a, K, V, S, A: Allocator> OccupiedEntry<'a, K, V, S, A> { /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn remove_entry(self) -> (K, V) { - unsafe { self.table.table.remove(self.elem).0 } + unsafe { self.table.table.raw.remove(self.elem).0 } } /// Gets a reference to the value in the entry. @@ -4116,6 +4118,7 @@ impl<'a, K, V, S, A: Allocator> OccupiedEntry<'a, K, V, S, A> { self.table .table + .raw .replace_bucket_with(self.elem.clone(), |(key, value)| { if let Some(new_value) = f(&key, value) { Some((key, new_value)) @@ -4196,7 +4199,7 @@ impl<'a, K, V, S, A: Allocator> VacantEntry<'a, K, V, S, A> { K: Hash, S: BuildHasher, { - let table = &mut self.table.table; + let table = &mut self.table.table.raw; let entry = table.insert_entry( self.hash, (self.key, value), @@ -4227,7 +4230,7 @@ impl<'a, K, V, S, A: Allocator> VacantEntry<'a, K, V, S, A> { K: Hash, S: BuildHasher, { - let elem = self.table.table.insert( + let elem = self.table.table.raw.insert( self.hash, (self.key, value), make_hasher::<_, V, S>(&self.table.hash_builder), @@ -4536,7 +4539,7 @@ impl<'map, 'key, K, Q: ?Sized, V, S, A: Allocator> VacantEntryRef<'map, 'key, K, Q: ToOwned, S: BuildHasher, { - let table = &mut self.table.table; + let table = &mut self.table.table.raw; let entry = table.insert_entry( self.hash, (self.key.to_owned(), value), @@ -4651,7 +4654,7 @@ impl<'map, 'key, K, Q: ?Sized, V, S, A: Allocator> VacantEntryRef<'map, 'key, K, Q: ToOwned, S: BuildHasher, { - let elem = self.table.table.insert( + let elem = self.table.table.raw.insert( self.hash, (self.key.to_owned(), value), make_hasher::<_, V, S>(&self.table.hash_builder), @@ -4750,7 +4753,7 @@ impl<'map, 'key, K, Q: ?Sized, V, S, A: Allocator> VacantEntryRef<'map, 'key, K, K: Hash, S: BuildHasher, { - let elem = self.table.table.insert( + let elem = self.table.table.raw.insert( self.hash, (key, value), make_hasher::<_, V, S>(&self.table.hash_builder), @@ -6592,7 +6595,7 @@ mod test_map { "panic_in_drop can be set with a type that doesn't need to be dropped", )); } - guard.table.insert( + guard.table.raw.insert( count, ( count, @@ -6766,8 +6769,8 @@ mod test_map { // the returned `RawIter / RawIterRange` iterator. assert_eq!(map.len(), 0); assert_eq!(map.iter().count(), 0); - assert_eq!(unsafe { map.table.iter().count() }, 0); - assert_eq!(unsafe { map.table.iter().iter.count() }, 0); + assert_eq!(unsafe { map.table.raw.iter().count() }, 0); + assert_eq!(unsafe { map.table.raw.iter().iter.count() }, 0); for idx in 0..map.table.num_buckets() { let idx = idx as u64; @@ -6834,8 +6837,8 @@ mod test_map { // the returned `RawIter / RawIterRange` iterator. assert_eq!(map.len(), 0); assert_eq!(map.iter().count(), 0); - assert_eq!(unsafe { map.table.iter().count() }, 0); - assert_eq!(unsafe { map.table.iter().iter.count() }, 0); + assert_eq!(unsafe { map.table.raw.iter().count() }, 0); + assert_eq!(unsafe { map.table.raw.iter().iter.count() }, 0); for idx in 0..map.table.num_buckets() { let idx = idx as u64; diff --git a/src/raw_entry.rs b/src/raw_entry.rs index 13b71bc3e1..bf81c1b610 100644 --- a/src/raw_entry.rs +++ b/src/raw_entry.rs @@ -593,14 +593,14 @@ impl<'a, K, V, S, A: Allocator> RawEntryBuilderMut<'a, K, V, S, A> { where for<'b> F: FnMut(&'b K) -> bool, { - match self.map.table.find(hash, |(k, _)| is_match(k)) { + match self.map.table.raw.find(hash, |(k, _)| is_match(k)) { Some(elem) => RawEntryMut::Occupied(RawOccupiedEntryMut { elem, - table: &mut self.map.table, + table: &mut self.map.table.raw, hash_builder: &self.map.hash_builder, }), None => RawEntryMut::Vacant(RawVacantEntryMut { - table: &mut self.map.table, + table: &mut self.map.table.raw, hash_builder: &self.map.hash_builder, }), } @@ -662,7 +662,7 @@ impl<'a, K, V, S, A: Allocator> RawEntryBuilder<'a, K, V, S, A> { where F: FnMut(&K) -> bool, { - match self.map.table.get(hash, |(k, _)| is_match(k)) { + match self.map.table.raw.get(hash, |(k, _)| is_match(k)) { Some((key, value)) => Some((key, value)), None => None, } diff --git a/src/rustc_entry.rs b/src/rustc_entry.rs index d442f79036..239c6a83d7 100644 --- a/src/rustc_entry.rs +++ b/src/rustc_entry.rs @@ -34,10 +34,10 @@ where #[cfg_attr(feature = "inline-more", inline)] pub fn rustc_entry(&mut self, key: K) -> RustcEntry<'_, K, V, A> { let hash = make_hash(&self.hash_builder, &key); - if let Some(elem) = self.table.find(hash, |q| q.0.eq(&key)) { + if let Some(elem) = self.table.raw.find(hash, |q| q.0.eq(&key)) { RustcEntry::Occupied(RustcOccupiedEntry { elem, - table: &mut self.table, + table: &mut self.table.raw, }) } else { // Ideally we would put this in VacantEntry::insert, but Entry is not @@ -48,7 +48,7 @@ where RustcEntry::Vacant(RustcVacantEntry { hash, key, - table: &mut self.table, + table: &mut self.table.raw, }) } } diff --git a/src/set.rs b/src/set.rs index 366a85af6e..b5e826ea41 100644 --- a/src/set.rs +++ b/src/set.rs @@ -401,8 +401,8 @@ impl HashSet { ExtractIf { f, inner: RawExtractIf { - iter: unsafe { self.map.table.iter() }, - table: &mut self.map.table, + iter: unsafe { self.map.table.raw.iter() }, + table: &mut self.map.table.raw, }, } } diff --git a/src/table.rs b/src/table.rs index 094350442b..011f88fb78 100644 --- a/src/table.rs +++ b/src/table.rs @@ -66,6 +66,7 @@ impl HashTable { /// assert_eq!(table.len(), 0); /// assert_eq!(table.capacity(), 0); /// ``` + #[cfg_attr(feature = "rustc-dep-of-std", rustc_const_stable_indirect)] pub const fn new() -> Self { Self { raw: RawTable::new(), @@ -133,6 +134,7 @@ where /// # test() /// # } /// ``` + #[cfg_attr(feature = "rustc-dep-of-std", rustc_const_stable_indirect)] pub const fn new_in(alloc: A) -> Self { Self { raw: RawTable::new_in(alloc), From b1d85798e312eb985e0e5ade28f1a28b536ba84b Mon Sep 17 00:00:00 2001 From: ltdk Date: Sat, 29 Nov 2025 16:25:54 -0500 Subject: [PATCH 2/4] Call HashTable methods where exact matches exist --- src/map.rs | 43 ++++++++++++++++++------------------------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/src/map.rs b/src/map.rs index ef4303d13a..f465ac47d3 100644 --- a/src/map.rs +++ b/src/map.rs @@ -483,7 +483,7 @@ impl HashMap { /// Returns a reference to the underlying allocator. #[inline] pub fn allocator(&self) -> &A { - self.table.raw.allocator() + self.table.allocator() } /// Creates an empty `HashMap` which will use the given hash builder to hash @@ -587,7 +587,7 @@ impl HashMap { /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn capacity(&self) -> usize { - self.table.raw.capacity() + self.table.capacity() } /// An iterator visiting all keys in arbitrary order. @@ -780,7 +780,7 @@ impl HashMap { #[cfg(test)] #[cfg_attr(feature = "inline-more", inline)] fn raw_capacity(&self) -> usize { - self.table.raw.num_buckets() + self.table.num_buckets() } /// Returns the number of elements in the map. @@ -797,7 +797,7 @@ impl HashMap { /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn len(&self) -> usize { - self.table.raw.len() + self.table.len() } /// Returns `true` if the map contains no elements. @@ -889,15 +889,8 @@ impl HashMap { where F: FnMut(&K, &mut V) -> bool, { - // Here we only use `iter` as a temporary, preventing use-after-free - unsafe { - for item in self.table.raw.iter() { - let &mut (ref key, ref mut value) = item.as_mut(); - if !f(key, value) { - self.table.raw.erase(item); - } - } - } + self.table + .retain(move |&mut (ref key, ref mut val)| f(key, val)) } /// Drains elements which are true under the given predicate, @@ -978,7 +971,7 @@ impl HashMap { /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn clear(&mut self) { - self.table.raw.clear(); + self.table.clear(); } /// Creates a consuming iterator visiting all the keys in arbitrary order. @@ -1273,9 +1266,9 @@ where Q: Hash + Equivalent + ?Sized, { // Avoid `Option::map` because it bloats LLVM IR. - if !self.table.raw.is_empty() { + if !self.table.is_empty() { let hash = make_hash::(&self.hash_builder, k); - match self.table.raw.get(hash, equivalent_key(k)) { + match self.table.find(hash, equivalent_key(k)) { Some((_, v)) => Some(v), None => None, } @@ -1306,9 +1299,9 @@ where Q: Hash + Equivalent + ?Sized, { // Avoid `Option::map` because it bloats LLVM IR. - if !self.table.raw.is_empty() { + if !self.table.is_empty() { let hash = make_hash::(&self.hash_builder, k); - match self.table.raw.get(hash, equivalent_key(k)) { + match self.table.find(hash, equivalent_key(k)) { Some((key, value)) => Some((key, value)), None => None, } @@ -1343,9 +1336,9 @@ where Q: Hash + Equivalent + ?Sized, { // Avoid `Option::map` because it bloats LLVM IR. - if !self.table.raw.is_empty() { + if !self.table.is_empty() { let hash = make_hash::(&self.hash_builder, k); - match self.table.raw.get_mut(hash, equivalent_key(k)) { + match self.table.find_mut(hash, equivalent_key(k)) { Some(&mut (ref key, ref mut value)) => Some((key, value)), None => None, } @@ -1375,9 +1368,9 @@ where where Q: Hash + Equivalent + ?Sized, { - if !self.table.raw.is_empty() { + if !self.table.is_empty() { let hash = make_hash::(&self.hash_builder, k); - self.table.raw.get(hash, equivalent_key(k)).is_some() + self.table.find(hash, equivalent_key(k)).is_some() } else { false } @@ -1409,9 +1402,9 @@ where Q: Hash + Equivalent + ?Sized, { // Avoid `Option::map` because it bloats LLVM IR. - if !self.table.raw.is_empty() { + if !self.table.is_empty() { let hash = make_hash::(&self.hash_builder, k); - match self.table.raw.get_mut(hash, equivalent_key(k)) { + match self.table.find_mut(hash, equivalent_key(k)) { Some(&mut (_, ref mut v)) => Some(v), None => None, } @@ -1998,7 +1991,7 @@ where /// primarily used for memory profiling. #[inline] pub fn allocation_size(&self) -> usize { - self.table.raw.allocation_size() + self.table.allocation_size() } } From 96a49135a25d5ad15e3e40db3fb545f891077ba8 Mon Sep 17 00:00:00 2001 From: ltdk Date: Sat, 29 Nov 2025 17:09:46 -0500 Subject: [PATCH 3/4] Use table iterators --- src/map.rs | 74 ++++++++++++++++++++++-------------------------------- 1 file changed, 30 insertions(+), 44 deletions(-) diff --git a/src/map.rs b/src/map.rs index f465ac47d3..3accc6b376 100644 --- a/src/map.rs +++ b/src/map.rs @@ -1,6 +1,7 @@ use crate::alloc::{Allocator, Global}; -use crate::raw::{Bucket, RawDrain, RawExtractIf, RawIntoIter, RawIter}; -use crate::{DefaultHashBuilder, Equivalent, HashTable, TryReserveError}; +use crate::raw::{Bucket, RawExtractIf}; +use crate::table::{self, HashTable}; +use crate::{DefaultHashBuilder, Equivalent, TryReserveError}; use core::borrow::Borrow; use core::fmt::{self, Debug}; use core::hash::{BuildHasher, Hash}; @@ -723,12 +724,8 @@ impl HashMap { /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn iter(&self) -> Iter<'_, K, V> { - // Here we tie the lifetime of self to the iter. - unsafe { - Iter { - inner: self.table.raw.iter(), - marker: PhantomData, - } + Iter { + inner: self.table.iter(), } } @@ -768,12 +765,9 @@ impl HashMap { /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn iter_mut(&mut self) -> IterMut<'_, K, V> { - // Here we tie the lifetime of self to the iter. - unsafe { - IterMut { - inner: self.table.raw.iter(), - marker: PhantomData, - } + IterMut { + inner: self.table.unsafe_iter(), + marker: PhantomData, } } @@ -858,7 +852,7 @@ impl HashMap { #[cfg_attr(feature = "inline-more", inline)] pub fn drain(&mut self) -> Drain<'_, K, V, A> { Drain { - inner: self.table.raw.drain(), + inner: self.table.drain(), } } @@ -2138,8 +2132,7 @@ where /// assert_eq!(iter.next(), None); /// ``` pub struct Iter<'a, K, V> { - inner: RawIter<(K, V)>, - marker: PhantomData<(&'a K, &'a V)>, + inner: table::Iter<'a, (K, V)>, } // FIXME(#26925) Remove in favor of `#[derive(Clone)]` @@ -2148,7 +2141,6 @@ impl Clone for Iter<'_, K, V> { fn clone(&self) -> Self { Iter { inner: self.inner.clone(), - marker: PhantomData, } } } @@ -2186,7 +2178,7 @@ impl fmt::Debug for Iter<'_, K, V> { /// assert_eq!(map.get(&2).unwrap(), &"Two Mississippi".to_owned()); /// ``` pub struct IterMut<'a, K, V> { - inner: RawIter<(K, V)>, + inner: table::UnsafeIter<'a, (K, V)>, // To ensure invariance with respect to V marker: PhantomData<(&'a K, &'a mut V)>, } @@ -2201,8 +2193,7 @@ impl IterMut<'_, K, V> { #[cfg_attr(feature = "inline-more", inline)] pub(super) fn iter(&self) -> Iter<'_, K, V> { Iter { - inner: self.inner.clone(), - marker: PhantomData, + inner: self.inner.iter(), } } } @@ -2236,7 +2227,7 @@ impl IterMut<'_, K, V> { /// assert_eq!(iter.next(), None); /// ``` pub struct IntoIter { - inner: RawIntoIter<(K, V), A>, + inner: table::IntoIter<(K, V), A>, } impl IntoIter { @@ -2245,7 +2236,6 @@ impl IntoIter { pub(super) fn iter(&self) -> Iter<'_, K, V> { Iter { inner: self.inner.iter(), - marker: PhantomData, } } } @@ -2527,7 +2517,7 @@ impl fmt::Debug for Values<'_, K, V> { /// assert_eq!(drain_iter.next(), None); /// ``` pub struct Drain<'a, K, V, A: Allocator = Global> { - inner: RawDrain<'a, (K, V), A>, + inner: table::Drain<'a, (K, V), A>, } impl Drain<'_, K, V, A> { @@ -2536,7 +2526,6 @@ impl Drain<'_, K, V, A> { pub(super) fn iter(&self) -> Iter<'_, K, V> { Iter { inner: self.inner.iter(), - marker: PhantomData, } } } @@ -3111,7 +3100,7 @@ impl IntoIterator for HashMap { #[cfg_attr(feature = "inline-more", inline)] fn into_iter(self) -> IntoIter { IntoIter { - inner: self.table.raw.into_iter(), + inner: self.table.into_iter(), } } } @@ -3121,21 +3110,17 @@ impl Default for Iter<'_, K, V> { fn default() -> Self { Self { inner: Default::default(), - marker: PhantomData, } } } -impl<'a, K, V> Iterator for Iter<'a, K, V> { +impl<'a, K: 'a, V: 'a> Iterator for Iter<'a, K, V> { type Item = (&'a K, &'a V); #[cfg_attr(feature = "inline-more", inline)] fn next(&mut self) -> Option<(&'a K, &'a V)> { // Avoid `Option::map` because it bloats LLVM IR. match self.inner.next() { - Some(x) => unsafe { - let r = x.as_ref(); - Some((&r.0, &r.1)) - }, + Some(x) => Some((&x.0, &x.1)), None => None, } } @@ -3149,20 +3134,17 @@ impl<'a, K, V> Iterator for Iter<'a, K, V> { Self: Sized, F: FnMut(B, Self::Item) -> B, { - self.inner.fold(init, |acc, x| unsafe { - let (k, v) = x.as_ref(); - f(acc, (k, v)) - }) + self.inner.fold(init, |acc, (k, v)| f(acc, (k, v))) } } -impl ExactSizeIterator for Iter<'_, K, V> { +impl<'a, K: 'a, V: 'a> ExactSizeIterator for Iter<'a, K, V> { #[cfg_attr(feature = "inline-more", inline)] fn len(&self) -> usize { self.inner.len() } } -impl FusedIterator for Iter<'_, K, V> {} +impl<'a, K: 'a, V: 'a> FusedIterator for Iter<'a, K, V> {} impl Default for IterMut<'_, K, V> { #[cfg_attr(feature = "inline-more", inline)] @@ -3180,10 +3162,12 @@ impl<'a, K, V> Iterator for IterMut<'a, K, V> { fn next(&mut self) -> Option<(&'a K, &'a mut V)> { // Avoid `Option::map` because it bloats LLVM IR. match self.inner.next() { - Some(x) => unsafe { - let r = x.as_mut(); - Some((&r.0, &mut r.1)) - }, + Some(mut x) => { + // SAFETY: We match the lifetime of the original iterator + // with the correct variance provided by PhantomData. + let &mut (ref k, ref mut v) = unsafe { x.as_mut() }; + Some((k, v)) + } None => None, } } @@ -3197,8 +3181,10 @@ impl<'a, K, V> Iterator for IterMut<'a, K, V> { Self: Sized, F: FnMut(B, Self::Item) -> B, { - self.inner.fold(init, |acc, x| unsafe { - let (k, v) = x.as_mut(); + self.inner.fold(init, |acc, mut x| { + // SAFETY: We match the lifetime of the original iterator + // with the correct variance provided by PhantomData. + let &mut (ref k, ref mut v) = unsafe { x.as_mut() }; f(acc, (k, v)) }) } From 8ee7fa9bd29fdce1ccf1862b1158487c8e063c74 Mon Sep 17 00:00:00 2001 From: ltdk Date: Tue, 2 Dec 2025 16:07:37 -0500 Subject: [PATCH 4/4] Use table entries --- src/map.rs | 178 +++++++++++++++++------------------------------ src/raw.rs | 13 ---- src/raw_entry.rs | 22 +++--- 3 files changed, 79 insertions(+), 134 deletions(-) diff --git a/src/map.rs b/src/map.rs index 3accc6b376..8baeb7db49 100644 --- a/src/map.rs +++ b/src/map.rs @@ -1,5 +1,5 @@ use crate::alloc::{Allocator, Global}; -use crate::raw::{Bucket, RawExtractIf}; +use crate::raw::RawExtractIf; use crate::table::{self, HashTable}; use crate::{DefaultHashBuilder, Equivalent, TryReserveError}; use core::borrow::Borrow; @@ -1185,18 +1185,17 @@ where #[cfg_attr(feature = "inline-more", inline)] pub fn entry(&mut self, key: K) -> Entry<'_, K, V, S, A> { let hash = make_hash::(&self.hash_builder, &key); - if let Some(elem) = self.table.raw.find(hash, equivalent_key(&key)) { - Entry::Occupied(OccupiedEntry { - hash, - elem, - table: self, - }) - } else { - Entry::Vacant(VacantEntry { - hash, + let hasher = make_hasher(&self.hash_builder); + match self.table.entry(hash, equivalent_key(&key), hasher) { + table::Entry::Occupied(inner) => Entry::Occupied(OccupiedEntry { + inner, + marker: PhantomData, + }), + table::Entry::Vacant(inner) => Entry::Vacant(VacantEntry { + inner, key, - table: self, - }) + marker: PhantomData, + }), } } @@ -1223,18 +1222,17 @@ where Q: Hash + Equivalent + ?Sized, { let hash = make_hash::(&self.hash_builder, key); - if let Some(elem) = self.table.raw.find(hash, equivalent_key(key)) { - EntryRef::Occupied(OccupiedEntry { - hash, - elem, - table: self, - }) - } else { - EntryRef::Vacant(VacantEntryRef { - hash, + let hasher = make_hasher(&self.hash_builder); + match self.table.entry(hash, equivalent_key(key), hasher) { + table::Entry::Occupied(inner) => EntryRef::Occupied(OccupiedEntry { + inner, + marker: PhantomData, + }), + table::Entry::Vacant(inner) => EntryRef::Vacant(VacantEntryRef { + inner, key, - table: self, - }) + marker: PhantomData, + }), } } @@ -1783,19 +1781,10 @@ where /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn insert(&mut self, k: K, v: V) -> Option { - let hash = make_hash(&self.hash_builder, &k); - let equivalent = equivalent_key(&k); - let hasher = make_hasher(&self.hash_builder); - match self - .table - .raw - .find_or_find_insert_index(hash, equivalent, hasher) - { - Ok(bucket) => Some(mem::replace(unsafe { &mut bucket.as_mut().1 }, v)), - Err(index) => { - unsafe { - self.table.raw.insert_at_index(hash, index, (k, v)); - } + match self.entry(k) { + Entry::Occupied(mut entry) => Some(entry.insert(v)), + Entry::Vacant(entry) => { + entry.insert(v); None } } @@ -1861,11 +1850,10 @@ where #[cfg_attr(feature = "inline-more", inline)] pub unsafe fn insert_unique_unchecked(&mut self, k: K, v: V) -> (&K, &mut V) { let hash = make_hash::(&self.hash_builder, &k); - let bucket = + let entry = self.table - .raw - .insert(hash, (k, v), make_hasher::<_, V, S>(&self.hash_builder)); - let (k_ref, v_ref) = unsafe { bucket.as_mut() }; + .insert_unique(hash, (k, v), make_hasher::<_, V, S>(&self.hash_builder)); + let (k_ref, v_ref) = entry.into_mut(); (k_ref, v_ref) } @@ -2741,9 +2729,8 @@ impl Debug for Entry<'_, K, V, S, A> { /// assert_eq!(map.len(), 2); /// ``` pub struct OccupiedEntry<'a, K, V, S = DefaultHashBuilder, A: Allocator = Global> { - hash: u64, - elem: Bucket<(K, V)>, - table: &'a mut HashMap, + inner: table::OccupiedEntry<'a, (K, V), A>, + marker: PhantomData<&'a mut S>, } unsafe impl Send for OccupiedEntry<'_, K, V, S, A> @@ -2801,9 +2788,9 @@ impl Debug for OccupiedEntry<'_, K, V, S, A /// assert!(map[&"b"] == 20 && map.len() == 2); /// ``` pub struct VacantEntry<'a, K, V, S = DefaultHashBuilder, A: Allocator = Global> { - hash: u64, + inner: table::VacantEntry<'a, (K, V), A>, key: K, - table: &'a mut HashMap, + marker: PhantomData<&'a mut S>, } impl Debug for VacantEntry<'_, K, V, S, A> { @@ -2939,9 +2926,9 @@ where /// assert!(map["b"] == 20 && map.len() == 2); /// ``` pub struct VacantEntryRef<'map, 'key, K, Q: ?Sized, V, S, A: Allocator = Global> { - hash: u64, + inner: table::VacantEntry<'map, (K, V), A>, key: &'key Q, - table: &'map mut HashMap, + marker: PhantomData<&'map mut S>, } impl Debug for VacantEntryRef<'_, '_, K, Q, V, S, A> @@ -3746,7 +3733,7 @@ impl<'a, K, V, S, A: Allocator> OccupiedEntry<'a, K, V, S, A> { /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn key(&self) -> &K { - unsafe { &self.elem.as_ref().0 } + &self.inner.get().0 } /// Replaces the key in the entry with a new one. @@ -3837,7 +3824,7 @@ impl<'a, K, V, S, A: Allocator> OccupiedEntry<'a, K, V, S, A> { /// ``` #[cfg_attr(feature = "inline-more", inline)] pub unsafe fn replace_key_unchecked(&mut self, key: K) -> K { - mem::replace(unsafe { &mut self.elem.as_mut().0 }, key) + mem::replace(&mut self.inner.get_mut().0, key) } /// Take the ownership of the key and value from the map. @@ -3866,7 +3853,7 @@ impl<'a, K, V, S, A: Allocator> OccupiedEntry<'a, K, V, S, A> { /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn remove_entry(self) -> (K, V) { - unsafe { self.table.table.raw.remove(self.elem).0 } + self.inner.remove().0 } /// Gets a reference to the value in the entry. @@ -3887,7 +3874,7 @@ impl<'a, K, V, S, A: Allocator> OccupiedEntry<'a, K, V, S, A> { /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn get(&self) -> &V { - unsafe { &self.elem.as_ref().1 } + &self.inner.get().1 } /// Gets a mutable reference to the value in the entry. @@ -3919,7 +3906,7 @@ impl<'a, K, V, S, A: Allocator> OccupiedEntry<'a, K, V, S, A> { /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn get_mut(&mut self) -> &mut V { - unsafe { &mut self.elem.as_mut().1 } + &mut self.inner.get_mut().1 } /// Converts the `OccupiedEntry` into a mutable reference to the value in the entry @@ -3950,7 +3937,7 @@ impl<'a, K, V, S, A: Allocator> OccupiedEntry<'a, K, V, S, A> { /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn into_mut(self) -> &'a mut V { - unsafe { &mut self.elem.as_mut().1 } + &mut self.inner.into_mut().1 } /// Converts the `OccupiedEntry` into a reference to the key and a @@ -3985,7 +3972,7 @@ impl<'a, K, V, S, A: Allocator> OccupiedEntry<'a, K, V, S, A> { /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn into_entry(self) -> (&'a K, &'a mut V) { - let (key, val) = unsafe { self.elem.as_mut() }; + let (key, val) = self.inner.into_mut(); (key, val) } @@ -4092,30 +4079,25 @@ impl<'a, K, V, S, A: Allocator> OccupiedEntry<'a, K, V, S, A> { where F: FnOnce(&K, V) -> Option, { - unsafe { - let mut spare_key = None; - - self.table - .table - .raw - .replace_bucket_with(self.elem.clone(), |(key, value)| { - if let Some(new_value) = f(&key, value) { - Some((key, new_value)) - } else { - spare_key = Some(key); - None - } - }); + let mut spare_key = None; - if let Some(key) = spare_key { - Entry::Vacant(VacantEntry { - hash: self.hash, - key, - table: self.table, - }) + match self.inner.replace_entry_with(|(key, value)| { + if let Some(new_value) = f(&key, value) { + Some((key, new_value)) } else { - Entry::Occupied(self) + spare_key = Some(key); + None } + }) { + table::Entry::Vacant(inner) => Entry::Vacant(VacantEntry { + inner, + key: unsafe { spare_key.unwrap_unchecked() }, + marker: PhantomData, + }), + table::Entry::Occupied(inner) => Entry::Occupied(OccupiedEntry { + inner, + marker: PhantomData, + }), } } } @@ -4178,13 +4160,7 @@ impl<'a, K, V, S, A: Allocator> VacantEntry<'a, K, V, S, A> { K: Hash, S: BuildHasher, { - let table = &mut self.table.table.raw; - let entry = table.insert_entry( - self.hash, - (self.key, value), - make_hasher::<_, V, S>(&self.table.hash_builder), - ); - &mut entry.1 + &mut self.inner.insert((self.key, value)).into_mut().1 } /// Sets the value of the entry with the [`VacantEntry`]'s key, @@ -4209,15 +4185,9 @@ impl<'a, K, V, S, A: Allocator> VacantEntry<'a, K, V, S, A> { K: Hash, S: BuildHasher, { - let elem = self.table.table.raw.insert( - self.hash, - (self.key, value), - make_hasher::<_, V, S>(&self.table.hash_builder), - ); OccupiedEntry { - hash: self.hash, - elem, - table: self.table, + inner: self.inner.insert((self.key, value)), + marker: PhantomData, } } } @@ -4518,13 +4488,7 @@ impl<'map, 'key, K, Q: ?Sized, V, S, A: Allocator> VacantEntryRef<'map, 'key, K, Q: ToOwned, S: BuildHasher, { - let table = &mut self.table.table.raw; - let entry = table.insert_entry( - self.hash, - (self.key.to_owned(), value), - make_hasher::<_, V, S>(&self.table.hash_builder), - ); - &mut entry.1 + &mut self.inner.insert((self.key.to_owned(), value)).into_mut().1 } /// Sets the key and value of the entry and returns a mutable reference to @@ -4633,15 +4597,9 @@ impl<'map, 'key, K, Q: ?Sized, V, S, A: Allocator> VacantEntryRef<'map, 'key, K, Q: ToOwned, S: BuildHasher, { - let elem = self.table.table.raw.insert( - self.hash, - (self.key.to_owned(), value), - make_hasher::<_, V, S>(&self.table.hash_builder), - ); OccupiedEntry { - hash: self.hash, - elem, - table: self.table, + inner: self.inner.insert((self.key.to_owned(), value)), + marker: PhantomData, } } @@ -4732,15 +4690,9 @@ impl<'map, 'key, K, Q: ?Sized, V, S, A: Allocator> VacantEntryRef<'map, 'key, K, K: Hash, S: BuildHasher, { - let elem = self.table.table.raw.insert( - self.hash, - (key, value), - make_hasher::<_, V, S>(&self.table.hash_builder), - ); OccupiedEntry { - hash: self.hash, - elem, - table: self.table, + inner: self.inner.insert((key, value)), + marker: PhantomData, } } } diff --git a/src/raw.rs b/src/raw.rs index 4bc1f7439e..aa765f987e 100644 --- a/src/raw.rs +++ b/src/raw.rs @@ -1059,19 +1059,6 @@ impl RawTable { } } - /// Inserts a new element into the table, and returns a mutable reference to it. - /// - /// This does not check if the given element already exists in the table. - #[cfg_attr(feature = "inline-more", inline)] - pub(crate) fn insert_entry( - &mut self, - hash: u64, - value: T, - hasher: impl Fn(&T) -> u64, - ) -> &mut T { - unsafe { self.insert(hash, value, hasher).as_mut() } - } - /// Inserts a new element into the table, without growing the table. /// /// There must be enough space in the table to insert the new element. diff --git a/src/raw_entry.rs b/src/raw_entry.rs index bf81c1b610..637651e7a8 100644 --- a/src/raw_entry.rs +++ b/src/raw_entry.rs @@ -1354,11 +1354,15 @@ impl<'a, K, V, S, A: Allocator> RawVacantEntryMut<'a, K, V, S, A> { K: Hash, S: BuildHasher, { - let &mut (ref mut k, ref mut v) = self.table.insert_entry( - hash, - (key, value), - make_hasher::<_, V, S>(self.hash_builder), - ); + let &mut (ref mut k, ref mut v) = unsafe { + self.table + .insert( + hash, + (key, value), + make_hasher::<_, V, S>(self.hash_builder), + ) + .as_mut() + }; (k, v) } @@ -1409,9 +1413,11 @@ impl<'a, K, V, S, A: Allocator> RawVacantEntryMut<'a, K, V, S, A> { where H: Fn(&K) -> u64, { - let &mut (ref mut k, ref mut v) = self - .table - .insert_entry(hash, (key, value), |x| hasher(&x.0)); + let &mut (ref mut k, ref mut v) = unsafe { + self.table + .insert(hash, (key, value), |x| hasher(&x.0)) + .as_mut() + }; (k, v) }