refactor(serde): drop internal tokenstream types

keyvalues-serde/src/de/mod.rs

@@ -20,7 +20,7 @@ use std::{
 use crate::{
     de::{map::ObjEater, seq::SeqBuilder},
     error::{Error, Result},
-    tokens::{Token, TokenStream},
+    tokens::{tokens_from_vdf, Token},
 };
 
 pub fn from_reader<R: Read, T: DeserializeOwned>(rdr: R) -> Result<T> {
@@ -84,15 +84,15 @@ pub struct Deserializer<'de> {
 impl<'de> Deserializer<'de> {
     /// Attempts to create a new VDF deserializer along with returning the top level VDF key
     pub fn new_with_key(vdf: Vdf<'de>) -> Result<(Self, Key<'de>)> {
-        let token_stream = TokenStream::from(vdf);
+        let token_stream = tokens_from_vdf(vdf);
 
         let key = if let Some(Token::Key(key)) = token_stream.first() {
             key.clone()
         } else {
             unreachable!("Tokenstream must start with key");
         };
 
-        let tokens = token_stream.0.into_iter().peekable();
+        let tokens = token_stream.into_iter().peekable();
         Ok((Self { tokens }, key.clone()))
     }
 

keyvalues-serde/src/ser.rs

@@ -1,13 +1,12 @@
 //! Serialize Rust types to VDF text
 
-use keyvalues_parser::Vdf;
 use serde_core::{ser, Serialize};
 
 use std::io::Write;
 
 use crate::{
     error::{Error, Result},
-    tokens::{NaiveToken, NaiveTokenStream},
+    tokens::{naive::vdf_from_naive_tokens, NaiveToken},
 };
 
 /// The struct for serializing Rust values into VDF text
@@ -16,7 +15,7 @@ use crate::{
 /// [`to_writer_with_key()`] can be used instead
 #[derive(Default)]
 pub struct Serializer {
-    tokens: NaiveTokenStream,
+    tokens: Vec<NaiveToken>,
 }
 
 impl Serializer {
@@ -79,7 +78,7 @@ where
         }
     }
 
-    let vdf = Vdf::try_from(&serializer.tokens)?;
+    let vdf = vdf_from_naive_tokens(&serializer.tokens)?;
     write!(writer, "{vdf}")?;
 
     Ok(())

keyvalues-serde/src/tokens/mod.rs

@@ -1,140 +1,62 @@
-// TODO: a lot of this can probably be slimmed down at this point
-// TODO: implement a validate function
-// TODO: make a note that this has invariants that must be upheld, so it is only exposed internally
+// TODO: replace with some kind of iterator that decomposes the original structure instead of using
+// an intermediate layer
 
-mod naive;
+pub(crate) mod naive;
 #[cfg(test)]
 mod tests;
 
 use keyvalues_parser::{Obj, Value, Vdf};
 
-use std::{
-    borrow::Cow,
-    ops::{Deref, DerefMut},
-};
-
-pub use crate::tokens::naive::{NaiveToken, NaiveTokenStream};
-
-// I've been struggling to get serde to play nice with using a more complex internal structure in a
-// `Deserializer`. I think the easiest solution I can come up with is to flatten out the `Vdf` into
-// a stream of tokens that serde can consume. In this way the Deserializer can just work on
-// munching through all the tokens instead of trying to mutate a more complex nested structure
-// containing different types
-/// A stream of [`Token`]s representing a [`Vdf`]
-///
-/// I think an example is the easiest way to understand the structure so something like
-///
-/// ```vdf
-/// "Outer Key"
-/// {
-///     "Inner Key" "Inner Value"
-///     "Inner Key"
-///     {
-///     }
-/// }
-/// ```
-///
-/// will be transformed into
-///
-/// ```ron
-/// Vdf(
-///     key: "Outer Key",
-///     value: Obj({
-///         "Inner Key": [
-///             Str("Inner Value"),
-///             Obj({})
-///         ]
-///     })
-/// )
-/// ```
-///
-/// which has the following token stream
-///
-/// ```ron
-/// TokenStream([
-///     Key("Outer Key"),
-///     ObjBegin,
-///     Key("Inner Key"),
-///     SeqBegin,
-///     Str("Inner Value"),
-///     ObjBegin,
-///     ObjEnd,
-///     SeqEnd,
-///     ObjEnd,
-/// )]
-/// ```
-///
-/// So in this way it's a linear sequence of keys and values where the value is either a str or an
-/// object.
-#[derive(Debug, PartialEq, Eq)]
-pub struct TokenStream<'a>(pub Vec<Token<'a>>);
+use std::borrow::Cow;
 
-impl<'a> Deref for TokenStream<'a> {
-    type Target = Vec<Token<'a>>;
+pub use crate::tokens::naive::NaiveToken;
 
-    fn deref(&self) -> &Self::Target {
-        &self.0
-    }
-}
+pub(crate) fn tokens_from_vdf(vdf: Vdf<'_>) -> Vec<Token<'_>> {
+    let Vdf { key, value } = vdf;
 
-impl DerefMut for TokenStream<'_> {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        &mut self.0
-    }
+    let mut tokens = vec![Token::Key(key)];
+    tokens.extend(tokens_from_value(value));
+    tokens
 }
 
-impl<'a> From<Vdf<'a>> for TokenStream<'a> {
-    fn from(vdf: Vdf<'a>) -> Self {
-        let Vdf { key, value } = vdf;
-
-        let mut inner = vec![Token::Key(key)];
-        inner.extend(TokenStream::from(value).0);
-
-        Self(inner)
-    }
-}
+// TODO: pass through a `&mut Vec<_>` instead of allocating new ones
+fn tokens_from_value(value: Value<'_>) -> Vec<Token<'_>> {
+    let mut tokens = Vec::new();
 
-impl<'a> From<Value<'a>> for TokenStream<'a> {
-    fn from(value: Value<'a>) -> Self {
-        let mut inner = Vec::new();
-
-        match value {
-            Value::Str(s) => inner.push(Token::Str(s)),
-            Value::Obj(obj) => {
-                inner.push(Token::ObjBegin);
-                inner.extend(Self::from(obj).0);
-                inner.push(Token::ObjEnd);
-            }
+    match value {
+        Value::Str(s) => tokens.push(Token::Str(s)),
+        Value::Obj(obj) => {
+            tokens.push(Token::ObjBegin);
+            tokens.extend(tokens_from_obj(obj));
+            tokens.push(Token::ObjEnd);
         }
-
-        Self(inner)
     }
-}
 
-impl<'a> From<Obj<'a>> for TokenStream<'a> {
-    fn from(obj: Obj<'a>) -> Self {
-        let mut inner = Vec::new();
+    tokens
+}
 
-        for (key, values) in obj.into_inner().into_iter() {
-            inner.push(Token::Key(key));
+fn tokens_from_obj(obj: Obj<'_>) -> Vec<Token<'_>> {
+    let mut tokens = Vec::new();
 
-            // For ease of use a sequence is only marked when len != 1
-            let num_values = values.len();
-            if num_values != 1 {
-                inner.push(Token::SeqBegin);
-            }
+    for (key, values) in obj.into_inner().into_iter() {
+        tokens.push(Token::Key(key));
 
-            for value in values {
-                inner.extend(TokenStream::from(value).0);
-            }
+        // For ease of use a sequence is only marked when len != 1
+        let num_values = values.len();
+        if num_values != 1 {
+            tokens.push(Token::SeqBegin);
+        }
 
-            if num_values != 1 {
-                inner.push(Token::SeqEnd);
-            }
+        for value in values {
+            tokens.extend(tokens_from_value(value));
         }
 
-        Self(inner)
+        if num_values != 1 {
+            tokens.push(Token::SeqEnd);
+        }
     }
+
+    tokens
 }
 
 /// A single VDF token