Text reader implementation cleanup (#763)

* Simplifies `EncodedTextValue` layout and shrinks several types
* Environment allocation uses a capacity hint
* Types now hold a ref (8 bytes) instead of the EncodingContext (24 bytes)
* Removes superfluous `MatchedRawTextValue` type
* Shrinks TemplateVariableReference's signature index from usize to u16

Author: zslayton

src/lazy/binary/raw/v1_1/value.rs

@@ -175,7 +175,6 @@ impl<'top> LazyRawBinaryValue_1_1<'top> {
     fn value_body(&self) -> IonResult<&'top [u8]> {
         let value_total_length = self.encoded_value.total_length();
         if self.input.len() < value_total_length {
-            eprintln!("[value_body] Incomplete {:?}", self);
             return IonResult::incomplete(
                 "only part of the requested value is available in the buffer",
                 self.input.offset(),

src/lazy/binary/raw/value.rs

@@ -231,7 +231,7 @@ impl<'a, 'top> EncodedBinaryValueData_1_0<'a, 'top> {
     pub fn trailing_length_span(&self) -> Span<'top> {
         let stream_range = self.trailing_length_range();
         let offset = self.value.input.offset();
-        let local_range = stream_range.start - offset .. stream_range.end - offset;
+        let local_range = stream_range.start - offset..stream_range.end - offset;
         let bytes = &self.value.input.bytes()[local_range];
         Span::with_offset(stream_range.start, bytes)
     }
@@ -252,7 +252,7 @@ impl<'a, 'top> EncodedBinaryValueData_1_0<'a, 'top> {
     pub fn body_span(&self) -> Span<'top> {
         let stream_range = self.body_range();
         let offset = self.value.input.offset();
-        let local_range = stream_range.start - offset .. stream_range.end - offset;
+        let local_range = stream_range.start - offset..stream_range.end - offset;
         let bytes = &self.span().bytes()[local_range];
         Span::with_offset(stream_range.start, bytes)
     }

src/lazy/decoder.rs

@@ -269,7 +269,7 @@ impl<'top, D: LazyDecoder> HasRange for LazyRawFieldExpr<'top, D> {
 // function while also preventing users from seeing or depending on it.
 pub(crate) mod private {
     use crate::lazy::expanded::r#struct::UnexpandedField;
-    use crate::lazy::expanded::EncodingContext;
+    use crate::lazy::expanded::EncodingContextRef;
     use crate::IonResult;
 
     use super::{LazyDecoder, LazyRawFieldExpr, LazyRawStruct};
@@ -286,15 +286,21 @@ pub(crate) mod private {
         /// expansion process.
         fn unexpanded_fields(
             &self,
-            context: EncodingContext<'top>,
+            context: EncodingContextRef<'top>,
         ) -> RawStructUnexpandedFieldsIterator<'top, D>;
     }
 
     pub struct RawStructUnexpandedFieldsIterator<'top, D: LazyDecoder> {
-        context: EncodingContext<'top>,
+        context: EncodingContextRef<'top>,
         raw_fields: <D::Struct<'top> as LazyRawStruct<'top, D>>::Iterator,
     }
 
+    impl<'top, D: LazyDecoder> RawStructUnexpandedFieldsIterator<'top, D> {
+        pub fn context(&self) -> EncodingContextRef<'top> {
+            self.context
+        }
+    }
+
     impl<'top, D: LazyDecoder> Iterator for RawStructUnexpandedFieldsIterator<'top, D> {
         type Item = IonResult<UnexpandedField<'top, D>>;
 
@@ -320,7 +326,7 @@ pub(crate) mod private {
     {
         fn unexpanded_fields(
             &self,
-            context: EncodingContext<'top>,
+            context: EncodingContextRef<'top>,
         ) -> RawStructUnexpandedFieldsIterator<'top, D> {
             let raw_fields = <Self as LazyRawStruct<'top, D>>::iter(self);
             RawStructUnexpandedFieldsIterator {

src/lazy/encoding.rs

@@ -29,7 +29,7 @@ use crate::lazy::text::raw::v1_1::reader::{
 };
 use crate::lazy::text::value::{
     LazyRawTextValue, LazyRawTextValue_1_0, LazyRawTextValue_1_1, LazyRawTextVersionMarker_1_0,
-    LazyRawTextVersionMarker_1_1, MatchedRawTextValue, RawTextAnnotationsIterator,
+    LazyRawTextVersionMarker_1_1, RawTextAnnotationsIterator,
 };
 use crate::{TextKind, WriteConfig};
 
@@ -100,26 +100,16 @@ pub trait BinaryEncoding<'top>: Encoding + LazyDecoder {}
 
 /// Marker trait for text encodings.
 pub trait TextEncoding<'top>:
-    Encoding + LazyDecoder<AnnotationsIterator<'top> = RawTextAnnotationsIterator<'top>>
+    Encoding
+    + LazyDecoder<
+        AnnotationsIterator<'top> = RawTextAnnotationsIterator<'top>,
+        Value<'top> = LazyRawTextValue<'top, Self>,
+    >
 {
-    fn value_from_matched(
-        matched: MatchedRawTextValue<'top, Self>,
-    ) -> <Self as LazyDecoder>::Value<'top>;
-}
-impl<'top> TextEncoding<'top> for TextEncoding_1_0 {
-    fn value_from_matched(
-        matched: MatchedRawTextValue<'_, Self>,
-    ) -> <Self as LazyDecoder>::Value<'_> {
-        LazyRawTextValue_1_0::from(matched)
-    }
-}
-impl<'top> TextEncoding<'top> for TextEncoding_1_1 {
-    fn value_from_matched(
-        matched: MatchedRawTextValue<'_, Self>,
-    ) -> <Self as LazyDecoder>::Value<'_> {
-        LazyRawTextValue_1_1::from(matched)
-    }
+    // No methods, just a marker
 }
+impl<'top> TextEncoding<'top> for TextEncoding_1_0 {}
+impl<'top> TextEncoding<'top> for TextEncoding_1_1 {}
 
 /// Marker trait for encodings that support macros.
 pub trait EncodingWithMacroSupport {}
@@ -192,7 +182,6 @@ impl LazyDecoder for BinaryEncoding_1_1 {
 // the implementation will conflict with the core `impl<T> From<T> for T` implementation.
 pub trait RawValueLiteral {}
 
-impl<'top, E: TextEncoding<'top>> RawValueLiteral for MatchedRawTextValue<'top, E> {}
 impl<'top, E: TextEncoding<'top>> RawValueLiteral for LazyRawTextValue<'top, E> {}
 impl<'top> RawValueLiteral for LazyRawBinaryValue_1_0<'top> {}
 impl<'top> RawValueLiteral for LazyRawBinaryValue_1_1<'top> {}

src/lazy/expanded/compiler.rs

@@ -9,7 +9,7 @@ use crate::lazy::expanded::template::{
     TemplateBodyMacroInvocation, TemplateBodyValueExpr, TemplateMacro, TemplateStructIndex,
     TemplateValue,
 };
-use crate::lazy::expanded::EncodingContext;
+use crate::lazy::expanded::EncodingContextRef;
 use crate::lazy::r#struct::LazyStruct;
 use crate::lazy::reader::LazyTextReader_1_1;
 use crate::lazy::sequence::{LazyList, LazySExp};
@@ -61,7 +61,7 @@ impl TemplateCompiler {
     /// to the template without interpretation. `(quote ...)` does not appear in the compiled
     /// template as there is nothing more for it to do at expansion time.
     pub fn compile_from_text(
-        context: EncodingContext,
+        context: EncodingContextRef,
         expression: &str,
     ) -> IonResult<TemplateMacro> {
         // TODO: This is a rudimentary implementation that panics instead of performing thorough
@@ -137,7 +137,7 @@ impl TemplateCompiler {
     ///
     /// If `is_quoted` is true, nested symbols and s-expressions will not be interpreted.
     fn compile_value<'top, D: LazyDecoder>(
-        context: EncodingContext<'top>,
+        context: EncodingContextRef<'top>,
         signature: &MacroSignature,
         definition: &mut TemplateBody,
         is_quoted: bool,
@@ -210,7 +210,7 @@ impl TemplateCompiler {
 
     /// Helper method for visiting all of the child expressions in a list.
     fn compile_list<'top, D: LazyDecoder>(
-        context: EncodingContext<'top>,
+        context: EncodingContextRef<'top>,
         signature: &MacroSignature,
         definition: &mut TemplateBody,
         is_quoted: bool,
@@ -238,7 +238,7 @@ impl TemplateCompiler {
 
     /// Helper method for visiting all of the child expressions in a sexp.
     fn compile_sexp<'top, D: LazyDecoder>(
-        context: EncodingContext<'top>,
+        context: EncodingContextRef<'top>,
         signature: &MacroSignature,
         definition: &mut TemplateBody,
         is_quoted: bool,
@@ -272,7 +272,7 @@ impl TemplateCompiler {
     /// Adds a `lazy_sexp` that has been determined to represent a macro invocation to the
     /// TemplateBody.
     fn compile_macro<'top, D: LazyDecoder>(
-        context: EncodingContext<'top>,
+        context: EncodingContextRef<'top>,
         signature: &MacroSignature,
         definition: &mut TemplateBody,
         lazy_sexp: LazySExp<'top, D>,
@@ -311,7 +311,7 @@ impl TemplateCompiler {
     /// Given a `LazyValue` that represents a macro ID (name or address), attempts to resolve the
     /// ID to a macro address.
     fn name_and_address_from_id_expr<'top, D: LazyDecoder>(
-        context: EncodingContext<'top>,
+        context: EncodingContextRef<'top>,
         id_expr: Option<IonResult<LazyValue<'top, D>>>,
     ) -> IonResult<(Option<String>, usize)> {
         match id_expr {
@@ -352,7 +352,7 @@ impl TemplateCompiler {
     /// without interpretation. `lazy_sexp` itself is the `quote` macro, and does not get added
     /// to the template body as there is nothing more for it to do at evaluation time.
     fn compile_quoted_elements<'top, D: LazyDecoder>(
-        context: EncodingContext<'top>,
+        context: EncodingContextRef<'top>,
         signature: &MacroSignature,
         definition: &mut TemplateBody,
         lazy_sexp: LazySExp<'top, D>,
@@ -375,7 +375,7 @@ impl TemplateCompiler {
 
     /// Adds `lazy_sexp` to the template body without interpretation.
     fn compile_quoted_sexp<'top, D: LazyDecoder>(
-        context: EncodingContext<'top>,
+        context: EncodingContextRef<'top>,
         signature: &MacroSignature,
         definition: &mut TemplateBody,
         annotations_range: Range<usize>,
@@ -419,7 +419,7 @@ impl TemplateCompiler {
 
     /// Recursively adds all of the expressions in `lazy_struct` to the `TemplateBody`.
     fn compile_struct<'top, D: LazyDecoder>(
-        context: EncodingContext<'top>,
+        context: EncodingContextRef<'top>,
         signature: &MacroSignature,
         definition: &mut TemplateBody,
         is_quoted: bool,
@@ -476,7 +476,7 @@ impl TemplateCompiler {
     /// Resolves `variable` to a parameter in the macro signature and adds a corresponding
     /// `TemplateExpansionStep` to the `TemplateBody`.
     fn compile_variable_reference(
-        _context: EncodingContext,
+        _context: EncodingContextRef,
         signature: &MacroSignature,
         definition: &mut TemplateBody,
         annotations_range: Range<usize>,
@@ -497,7 +497,10 @@ impl TemplateCompiler {
             .ok_or_else(|| {
                 IonError::decoding_error(format!("variable '{name}' is not recognized"))
             })?;
-        definition.push_variable(signature_index);
+        if signature_index > u16::MAX as usize {
+            return IonResult::decoding_error("this implementation supports up to 65K parameters");
+        }
+        definition.push_variable(signature_index as u16);
         Ok(())
     }
 }
@@ -558,7 +561,7 @@ mod tests {
             definition,
             index,
             TemplateBodyValueExpr::Variable(TemplateBodyVariableReference::new(
-                expected_signature_index,
+                expected_signature_index as u16,
             )),
         )
     }
@@ -630,7 +633,7 @@ mod tests {
 
         let expression = "(macro foo () 42)";
 
-        let template = TemplateCompiler::compile_from_text(context, expression)?;
+        let template = TemplateCompiler::compile_from_text(context.get_ref(), expression)?;
         assert_eq!(template.name(), "foo");
         assert_eq!(template.signature().parameters().len(), 0);
         expect_value(&template, 0, TemplateValue::Int(42.into()))?;
@@ -644,7 +647,7 @@ mod tests {
 
         let expression = "(macro foo () [1, 2, 3])";
 
-        let template = TemplateCompiler::compile_from_text(context, expression)?;
+        let template = TemplateCompiler::compile_from_text(context.get_ref(), expression)?;
         assert_eq!(template.name(), "foo");
         assert_eq!(template.signature().parameters().len(), 0);
         expect_value(&template, 0, TemplateValue::List(ExprRange::new(1..4)))?;
@@ -661,7 +664,7 @@ mod tests {
 
         let expression = r#"(macro foo () (values 42 "hello" false))"#;
 
-        let template = TemplateCompiler::compile_from_text(context, expression)?;
+        let template = TemplateCompiler::compile_from_text(context.get_ref(), expression)?;
         assert_eq!(template.name(), "foo");
         assert_eq!(template.signature().parameters().len(), 0);
         expect_macro(
@@ -683,7 +686,7 @@ mod tests {
 
         let expression = "(macro foo (x y z) [100, [200, a::b::300], x, {y: [true, false, z]}])";
 
-        let template = TemplateCompiler::compile_from_text(context, expression)?;
+        let template = TemplateCompiler::compile_from_text(context.get_ref(), expression)?;
         expect_value(&template, 0, TemplateValue::List(ExprRange::new(1..12)))?;
         expect_value(&template, 1, TemplateValue::Int(Int::from(100)))?;
         expect_value(&template, 2, TemplateValue::List(ExprRange::new(3..5)))?;
@@ -713,7 +716,7 @@ mod tests {
 
         let expression = "(macro identity (x) x)";
 
-        let template = TemplateCompiler::compile_from_text(context, expression)?;
+        let template = TemplateCompiler::compile_from_text(context.get_ref(), expression)?;
         assert_eq!(template.name(), "identity");
         assert_eq!(template.signature().parameters().len(), 1);
         expect_variable(&template, 0, 0)?;
@@ -736,7 +739,7 @@ mod tests {
                         (values x))))
         "#;
 
-        let template = TemplateCompiler::compile_from_text(context, expression)?;
+        let template = TemplateCompiler::compile_from_text(context.get_ref(), expression)?;
         assert_eq!(template.name(), "foo");
         assert_eq!(template.signature().parameters().len(), 1);
         // Outer `values`

src/lazy/expanded/e_expression.rs

@@ -5,20 +5,23 @@ use crate::lazy::decoder::{LazyDecoder, LazyRawValueExpr};
 use crate::lazy::encoding::TextEncoding_1_1;
 use crate::lazy::expanded::macro_evaluator::{MacroExpr, RawEExpression, ValueExpr};
 use crate::lazy::expanded::macro_table::MacroRef;
-use crate::lazy::expanded::{EncodingContext, LazyExpandedValue};
+use crate::lazy::expanded::{EncodingContextRef, LazyExpandedValue};
 use crate::lazy::text::raw::v1_1::reader::MacroIdRef;
 use crate::IonResult;
 use std::fmt::{Debug, Formatter};
 
 /// An e-expression (in Ion format `D`) that has been resolved in the current encoding context.
 #[derive(Copy, Clone)]
 pub struct EExpression<'top, D: LazyDecoder> {
-    pub(crate) context: EncodingContext<'top>,
+    pub(crate) context: EncodingContextRef<'top>,
     pub(crate) raw_invocation: D::EExp<'top>,
     pub(crate) invoked_macro: MacroRef<'top>,
 }
 
 impl<'top, D: LazyDecoder> EExpression<'top, D> {
+    pub fn context(&self) -> EncodingContextRef<'top> {
+        self.context
+    }
     pub fn raw_invocation(&self) -> D::EExp<'top> {
         self.raw_invocation
     }
@@ -35,7 +38,7 @@ impl<'top, D: LazyDecoder> Debug for EExpression<'top, D> {
 
 impl<'top, D: LazyDecoder> EExpression<'top, D> {
     pub fn new(
-        context: EncodingContext<'top>,
+        context: EncodingContextRef<'top>,
         raw_invocation: D::EExp<'top>,
         invoked_macro: MacroRef<'top>,
     ) -> Self {
@@ -67,7 +70,7 @@ impl<'top, D: LazyDecoder> From<EExpression<'top, D>> for MacroExpr<'top, D> {
 }
 
 pub struct EExpressionArgsIterator<'top, D: LazyDecoder> {
-    context: EncodingContext<'top>,
+    context: EncodingContextRef<'top>,
     raw_args: <D::EExp<'top> as RawEExpression<'top, D>>::RawArgumentsIterator<'top>,
 }