Add original-resolution view_image support

git-stack-id: fjord/original_image_res---4hw7wu3447irye
git-stack-title: Add original-resolution view_image support

Author: fjord-oai

codex-rs/Cargo.lock

@@ -399,7 +399,10 @@ async fn dynamic_tool_call_round_trip_sends_content_items_to_model() -> Result<(
                 FunctionCallOutputContentItem::InputText { text }
             }
             DynamicToolCallOutputContentItem::InputImage { image_url } => {
-                FunctionCallOutputContentItem::InputImage { image_url }
+                FunctionCallOutputContentItem::InputImage {
+                    image_url,
+                    detail: None,
+                }
             }
         })
         .collect::<Vec<FunctionCallOutputContentItem>>();

codex-rs/app-server/tests/suite/v2/dynamic_tools.rs

@@ -46,6 +46,7 @@ codex-protocol = { workspace = true }
 codex-rmcp-client = { workspace = true }
 codex-state = { workspace = true }
 codex-utils-absolute-path = { workspace = true }
+codex-utils-image = { workspace = true }
 codex-utils-home-dir = { workspace = true }
 codex-utils-pty = { workspace = true }
 codex-utils-readiness = { workspace = true }
@@ -62,6 +63,7 @@ eventsource-stream = { workspace = true }
 futures = { workspace = true }
 http = { workspace = true }
 iana-time-zone = { workspace = true }
+image = { workspace = true, features = ["jpeg", "png"] }
 indexmap = { workspace = true }
 keyring = { workspace = true, features = ["crypto-rust"] }
 libc = { workspace = true }
@@ -86,7 +88,6 @@ sha2 = { workspace = true }
 shlex = { workspace = true }
 similar = { workspace = true }
 tempfile = { workspace = true }
-test-case = "3.3.1"
 test-log = { workspace = true }
 thiserror = { workspace = true }
 time = { workspace = true, features = [
@@ -155,11 +156,11 @@ codex-test-macros = { workspace = true }
 codex-utils-cargo-bin = { workspace = true }
 core_test_support = { workspace = true }
 ctor = { workspace = true }
-image = { workspace = true, features = ["jpeg", "png"] }
 insta = { workspace = true }
 maplit = { workspace = true }
 predicates = { workspace = true }
 pretty_assertions = { workspace = true }
+test-case = "3.3.1"
 opentelemetry_sdk = { workspace = true, features = [
     "experimental_metrics_custom_reader",
     "metrics",

codex-rs/core/Cargo.toml

@@ -430,6 +430,9 @@
             "use_linux_sandbox_bwrap": {
               "type": "boolean"
             },
+            "view_image_original_resolution": {
+              "type": "boolean"
+            },
             "voice_transcription": {
               "type": "boolean"
             },
@@ -1761,6 +1764,9 @@
         "use_linux_sandbox_bwrap": {
           "type": "boolean"
         },
+        "view_image_original_resolution": {
+          "type": "boolean"
+        },
         "voice_transcription": {
           "type": "boolean"
         },

codex-rs/core/config.schema.json

@@ -6,11 +6,14 @@ use crate::truncate::approx_token_count;
 use crate::truncate::approx_tokens_from_byte_count_i64;
 use crate::truncate::truncate_function_output_items_with_policy;
 use crate::truncate::truncate_text;
+use base64::Engine;
+use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
 use codex_protocol::models::BaseInstructions;
 use codex_protocol::models::ContentItem;
 use codex_protocol::models::FunctionCallOutputBody;
 use codex_protocol::models::FunctionCallOutputContentItem;
 use codex_protocol::models::FunctionCallOutputPayload;
+use codex_protocol::models::ImageDetail;
 use codex_protocol::models::ResponseItem;
 use codex_protocol::openai_models::InputModality;
 use codex_protocol::protocol::TokenUsage;
@@ -428,7 +431,15 @@ fn estimate_item_token_count(item: &ResponseItem) -> i64 {
 ///
 /// The estimator later converts bytes to tokens using a 4-bytes/token heuristic
 /// with ceiling division, so 7,373 bytes maps to approximately 1,844 tokens.
-const IMAGE_BYTES_ESTIMATE: i64 = 7373;
+const RESIZED_IMAGE_BYTES_ESTIMATE: i64 = 7373;
+// See https://developers.openai.com/api/docs/guides/images-vision#calculating-costs.
+// Use the documented GPT-5 high-detail calculation only for `detail: "original"`;
+// all other image inputs continue to use `RESIZED_IMAGE_BYTES_ESTIMATE`.
+const ORIGINAL_IMAGE_MAX_DIM: u32 = 2048;
+const ORIGINAL_IMAGE_TARGET_SHORT_SIDE: u32 = 768;
+const ORIGINAL_IMAGE_TILE_SIZE: u32 = 512;
+const ORIGINAL_IMAGE_BASE_TOKENS: i64 = 70;
+const ORIGINAL_IMAGE_TILE_TOKENS: i64 = 140;
 
 pub(crate) fn estimate_response_item_model_visible_bytes(item: &ResponseItem) -> i64 {
     match item {
@@ -444,15 +455,15 @@ pub(crate) fn estimate_response_item_model_visible_bytes(item: &ResponseItem) ->
             let raw = serde_json::to_string(item)
                 .map(|serialized| i64::try_from(serialized.len()).unwrap_or(i64::MAX))
                 .unwrap_or_default();
-            let (payload_bytes, image_count) = image_data_url_estimate_adjustment(item);
-            if payload_bytes == 0 || image_count == 0 {
+            let (payload_bytes, replacement_bytes) = image_data_url_estimate_adjustment(item);
+            if payload_bytes == 0 || replacement_bytes == 0 {
                 raw
             } else {
-                // Replace raw base64 payload bytes with a fixed per-image cost.
-                // We intentionally preserve the data URL prefix and JSON wrapper
-                // bytes already included in `raw`.
+                // Replace raw base64 payload bytes with a per-image estimate.
+                // We intentionally preserve the data URL prefix and JSON
+                // wrapper bytes already included in `raw`.
                 raw.saturating_sub(payload_bytes)
-                    .saturating_add(image_count.saturating_mul(IMAGE_BYTES_ESTIMATE))
+                    .saturating_add(replacement_bytes)
             }
         }
     }
@@ -463,7 +474,7 @@ pub(crate) fn estimate_response_item_model_visible_bytes(item: &ResponseItem) ->
 ///
 /// We only discount payloads for `data:image/...;base64,...` URLs (case
 /// insensitive markers) and leave everything else at raw serialized size.
-fn base64_data_url_payload_len(url: &str) -> Option<usize> {
+fn parse_base64_image_data_url(url: &str) -> Option<&str> {
     if !url
         .get(.."data:".len())
         .is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:"))
@@ -489,47 +500,94 @@ fn base64_data_url_payload_len(url: &str) -> Option<usize> {
     if !has_base64_marker {
         return None;
     }
-    Some(payload.len())
+    Some(payload)
+}
+
+fn base64_data_url_payload_len(url: &str) -> Option<usize> {
+    parse_base64_image_data_url(url).map(str::len)
+}
+
+fn estimate_original_image_bytes(image_url: &str) -> Option<i64> {
+    let payload = parse_base64_image_data_url(image_url)?;
+    let bytes = BASE64_STANDARD.decode(payload).ok()?;
+    let dynamic = image::load_from_memory(&bytes).ok()?;
+    let mut width = dynamic.width();
+    let mut height = dynamic.height();
+
+    if width > ORIGINAL_IMAGE_MAX_DIM || height > ORIGINAL_IMAGE_MAX_DIM {
+        let scale = f64::min(
+            f64::from(ORIGINAL_IMAGE_MAX_DIM) / f64::from(width),
+            f64::from(ORIGINAL_IMAGE_MAX_DIM) / f64::from(height),
+        );
+        width = (f64::from(width) * scale).round().max(1.0) as u32;
+        height = (f64::from(height) * scale).round().max(1.0) as u32;
+    }
+
+    let shortest_side = width.min(height);
+    if shortest_side > 0 {
+        let scale = f64::from(ORIGINAL_IMAGE_TARGET_SHORT_SIDE) / f64::from(shortest_side);
+        width = (f64::from(width) * scale).round().max(1.0) as u32;
+        height = (f64::from(height) * scale).round().max(1.0) as u32;
+    }
+
+    let tiles_wide = i64::from(
+        width.saturating_add(ORIGINAL_IMAGE_TILE_SIZE.saturating_sub(1)) / ORIGINAL_IMAGE_TILE_SIZE,
+    );
+    let tiles_high = i64::from(
+        height.saturating_add(ORIGINAL_IMAGE_TILE_SIZE.saturating_sub(1))
+            / ORIGINAL_IMAGE_TILE_SIZE,
+    );
+    let tile_count = tiles_wide.saturating_mul(tiles_high);
+    let tokens = ORIGINAL_IMAGE_BASE_TOKENS
+        .saturating_add(tile_count.saturating_mul(ORIGINAL_IMAGE_TILE_TOKENS));
+    Some(tokens.saturating_mul(4))
 }
 
 /// Scans one response item for discount-eligible inline image data URLs and
 /// returns:
 /// - total base64 payload bytes to subtract from raw serialized size
-/// - count of qualifying images to replace with `IMAGE_BYTES_ESTIMATE`
+/// - total replacement byte estimate for those images
 fn image_data_url_estimate_adjustment(item: &ResponseItem) -> (i64, i64) {
     let mut payload_bytes = 0i64;
-    let mut image_count = 0i64;
+    let mut replacement_bytes = 0i64;
 
-    let mut accumulate = |image_url: &str| {
+    let mut accumulate = |image_url: &str, detail: Option<ImageDetail>| {
         if let Some(payload_len) = base64_data_url_payload_len(image_url) {
             payload_bytes =
                 payload_bytes.saturating_add(i64::try_from(payload_len).unwrap_or(i64::MAX));
-            image_count = image_count.saturating_add(1);
+            replacement_bytes = replacement_bytes.saturating_add(match detail {
+                Some(ImageDetail::Original) => {
+                    estimate_original_image_bytes(image_url).unwrap_or(RESIZED_IMAGE_BYTES_ESTIMATE)
+                }
+                _ => RESIZED_IMAGE_BYTES_ESTIMATE,
+            });
         }
     };
 
     match item {
         ResponseItem::Message { content, .. } => {
             for content_item in content {
                 if let ContentItem::InputImage { image_url } = content_item {
-                    accumulate(image_url);
+                    accumulate(image_url, None);
                 }
             }
         }
         ResponseItem::FunctionCallOutput { output, .. }
         | ResponseItem::CustomToolCallOutput { output, .. } => {
             if let FunctionCallOutputBody::ContentItems(items) = &output.body {
                 for content_item in items {
-                    if let FunctionCallOutputContentItem::InputImage { image_url } = content_item {
-                        accumulate(image_url);
+                    if let FunctionCallOutputContentItem::InputImage { image_url, detail } =
+                        content_item
+                    {
+                        accumulate(image_url, *detail);
                     }
                 }
             }
         }
         _ => {}
     }
 
-    (payload_bytes, image_count)
+    (payload_bytes, replacement_bytes)
 }
 
 fn is_model_generated_item(item: &ResponseItem) -> bool {