Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 94 additions & 0 deletions components/image-credit-overlay.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
/* eslint-disable @next/next/no-img-element */
'use client'

import { displayUrlName } from '@/lib/utils/domain'

/**
* Normalize a user- or model-supplied URL to a safe http(s) URL, or return
* null if the scheme is not allowed. Prevents XSS via javascript:/data: URLs
* embedded in model-generated spec blocks.
*/
const sanitizeHttpUrl = (raw: string | undefined): string | null => {
if (!raw) return null
try {
const parsed = new URL(raw)
if (parsed.protocol === 'http:' || parsed.protocol === 'https:') {
return parsed.toString()
}
return null
} catch {
return null
}
}

export const getFaviconUrl = (imageUrl: string): string => {
try {
const hostname = new URL(imageUrl).hostname
return `https://www.google.com/s2/favicons?domain=${hostname}&sz=128`
} catch {
return ''
}
}

type ImageCreditOverlayProps = {
/** Fallback link URL (e.g. the image URL itself). Used when sourceUrl is not set. */
url: string
/** Original referring page URL; preferred for link/favicon/hostname when present. */
sourceUrl?: string
title?: string
description?: string
}

export function ImageCreditOverlay({
url,
sourceUrl,
title,
description
}: ImageCreditOverlayProps) {
const safeLink = sanitizeHttpUrl(sourceUrl) ?? sanitizeHttpUrl(url)
const label = title || description

const content = (
<>
{safeLink && (
<img
src={getFaviconUrl(safeLink)}
alt=""
className="size-7 rounded-lg shrink-0"
/>
)}
<div className="min-w-0 flex-1">
{safeLink && (
<div className="text-white/70 text-xs">
{displayUrlName(safeLink)}
</div>
)}
{label && (
<div className="text-white text-sm font-medium line-clamp-1">
{label}
</div>
)}
</div>
</>
)

const className =
'absolute bottom-3 left-3 max-w-[80%] bg-black/70 backdrop-blur-sm rounded-xl px-3 py-2.5 flex items-center gap-2.5 no-underline hover:bg-black/80 transition-colors'

if (!safeLink) {
// Untrusted or missing URL: render a non-clickable label instead of an <a>.
return <div className={className}>{content}</div>
}

return (
<a
href={safeLink}
target="_blank"
rel="noopener noreferrer"
className={className}
onClick={e => e.stopPropagation()}
>
{content}
</a>
)
}
39 changes: 6 additions & 33 deletions components/search-results-image.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ import {
import { Images } from 'lucide-react'

import { SearchResultImage } from '@/lib/types'
import { displayUrlName } from '@/lib/utils/domain'

import {
Carousel,
Expand All @@ -33,6 +32,8 @@ import {
DialogTrigger
} from '@/components/ui/dialog'

import { ImageCreditOverlay } from '@/components/image-credit-overlay'

interface SearchResultsImageSectionProps {
images: SearchResultImage[]
query?: string
Expand Down Expand Up @@ -226,15 +227,6 @@ const useCarouselMetrics = ({
return { current: currentValue }
}

const getFaviconUrl = (imageUrl: string): string => {
try {
const hostname = new URL(imageUrl).hostname
return `https://www.google.com/s2/favicons?domain=${hostname}&sz=128`
} catch {
return ''
}
}

const cornerClassForIndex = (actualIndex: number, isFullMode: boolean) => {
if (!isFullMode) {
return 'rounded-lg'
Expand Down Expand Up @@ -351,29 +343,10 @@ export const SearchResultsImageSection: React.FC<
className="max-w-full max-h-[60vh] object-contain"
onError={() => removeImage(img.id)}
/>
<a
href={img.url}
target="_blank"
rel="noopener noreferrer"
className="absolute bottom-3 left-3 max-w-[80%] bg-black/70 backdrop-blur-sm rounded-xl px-3 py-2.5 flex items-center gap-2.5 no-underline hover:bg-black/80 transition-colors"
onClick={e => e.stopPropagation()}
>
<img
src={getFaviconUrl(img.url)}
alt=""
className="size-7 rounded-lg shrink-0"
/>
<div className="min-w-0 flex-1">
<div className="text-white/70 text-xs">
{displayUrlName(img.url)}
</div>
{img.description && (
<div className="text-white text-sm font-medium line-clamp-1">
{img.description}
</div>
)}
</div>
</a>
<ImageCreditOverlay
url={img.url}
description={img.description}
/>
</div>
</CarouselItem>
))}
Expand Down
2 changes: 1 addition & 1 deletion components/ui/dialog.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ const DialogContent = React.forwardRef<
{...props}
>
{children}
<DialogPrimitive.Close className="absolute right-4 top-4 rounded-sm opacity-70 ring-offset-background transition-opacity hover:opacity-100 focus:outline-hidden focus:ring-2 focus:ring-ring focus:ring-offset-2 disabled:pointer-events-none data-[state=open]:bg-accent data-[state=open]:text-muted-foreground">
<DialogPrimitive.Close className="absolute right-4 top-4 rounded-sm opacity-70 transition-opacity hover:opacity-100 focus-visible:outline-hidden focus-visible:ring-1 focus-visible:ring-muted-foreground/40 focus-visible:ring-offset-0 disabled:pointer-events-none data-[state=open]:bg-accent data-[state=open]:text-muted-foreground">
<X className="h-4 w-4" />
<span className="sr-only">Close</span>
</DialogPrimitive.Close>
Expand Down
9 changes: 8 additions & 1 deletion lib/agents/prompts/search-mode-prompts.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
import { getRelatedQuestionsSpecPrompt } from '@/lib/render/prompt'
import {
getImageSpecPrompt,
getRelatedQuestionsSpecPrompt
} from '@/lib/render/prompt'
import {
getContentTypesGuidance,
isGeneralSearchProviderAvailable
Expand Down Expand Up @@ -139,6 +142,8 @@ Example approach:

End with a synthesizing conclusion that ties the main points together into a clear overall picture.

${getImageSpecPrompt()}

${getRelatedQuestionsSpecPrompt()}
`
}
Expand Down Expand Up @@ -311,6 +316,8 @@ Flexible example:

Conclude with a brief synthesis that ties together the main insights into a clear overall understanding.

${getImageSpecPrompt()}

${getRelatedQuestionsSpecPrompt()}
`
}
Expand Down
125 changes: 125 additions & 0 deletions lib/render/__tests__/llm-image-output.e2e.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
/**
* E2E test: verify that an LLM actually emits well-formed inline image
* spec blocks when given the researcher system prompt and a mocked search
* tool returning image results.
*
* This test hits a real LLM API and is therefore gated behind the
* `RUN_LLM_E2E=1` environment variable. CI runs without the flag and the
* test is skipped. To run locally:
*
* RUN_LLM_E2E=1 OPENAI_API_KEY=sk-... bun run test llm-image-output.e2e
*/
import { generateText, stepCountIs, tool } from 'ai'
import { describe, expect, test } from 'vitest'
import { z } from 'zod'

import { getAdaptiveModePrompt } from '@/lib/agents/prompts/search-mode-prompts'
import { getModel } from '@/lib/utils/registry'

import { parseSpecBlock } from '../parse-spec-block'

const RUN = process.env.RUN_LLM_E2E === '1'
const MODEL = process.env.LLM_E2E_MODEL || 'openai:gpt-4o-mini'

const FIXTURE_IMAGES = [
{
url: 'https://upload.wikimedia.org/wikipedia/commons/thumb/1/1c/MtFuji_FujiCity.jpg/1280px-MtFuji_FujiCity.jpg',
description: 'Mount Fuji seen from Fuji City'
},
{
url: 'https://upload.wikimedia.org/wikipedia/commons/thumb/d/da/070722_Mt.Fuji_Yoshidaguchi_Trail.jpg/1280px-070722_Mt.Fuji_Yoshidaguchi_Trail.jpg',
description: 'Yoshida trail on Mount Fuji'
},
{
url: 'https://upload.wikimedia.org/wikipedia/commons/thumb/5/5b/MtFuji_FujiCity_winter.jpg/1280px-MtFuji_FujiCity_winter.jpg',
description: 'Mount Fuji in winter'
}
]

const FIXTURE_RESULTS = [
{
title: 'Mount Fuji - Wikipedia',
url: 'https://en.wikipedia.org/wiki/Mount_Fuji',
content:
'Mount Fuji is the tallest mountain in Japan, standing 3,776 meters. It is an active stratovolcano located on Honshu island.'
}
]

function createMockSearchTool() {
return tool({
description:
'Search the web for information. Returns text results and related images.',
inputSchema: z.object({
query: z.string(),
type: z.enum(['optimized', 'general']).optional(),
max_results: z.number().optional()
}),
async execute({ query }) {
return {
state: 'complete' as const,
query,
results: FIXTURE_RESULTS,
images: FIXTURE_IMAGES,
number_of_results: FIXTURE_RESULTS.length,
toolCallId: 'mock-search-1'
}
}
})
}

function extractSpecBlocks(markdown: string): string[] {
const blocks: string[] = []
const regex = /```spec\n([\s\S]*?)```/g
let match: RegExpExecArray | null
while ((match = regex.exec(markdown)) !== null) {
blocks.push(match[1])
}
return blocks
}

describe.skipIf(!RUN)('LLM inline image output (E2E)', () => {
test('LLM emits inline image spec block using search tool images', async () => {
const { text } = await generateText({
model: getModel(MODEL),
system: getAdaptiveModePrompt(),
tools: {
search: createMockSearchTool()
},
stopWhen: stepCountIs(6),
prompt:
'Show me photos of Mount Fuji and a brief description. Please include relevant images inline.'
})

console.log('[E2E] Model output:\n', text)

const blocks = extractSpecBlocks(text)
expect(blocks.length).toBeGreaterThanOrEqual(1)

// Parse every spec block and check at least one contains an Image.
const fixtureSrcs = new Set(FIXTURE_IMAGES.map(i => i.url))
let imageCount = 0
let hasRelatedQuestions = false

for (const source of blocks) {
const spec = parseSpecBlock(source)
for (const el of Object.values(spec.elements) as Array<{
type: string
props: Record<string, unknown>
}>) {
if (el.type === 'Image') {
imageCount++
const src = el.props.src as string
// src MUST be one of the fixture image URLs — no fabrication.
expect(fixtureSrcs.has(src)).toBe(true)
}
if (el.type === 'QuestionButton') {
hasRelatedQuestions = true
}
}
}

expect(imageCount).toBeGreaterThanOrEqual(1)
// The related-questions block must still be emitted alongside images.
expect(hasRelatedQuestions).toBe(true)
}, 120_000)
})
29 changes: 29 additions & 0 deletions lib/render/__tests__/parse-spec-block.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -114,4 +114,33 @@ describe('parseSpecBlock', () => {

expect(() => parseSpecBlock(source)).toThrow()
})

test('parses a Grid image group with Image children', () => {
const source = [
'{"op":"add","path":"/root","value":"grid"}',
'{"op":"add","path":"/elements/grid","value":{"type":"Grid","props":{"columns":2,"gap":"sm"},"children":["img1","img2"]}}',
'{"op":"add","path":"/elements/img1","value":{"type":"Image","props":{"src":"https://example.com/a.jpg","title":"A","description":"Alpha","aspectRatio":"4:3"},"children":[]}}',
'{"op":"add","path":"/elements/img2","value":{"type":"Image","props":{"src":"https://example.com/b.jpg","title":"B","aspectRatio":"4:3"},"children":[]}}'
].join('\n')

const spec = parseSpecBlock(source)
expect(spec.root).toBe('grid')
expect(spec.elements['grid'].type).toBe('Grid')
expect(spec.elements['grid'].props).toMatchObject({
columns: 2,
gap: 'sm'
})
expect(spec.elements['img1'].type).toBe('Image')
expect(spec.elements['img1'].props).toMatchObject({
src: 'https://example.com/a.jpg',
title: 'A',
description: 'Alpha',
aspectRatio: '4:3'
})
expect(spec.elements['img2'].props).toMatchObject({
src: 'https://example.com/b.jpg',
title: 'B',
aspectRatio: '4:3'
})
})
})
Loading
Loading