diff --git a/components/image-credit-overlay.tsx b/components/image-credit-overlay.tsx new file mode 100644 index 000000000..72f45658e --- /dev/null +++ b/components/image-credit-overlay.tsx @@ -0,0 +1,94 @@ +/* eslint-disable @next/next/no-img-element */ +'use client' + +import { displayUrlName } from '@/lib/utils/domain' + +/** + * Normalize a user- or model-supplied URL to a safe http(s) URL, or return + * null if the scheme is not allowed. Prevents XSS via javascript:/data: URLs + * embedded in model-generated spec blocks. + */ +const sanitizeHttpUrl = (raw: string | undefined): string | null => { + if (!raw) return null + try { + const parsed = new URL(raw) + if (parsed.protocol === 'http:' || parsed.protocol === 'https:') { + return parsed.toString() + } + return null + } catch { + return null + } +} + +export const getFaviconUrl = (imageUrl: string): string => { + try { + const hostname = new URL(imageUrl).hostname + return `https://www.google.com/s2/favicons?domain=${hostname}&sz=128` + } catch { + return '' + } +} + +type ImageCreditOverlayProps = { + /** Fallback link URL (e.g. the image URL itself). Used when sourceUrl is not set. */ + url: string + /** Original referring page URL; preferred for link/favicon/hostname when present. */ + sourceUrl?: string + title?: string + description?: string +} + +export function ImageCreditOverlay({ + url, + sourceUrl, + title, + description +}: ImageCreditOverlayProps) { + const safeLink = sanitizeHttpUrl(sourceUrl) ?? sanitizeHttpUrl(url) + const label = title || description + + const content = ( + <> + {safeLink && ( + + )} +
+ {safeLink && ( +
+ {displayUrlName(safeLink)} +
+ )} + {label && ( +
+ {label} +
+ )} +
+ + ) + + const className = + 'absolute bottom-3 left-3 max-w-[80%] bg-black/70 backdrop-blur-sm rounded-xl px-3 py-2.5 flex items-center gap-2.5 no-underline hover:bg-black/80 transition-colors' + + if (!safeLink) { + // Untrusted or missing URL: render a non-clickable label instead of an . + return
{content}
+ } + + return ( + e.stopPropagation()} + > + {content} + + ) +} diff --git a/components/search-results-image.tsx b/components/search-results-image.tsx index 8839b0df9..611971f7a 100644 --- a/components/search-results-image.tsx +++ b/components/search-results-image.tsx @@ -14,7 +14,6 @@ import { import { Images } from 'lucide-react' import { SearchResultImage } from '@/lib/types' -import { displayUrlName } from '@/lib/utils/domain' import { Carousel, @@ -33,6 +32,8 @@ import { DialogTrigger } from '@/components/ui/dialog' +import { ImageCreditOverlay } from '@/components/image-credit-overlay' + interface SearchResultsImageSectionProps { images: SearchResultImage[] query?: string @@ -226,15 +227,6 @@ const useCarouselMetrics = ({ return { current: currentValue } } -const getFaviconUrl = (imageUrl: string): string => { - try { - const hostname = new URL(imageUrl).hostname - return `https://www.google.com/s2/favicons?domain=${hostname}&sz=128` - } catch { - return '' - } -} - const cornerClassForIndex = (actualIndex: number, isFullMode: boolean) => { if (!isFullMode) { return 'rounded-lg' @@ -351,29 +343,10 @@ export const SearchResultsImageSection: React.FC< className="max-w-full max-h-[60vh] object-contain" onError={() => removeImage(img.id)} /> - e.stopPropagation()} - > - -
-
- {displayUrlName(img.url)} -
- {img.description && ( -
- {img.description} -
- )} -
- + ))} diff --git a/components/ui/dialog.tsx b/components/ui/dialog.tsx index 229330c14..dede5b1e6 100644 --- a/components/ui/dialog.tsx +++ b/components/ui/dialog.tsx @@ -45,7 +45,7 @@ const DialogContent = React.forwardRef< {...props} > {children} - + Close diff --git a/lib/agents/prompts/search-mode-prompts.ts b/lib/agents/prompts/search-mode-prompts.ts index eb68b0b5a..2bb1e8d00 100644 --- a/lib/agents/prompts/search-mode-prompts.ts +++ b/lib/agents/prompts/search-mode-prompts.ts @@ -1,4 +1,7 @@ -import { getRelatedQuestionsSpecPrompt } from '@/lib/render/prompt' +import { + getImageSpecPrompt, + getRelatedQuestionsSpecPrompt +} from '@/lib/render/prompt' import { getContentTypesGuidance, isGeneralSearchProviderAvailable @@ -139,6 +142,8 @@ Example approach: End with a synthesizing conclusion that ties the main points together into a clear overall picture. +${getImageSpecPrompt()} + ${getRelatedQuestionsSpecPrompt()} ` } @@ -311,6 +316,8 @@ Flexible example: Conclude with a brief synthesis that ties together the main insights into a clear overall understanding. +${getImageSpecPrompt()} + ${getRelatedQuestionsSpecPrompt()} ` } diff --git a/lib/render/__tests__/llm-image-output.e2e.test.ts b/lib/render/__tests__/llm-image-output.e2e.test.ts new file mode 100644 index 000000000..cced67595 --- /dev/null +++ b/lib/render/__tests__/llm-image-output.e2e.test.ts @@ -0,0 +1,125 @@ +/** + * E2E test: verify that an LLM actually emits well-formed inline image + * spec blocks when given the researcher system prompt and a mocked search + * tool returning image results. + * + * This test hits a real LLM API and is therefore gated behind the + * `RUN_LLM_E2E=1` environment variable. CI runs without the flag and the + * test is skipped. To run locally: + * + * RUN_LLM_E2E=1 OPENAI_API_KEY=sk-... bun run test llm-image-output.e2e + */ +import { generateText, stepCountIs, tool } from 'ai' +import { describe, expect, test } from 'vitest' +import { z } from 'zod' + +import { getAdaptiveModePrompt } from '@/lib/agents/prompts/search-mode-prompts' +import { getModel } from '@/lib/utils/registry' + +import { parseSpecBlock } from '../parse-spec-block' + +const RUN = process.env.RUN_LLM_E2E === '1' +const MODEL = process.env.LLM_E2E_MODEL || 'openai:gpt-4o-mini' + +const FIXTURE_IMAGES = [ + { + url: 'https://upload.wikimedia.org/wikipedia/commons/thumb/1/1c/MtFuji_FujiCity.jpg/1280px-MtFuji_FujiCity.jpg', + description: 'Mount Fuji seen from Fuji City' + }, + { + url: 'https://upload.wikimedia.org/wikipedia/commons/thumb/d/da/070722_Mt.Fuji_Yoshidaguchi_Trail.jpg/1280px-070722_Mt.Fuji_Yoshidaguchi_Trail.jpg', + description: 'Yoshida trail on Mount Fuji' + }, + { + url: 'https://upload.wikimedia.org/wikipedia/commons/thumb/5/5b/MtFuji_FujiCity_winter.jpg/1280px-MtFuji_FujiCity_winter.jpg', + description: 'Mount Fuji in winter' + } +] + +const FIXTURE_RESULTS = [ + { + title: 'Mount Fuji - Wikipedia', + url: 'https://en.wikipedia.org/wiki/Mount_Fuji', + content: + 'Mount Fuji is the tallest mountain in Japan, standing 3,776 meters. It is an active stratovolcano located on Honshu island.' + } +] + +function createMockSearchTool() { + return tool({ + description: + 'Search the web for information. Returns text results and related images.', + inputSchema: z.object({ + query: z.string(), + type: z.enum(['optimized', 'general']).optional(), + max_results: z.number().optional() + }), + async execute({ query }) { + return { + state: 'complete' as const, + query, + results: FIXTURE_RESULTS, + images: FIXTURE_IMAGES, + number_of_results: FIXTURE_RESULTS.length, + toolCallId: 'mock-search-1' + } + } + }) +} + +function extractSpecBlocks(markdown: string): string[] { + const blocks: string[] = [] + const regex = /```spec\n([\s\S]*?)```/g + let match: RegExpExecArray | null + while ((match = regex.exec(markdown)) !== null) { + blocks.push(match[1]) + } + return blocks +} + +describe.skipIf(!RUN)('LLM inline image output (E2E)', () => { + test('LLM emits inline image spec block using search tool images', async () => { + const { text } = await generateText({ + model: getModel(MODEL), + system: getAdaptiveModePrompt(), + tools: { + search: createMockSearchTool() + }, + stopWhen: stepCountIs(6), + prompt: + 'Show me photos of Mount Fuji and a brief description. Please include relevant images inline.' + }) + + console.log('[E2E] Model output:\n', text) + + const blocks = extractSpecBlocks(text) + expect(blocks.length).toBeGreaterThanOrEqual(1) + + // Parse every spec block and check at least one contains an Image. + const fixtureSrcs = new Set(FIXTURE_IMAGES.map(i => i.url)) + let imageCount = 0 + let hasRelatedQuestions = false + + for (const source of blocks) { + const spec = parseSpecBlock(source) + for (const el of Object.values(spec.elements) as Array<{ + type: string + props: Record + }>) { + if (el.type === 'Image') { + imageCount++ + const src = el.props.src as string + // src MUST be one of the fixture image URLs — no fabrication. + expect(fixtureSrcs.has(src)).toBe(true) + } + if (el.type === 'QuestionButton') { + hasRelatedQuestions = true + } + } + } + + expect(imageCount).toBeGreaterThanOrEqual(1) + // The related-questions block must still be emitted alongside images. + expect(hasRelatedQuestions).toBe(true) + }, 120_000) +}) diff --git a/lib/render/__tests__/parse-spec-block.test.ts b/lib/render/__tests__/parse-spec-block.test.ts index c8ad23a05..5d88d500d 100644 --- a/lib/render/__tests__/parse-spec-block.test.ts +++ b/lib/render/__tests__/parse-spec-block.test.ts @@ -114,4 +114,33 @@ describe('parseSpecBlock', () => { expect(() => parseSpecBlock(source)).toThrow() }) + + test('parses a Grid image group with Image children', () => { + const source = [ + '{"op":"add","path":"/root","value":"grid"}', + '{"op":"add","path":"/elements/grid","value":{"type":"Grid","props":{"columns":2,"gap":"sm"},"children":["img1","img2"]}}', + '{"op":"add","path":"/elements/img1","value":{"type":"Image","props":{"src":"https://example.com/a.jpg","title":"A","description":"Alpha","aspectRatio":"4:3"},"children":[]}}', + '{"op":"add","path":"/elements/img2","value":{"type":"Image","props":{"src":"https://example.com/b.jpg","title":"B","aspectRatio":"4:3"},"children":[]}}' + ].join('\n') + + const spec = parseSpecBlock(source) + expect(spec.root).toBe('grid') + expect(spec.elements['grid'].type).toBe('Grid') + expect(spec.elements['grid'].props).toMatchObject({ + columns: 2, + gap: 'sm' + }) + expect(spec.elements['img1'].type).toBe('Image') + expect(spec.elements['img1'].props).toMatchObject({ + src: 'https://example.com/a.jpg', + title: 'A', + description: 'Alpha', + aspectRatio: '4:3' + }) + expect(spec.elements['img2'].props).toMatchObject({ + src: 'https://example.com/b.jpg', + title: 'B', + aspectRatio: '4:3' + }) + }) }) diff --git a/lib/render/__tests__/strip-spec-blocks.test.ts b/lib/render/__tests__/strip-spec-blocks.test.ts new file mode 100644 index 000000000..2f3b90ed8 --- /dev/null +++ b/lib/render/__tests__/strip-spec-blocks.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, test } from 'vitest' + +import { stripSpecBlocks } from '../strip-spec-blocks' + +describe('stripSpecBlocks', () => { + test('removes a single spec block', () => { + const input = [ + '## Heading', + '', + '```spec', + '{"op":"add","path":"/root","value":"x"}', + '```', + '', + 'tail' + ].join('\n') + + const result = stripSpecBlocks(input) + expect(result).not.toContain('```spec') + expect(result).toContain('## Heading') + expect(result).toContain('tail') + }) + + test('removes multiple spec blocks interleaved with markdown', () => { + const input = [ + '## Section A', + '', + '```spec', + '{"op":"add","path":"/root","value":"imgs1"}', + '```', + '', + 'Body paragraph between blocks.', + '', + '```spec', + '{"op":"add","path":"/root","value":"imgs2"}', + '```', + '', + 'Closing.', + '', + '```spec', + '{"op":"add","path":"/root","value":"related"}', + '```' + ].join('\n') + + const result = stripSpecBlocks(input) + expect(result).not.toContain('```spec') + expect(result).not.toContain('imgs1') + expect(result).not.toContain('imgs2') + expect(result).not.toContain('related') + expect(result).toContain('## Section A') + expect(result).toContain('Body paragraph between blocks.') + expect(result).toContain('Closing.') + }) + + test('leaves non-spec fenced blocks intact', () => { + const input = [ + '```ts', + 'const x = 1', + '```', + '', + '```spec', + '{"op":"add","path":"/root","value":"r"}', + '```' + ].join('\n') + + const result = stripSpecBlocks(input) + expect(result).toContain('```ts') + expect(result).toContain('const x = 1') + expect(result).not.toContain('```spec') + }) +}) diff --git a/lib/render/catalog.ts b/lib/render/catalog.ts index b1542a619..346fede64 100644 --- a/lib/render/catalog.ts +++ b/lib/render/catalog.ts @@ -26,6 +26,30 @@ export const catalog = schema.createCatalog({ text: z.string() }), description: 'A related follow-up question the user can click to ask' + }, + Grid: { + props: z.object({ + columns: z.union([ + z.literal(1), + z.literal(2), + z.literal(3), + z.literal(4) + ]), + gap: z.enum(['xs', 'sm', 'md', 'lg']).optional() + }), + description: + 'A fixed-column CSS grid container. Used to lay out images so that cell widths are determined upfront.' + }, + Image: { + props: z.object({ + src: z.string(), + sourceUrl: z.string().optional(), + title: z.string().optional(), + description: z.string().optional(), + aspectRatio: z.enum(['1:1', '16:9', '4:3']).optional() + }), + description: + 'An inline image referencing a web source; click to expand with credit' } }, actions: { diff --git a/lib/render/components/grid.tsx b/lib/render/components/grid.tsx new file mode 100644 index 000000000..561700138 --- /dev/null +++ b/lib/render/components/grid.tsx @@ -0,0 +1,23 @@ +'use client' + +import type { ComponentFn } from '@json-render/react' + +import { cn } from '@/lib/utils' + +import { type CatalogType, stackGap } from './shared' + +const gridCols = { + 1: 'grid-cols-1', + 2: 'grid-cols-2', + 3: 'grid-cols-3', + 4: 'grid-cols-4' +} as const + +export const Grid: ComponentFn = ({ props, children }) => { + const { columns, gap = 'sm' } = props + return ( +
+ {children} +
+ ) +} diff --git a/lib/render/components/image.tsx b/lib/render/components/image.tsx new file mode 100644 index 000000000..414566703 --- /dev/null +++ b/lib/render/components/image.tsx @@ -0,0 +1,76 @@ +/* eslint-disable @next/next/no-img-element */ +'use client' + +import { useState } from 'react' + +import type { ComponentFn } from '@json-render/react' + +import { cn } from '@/lib/utils' + +import { + Dialog, + DialogContent, + DialogHeader, + DialogTitle, + DialogTrigger +} from '@/components/ui/dialog' + +import { ImageCreditOverlay } from '@/components/image-credit-overlay' + +import type { CatalogType } from './shared' + +const aspectRatioClass = { + '1:1': 'aspect-square', + '16:9': 'aspect-video', + '4:3': 'aspect-[4/3]' +} as const + +export const Image: ComponentFn = ({ props }) => { + const { src, sourceUrl, title, description, aspectRatio = '4:3' } = props + const [errored, setErrored] = useState(false) + + if (!src || errored) { + return null + } + + const alt = title || description || 'Image' + + return ( + + + + + + + {alt} + +
+ {alt} + +
+ + + ) +} diff --git a/lib/render/components/question-button.tsx b/lib/render/components/question-button.tsx new file mode 100644 index 000000000..bc56c76b4 --- /dev/null +++ b/lib/render/components/question-button.tsx @@ -0,0 +1,25 @@ +'use client' + +import type { ComponentFn } from '@json-render/react' +import { ArrowRight } from 'lucide-react' + +import type { CatalogType } from './shared' + +export const QuestionButton: ComponentFn = ({ + props, + on +}) => { + const handle = on('press') + return ( +
+ + +
+ ) +} diff --git a/lib/render/components/section-header.tsx b/lib/render/components/section-header.tsx new file mode 100644 index 000000000..84e4df6d6 --- /dev/null +++ b/lib/render/components/section-header.tsx @@ -0,0 +1,22 @@ +'use client' + +import type { ComponentFn } from '@json-render/react' +import { Repeat2 } from 'lucide-react' + +import type { CatalogType } from './shared' + +const iconMap = { + related: Repeat2 +} as const + +export const SectionHeader: ComponentFn = ({ + props +}) => { + const Icon = props.icon ? iconMap[props.icon] : null + return ( +
+ {Icon && } + {props.title} +
+ ) +} diff --git a/lib/render/components/shared.ts b/lib/render/components/shared.ts new file mode 100644 index 000000000..562699cef --- /dev/null +++ b/lib/render/components/shared.ts @@ -0,0 +1,10 @@ +import { catalog } from '../catalog' + +export type CatalogType = typeof catalog + +export const stackGap = { + xs: 'gap-1', + sm: 'gap-2', + md: 'gap-3', + lg: 'gap-4' +} as const diff --git a/lib/render/components/stack.tsx b/lib/render/components/stack.tsx new file mode 100644 index 000000000..ce4023e58 --- /dev/null +++ b/lib/render/components/stack.tsx @@ -0,0 +1,25 @@ +'use client' + +import type { ComponentFn } from '@json-render/react' + +import { cn } from '@/lib/utils' + +import { type CatalogType, stackGap } from './shared' + +export const Stack: ComponentFn = ({ + props, + children +}) => { + const { direction = 'vertical', gap = 'md' } = props + return ( +
+ {children} +
+ ) +} diff --git a/lib/render/prompt.ts b/lib/render/prompt.ts index 4b37675de..4dcdd57b7 100644 --- a/lib/render/prompt.ts +++ b/lib/render/prompt.ts @@ -33,11 +33,64 @@ AVAILABLE ACTIONS: - submitQuery: { query: string } - Submit a follow-up query SPEC RULES: -1. Always wrap JSONL patches in a single \`\`\`spec fence at the END of your response. -2. The \`\`\`spec block must contain ONLY JSONL patches — no commentary inside. +1. The related questions \`\`\`spec fence must appear at the END of your response. +2. Every \`\`\`spec block must contain ONLY JSONL patches — no commentary inside. 3. Keep each JSON object on a single line. 4. The "text" prop and "query" param must be identical for each question. -5. Do NOT open more than one \`\`\`spec block in a single answer. +5. Emit exactly ONE related questions spec block per answer (image spec blocks are separate and may appear inline). 6. Do NOT include follow-up suggestions or questions in your markdown text. Only use the spec block for them. ` } + +/** + * Returns the prompt section that instructs the LLM to optionally embed + * inline image groups as ```spec fenced blocks within the response body. + */ +export function getImageSpecPrompt(): string { + return ` +INLINE IMAGE EMBEDDING: +When images would meaningfully enhance the answer (e.g. visual subjects, people, places, products, events), +embed one or more inline image groups anywhere in the markdown body using \`\`\`spec fenced code blocks. +Actively include images whenever they help the reader's understanding of the answer — visual context often +communicates faster and more clearly than prose. + +Use this when: +- The search tool results contain relevant images (check the "images" array in tool output). +- A picture helps the reader understand the topic (subjects, places, products, events, diagrams, comparisons). + +Skip images only for purely abstract or text-only topics where no image would add value. + +AVAILABLE COMPONENTS FOR IMAGES: +- Grid: { columns: 1 | 2 | 3 | 4, gap?: "xs" | "sm" | "md" | "lg" } - A fixed-column container that reserves cell widths upfront so streaming images don't reflow. +- Image: { src: string, sourceUrl?: string, title?: string, description?: string, aspectRatio?: "1:1" | "16:9" | "4:3" } + +IMAGE SPEC RULES: +1. Only use image URLs taken verbatim from the search tool's "images" array — NEVER fabricate or guess URLs. +2. Map tool output fields to Image props as follows, copying values EXACTLY without rewording: + - image.url → "src" + - image.sourceUrl → "sourceUrl" (omit if not present in the tool output — do NOT invent) + - image.title → "title" (omit if not present) + - image.description → "description" (omit if not present) +3. The "aspectRatio" field SHOULD reflect the natural orientation of the subject: "1:1" for square (logos, portraits), "16:9" for wide (landscapes, scenes), "4:3" for standard photos. Images within the same Grid should generally use the SAME aspectRatio so they render at identical heights. +4. Always wrap image groups in a Grid. Set "columns" to the exact number of Image children (1–4). For 1 image use columns=1, for 2 use columns=2, etc. Choose the number of images based on the situation — the variety and relevance of available images and how much visual context genuinely helps the answer. +5. You MAY emit multiple \`\`\`spec image blocks, each placed at the position in the markdown where they are contextually relevant (e.g. right after the heading or paragraph they illustrate). +6. Image spec blocks are separate from the mandatory related-questions spec block at the end. +7. Each image spec block must contain ONLY JSONL patches — no commentary inside. + +Example (inline image group embedded in markdown body): + +## Mount Fuji + +Mount Fuji is Japan's tallest peak. + +\`\`\`spec +{"op":"add","path":"/root","value":"grid"} +{"op":"add","path":"/elements/grid","value":{"type":"Grid","props":{"columns":3,"gap":"sm"},"children":["img1","img2","img3"]}} +{"op":"add","path":"/elements/img1","value":{"type":"Image","props":{"src":"https://cdn.example.com/fuji-1.jpg","sourceUrl":"https://en.wikipedia.org/wiki/Mount_Fuji","title":"Mount Fuji - Wikipedia","description":"Snow-capped peak at sunrise","aspectRatio":"4:3"},"children":[]}} +{"op":"add","path":"/elements/img2","value":{"type":"Image","props":{"src":"https://cdn.example.com/fuji-2.jpg","sourceUrl":"https://travel.example.com/mount-fuji","title":"Mount Fuji Travel Guide","aspectRatio":"4:3"},"children":[]}} +{"op":"add","path":"/elements/img3","value":{"type":"Image","props":{"src":"https://cdn.example.com/fuji-3.jpg","title":"Cherry blossoms in spring","aspectRatio":"4:3"},"children":[]}} +\`\`\` + +It rises 3,776 meters above sea level... +` +} diff --git a/lib/render/registry.tsx b/lib/render/registry.tsx index bf6b6d605..c8233528f 100644 --- a/lib/render/registry.tsx +++ b/lib/render/registry.tsx @@ -1,77 +1,21 @@ 'use client' -import type { ComponentFn } from '@json-render/react' import { defineRegistry } from '@json-render/react' -import { ArrowRight, Repeat2 } from 'lucide-react' - -import { cn } from '@/lib/utils' +import { Grid } from './components/grid' +import { Image } from './components/image' +import { QuestionButton } from './components/question-button' +import { SectionHeader } from './components/section-header' +import { Stack } from './components/stack' import { catalog } from './catalog' -type CatalogType = typeof catalog - -const iconMap = { - related: Repeat2 -} as const - -const SectionHeader: ComponentFn = ({ - props -}) => { - const Icon = props.icon ? iconMap[props.icon] : null - return ( -
- {Icon && } - {props.title} -
- ) -} - -const stackGap = { - xs: 'gap-1', - sm: 'gap-2', - md: 'gap-3', - lg: 'gap-4' -} - -const Stack: ComponentFn = ({ props, children }) => { - const { direction = 'vertical', gap = 'md' } = props - return ( -
- {children} -
- ) -} - -const QuestionButton: ComponentFn = ({ - props, - on -}) => { - const handle = on('press') - return ( -
- - -
- ) -} - export const { registry } = defineRegistry(catalog, { components: { SectionHeader, Stack, - QuestionButton + QuestionButton, + Grid, + Image }, actions: { submitQuery: async () => { diff --git a/lib/tools/search/providers/tavily.ts b/lib/tools/search/providers/tavily.ts index 3cff48f3f..1f65c988c 100644 --- a/lib/tools/search/providers/tavily.ts +++ b/lib/tools/search/providers/tavily.ts @@ -1,4 +1,4 @@ -import { SearchResultImage, SearchResults } from '@/lib/types' +import { SearchResults } from '@/lib/types' import { sanitizeUrl } from '@/lib/utils' import { BaseSearchProvider } from './base' @@ -45,18 +45,41 @@ export class TavilySearchProvider extends BaseSearchProvider { } const data = await response.json() + + // Tavily returns top-level images with { url, title?, description? }. We try + // to match each image to a result by title so the UI can link back to the + // original article rather than just the image host. + const resultTitleToUrl = new Map() + for (const r of (data.results ?? []) as Array<{ + title?: string + url?: string + }>) { + if (r.title && r.url) { + resultTitleToUrl.set(r.title, r.url) + } + } + const processedImages = includeImageDescriptions - ? data.images - .map( - ({ url, description }: { url: string; description: string }) => ({ - url: sanitizeUrl(url), - description - }) - ) + ? ( + data.images as Array<{ + url: string + title?: string + description?: string + }> + ) + .map(image => { + const sourceUrl = image.title + ? resultTitleToUrl.get(image.title) + : undefined + return { + url: sanitizeUrl(image.url), + description: image.description ?? '', + ...(image.title ? { title: image.title } : {}), + ...(sourceUrl ? { sourceUrl } : {}) + } + }) .filter( - ( - image: SearchResultImage - ): image is { url: string; description: string } => + (image): image is { url: string; description: string } => typeof image === 'object' && image.description !== undefined && image.description !== '' diff --git a/lib/types/index.ts b/lib/types/index.ts index 43676fe71..f570d1df7 100644 --- a/lib/types/index.ts +++ b/lib/types/index.ts @@ -11,13 +11,16 @@ export type SearchResults = { citationMap?: Record // Maps citation number to search result } -// If enabled the include_images_description is true, the images will be an array of { url: string, description: string } -// Otherwise, the images will be an array of strings +// If include_images_description is true, images are objects with url/description. +// When the provider can resolve the referring page, sourceUrl and title are also set. +// Otherwise, the images are an array of strings. export type SearchResultImage = | string | { url: string description: string + title?: string + sourceUrl?: string number_of_results?: number }