feat: multimodal support (image) (#27793)

Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

web/models/common.ts

@@ -232,6 +232,9 @@ export type PluginProvider = {
 export type FileUploadConfigResponse = {
   batch_count_limit: number
   image_file_size_limit?: number | string // default is 10MB
+  image_file_batch_limit: number // default is 10, for dataset attachment upload only
+  single_chunk_attachment_limit: number // default is 10, for dataset attachment upload only
+  attachment_image_file_size_limit: number // default is 2MB, for dataset attachment upload only
   file_size_limit: number // default is 15MB
   audio_file_size_limit?: number // default is 50MB
   video_file_size_limit?: number // default is 100MB

web/models/datasets.ts

@@ -85,7 +85,8 @@ export type DataSet = {
   pipeline_id?: string
   is_published?: boolean // Indicates if the pipeline is published
   runtime_mode: 'rag_pipeline' | 'general'
-  enable_api: boolean
+  enable_api: boolean // Indicates if the service API is enabled
+  is_multimodal: boolean // Indicates if the dataset supports multimodal
 }
 
 export type ExternalAPIItem = {
@@ -541,6 +542,15 @@ export type SegmentsQuery = {
   enabled?: boolean | 'all'
 }
 
+export type Attachment = {
+  id: string
+  name: string
+  size: number
+  extension: string
+  mime_type: string
+  source_url: string
+}
+
 export type SegmentDetailModel = {
   id: string
   position: number
@@ -566,6 +576,7 @@ export type SegmentDetailModel = {
   answer?: string
   child_chunks?: ChildChunkDetail[]
   updated_at: number
+  attachments: Attachment[]
 }
 
 export type SegmentsResponse = {
@@ -577,14 +588,20 @@ export type SegmentsResponse = {
   page: number
 }
 
+export type Query = {
+  content: string
+  content_type: 'text_query' | 'image_query',
+  file_info: Attachment | null
+}
+
 export type HitTestingRecord = {
   id: string
-  content: string
   source: 'app' | 'hit_testing' | 'plugin'
   source_app_id: string
   created_by_role: 'account' | 'end_user'
   created_by: string
   created_at: number
+  queries: Query[]
 }
 
 export type HitTestingChildChunk = {
@@ -598,7 +615,8 @@ export type HitTesting = {
   content: Segment
   score: number
   tsne_position: TsnePosition
-  child_chunks?: HitTestingChildChunk[] | null
+  child_chunks: HitTestingChildChunk[] | null
+  files: Attachment[]
 }
 
 export type ExternalKnowledgeBaseHitTesting = {
@@ -680,6 +698,7 @@ export type SegmentUpdater = {
   answer?: string
   keywords?: string[]
   regenerate_child_chunks?: boolean
+  attachment_ids?: string[]
 }
 
 export type ErrorDocsResponse = {
@@ -814,3 +833,24 @@ export type IndexingStatusBatchRequest = {
   datasetId: string
   batchId: string
 }
+
+export type HitTestingRecordsRequest = {
+  datasetId: string
+  page: number
+  limit: number
+}
+
+export type HitTestingRequest = {
+  query: string
+  attachment_ids: string[]
+  retrieval_model: RetrievalConfig
+}
+
+export type ExternalKnowledgeBaseHitTestingRequest = {
+  query: string
+  external_retrieval_model: {
+    top_k: number
+    score_threshold: number
+    score_threshold_enabled: boolean
+  }
+}