Underground-Digital/Workflow-Engine

1import type { DataSourceNotionPage, DataSourceProvider } from './common'2import type { AppIconType, AppMode, RetrievalConfig } from '@/types/app'3import type { Tag } from '@/app/components/base/tag-management/constant'4 5export enum DataSourceType {6  FILE = 'upload_file',7  NOTION = 'notion_import',8  WEB = 'website_crawl',9}10 11export type DatasetPermission = 'only_me' | 'all_team_members' | 'partial_members'12 13export type DataSet = {14  id: string15  name: string16  icon: string17  icon_background: string18  description: string19  permission: DatasetPermission20  data_source_type: DataSourceType21  indexing_technique: 'high_quality' | 'economy'22  created_by: string23  updated_by: string24  updated_at: number25  app_count: number26  document_count: number27  word_count: number28  provider: string29  embedding_model: string30  embedding_model_provider: string31  embedding_available: boolean32  retrieval_model_dict: RetrievalConfig33  retrieval_model: RetrievalConfig34  tags: Tag[]35  partial_member_list?: any[]36  external_knowledge_info: {37    external_knowledge_id: string38    external_knowledge_api_id: string39    external_knowledge_api_name: string40    external_knowledge_api_endpoint: string41  }42  external_retrieval_model: {43    top_k: number44    score_threshold: number45    score_threshold_enabled: boolean46  }47}48 49export type ExternalAPIItem = {50  id: string51  tenant_id: string52  name: string53  description: string54  settings: {55    endpoint: string56    api_key: string57  }58  dataset_bindings: { id: string; name: string }[]59  created_by: string60  created_at: string61}62 63export type ExternalKnowledgeItem = {64  id: string65  name: string66  description: string | null67  provider: 'external'68  permission: DatasetPermission69  data_source_type: null70  indexing_technique: null71  app_count: number72  document_count: number73  word_count: number74  created_by: string75  created_at: string76  updated_by: string77  updated_at: string78  tags: Tag[]79}80 81export type ExternalAPIDeleteResponse = {82  result: 'success' | 'error'83}84 85export type ExternalAPIUsage = {86  is_using: boolean87  count: number88}89 90export type CustomFile = File & {91  id?: string92  extension?: string93  mime_type?: string94  created_by?: string95  created_at?: number96}97 98export type CrawlOptions = {99  crawl_sub_pages: boolean100  only_main_content: boolean101  includes: string102  excludes: string103  limit: number | string104  max_depth: number | string105  use_sitemap: boolean106}107 108export type CrawlResultItem = {109  title: string110  markdown: string111  description: string112  source_url: string113}114 115export type FileItem = {116  fileID: string117  file: CustomFile118  progress: number119}120 121export type DataSetListResponse = {122  data: DataSet[]123  has_more: boolean124  limit: number125  page: number126  total: number127}128 129export type ExternalAPIListResponse = {130  data: ExternalAPIItem[]131  has_more: boolean132  limit: number133  page: number134  total: number135}136 137export type QA = {138  question: string139  answer: string140}141 142export type IndexingEstimateResponse = {143  tokens: number144  total_price: number145  currency: string146  total_segments: number147  preview: string[]148  qa_preview?: QA[]149}150 151export type FileIndexingEstimateResponse = {152  total_nodes: number153} & IndexingEstimateResponse154 155export type IndexingStatusResponse = {156  id: string157  indexing_status: DocumentIndexingStatus158  processing_started_at: number159  parsing_completed_at: number160  cleaning_completed_at: number161  splitting_completed_at: number162  completed_at: any163  paused_at: any164  error: any165  stopped_at: any166  completed_segments: number167  total_segments: number168}169export type IndexingStatusBatchResponse = {170  data: IndexingStatusResponse[]171}172 173export type ProcessMode = 'automatic' | 'custom'174 175export type ProcessRuleResponse = {176  mode: ProcessMode177  rules: Rules178}179 180export type Rules = {181  pre_processing_rules: PreProcessingRule[]182  segmentation: Segmentation183}184 185export type PreProcessingRule = {186  id: string187  enabled: boolean188}189 190export type Segmentation = {191  separator: string192  max_tokens: number193  chunk_overlap: number194}195 196export const DocumentIndexingStatusList = [197  'waiting',198  'parsing',199  'cleaning',200  'splitting',201  'indexing',202  'paused',203  'error',204  'completed',205] as const206 207export type DocumentIndexingStatus = typeof DocumentIndexingStatusList[number]208 209export const DisplayStatusList = [210  'queuing',211  'indexing',212  'paused',213  'error',214  'available',215  'enabled',216  'disabled',217  'archived',218] as const219 220export type DocumentDisplayStatus = typeof DisplayStatusList[number]221 222export type DataSourceInfo = {223  upload_file: {224    id: string225    name: string226    size: number227    mime_type: string228    created_at: number229    created_by: string230    extension: string231  }232  notion_page_icon?: string233  notion_workspace_id?: string234  notion_page_id?: string235  provider?: DataSourceProvider236  job_id: string237  url: string238}239 240export type InitialDocumentDetail = {241  id: string242  batch: string243  position: number244  dataset_id: string245  data_source_type: DataSourceType246  data_source_info: DataSourceInfo247  dataset_process_rule_id: string248  name: string249  created_from: 'api' | 'web'250  created_by: string251  created_at: number252  indexing_status: DocumentIndexingStatus253  display_status: DocumentDisplayStatus254  completed_segments?: number255  total_segments?: number256  doc_form: 'text_model' | 'qa_model'257  doc_language: string258}259 260export type SimpleDocumentDetail = InitialDocumentDetail & {261  enabled: boolean262  word_count: number263  error?: string | null264  archived: boolean265  updated_at: number266  hit_count: number267  dataset_process_rule_id?: string268  data_source_detail_dict?: {269    upload_file: {270      name: string271      extension: string272    }273  }274}275 276export type DocumentListResponse = {277  data: SimpleDocumentDetail[]278  has_more: boolean279  total: number280  page: number281  limit: number282}283 284export type DocumentReq = {285  original_document_id?: string286  indexing_technique?: string287  doc_form: 'text_model' | 'qa_model'288  doc_language: string289  process_rule: ProcessRule290}291 292export type CreateDocumentReq = DocumentReq & {293  data_source: DataSource294  retrieval_model: RetrievalConfig295  embedding_model: string296  embedding_model_provider: string297}298 299export type IndexingEstimateParams = DocumentReq & Partial<DataSource> & {300  dataset_id: string301}302 303export type DataSource = {304  type: DataSourceType305  info_list: {306    data_source_type: DataSourceType307    notion_info_list?: NotionInfo[]308    file_info_list?: {309      file_ids: string[]310    }311    website_info_list?: {312      provider: string313      job_id: string314      urls: string[]315    }316  }317}318 319export type NotionInfo = {320  workspace_id: string321  pages: DataSourceNotionPage[]322}323export type NotionPage = {324  page_id: string325  type: string326}327 328export type ProcessRule = {329  mode: string330  rules: Rules331}332 333export type createDocumentResponse = {334  dataset?: DataSet335  batch: string336  documents: InitialDocumentDetail[]337}338 339export type FullDocumentDetail = SimpleDocumentDetail & {340  batch: string341  created_api_request_id: string342  processing_started_at: number343  parsing_completed_at: number344  cleaning_completed_at: number345  splitting_completed_at: number346  tokens: number347  indexing_latency: number348  completed_at: number349  paused_by: string350  paused_at: number351  stopped_at: number352  indexing_status: string353  disabled_at: number354  disabled_by: string355  archived_reason: 'rule_modified' | 're_upload'356  archived_by: string357  archived_at: number358  doc_type?: DocType | null | 'others'359  doc_metadata?: DocMetadata | null360  segment_count: number361  [key: string]: any362}363 364export type DocMetadata = {365  title: string366  language: string367  author: string368  publisher: string369  publicationDate: string370  ISBN: string371  category: string372  [key: string]: string373}374 375export const CUSTOMIZABLE_DOC_TYPES = [376  'book',377  'web_page',378  'paper',379  'social_media_post',380  'personal_document',381  'business_document',382  'im_chat_log',383] as const384 385export const FIXED_DOC_TYPES = ['synced_from_github', 'synced_from_notion', 'wikipedia_entry'] as const386 387export type CustomizableDocType = typeof CUSTOMIZABLE_DOC_TYPES[number]388export type FixedDocType = typeof FIXED_DOC_TYPES[number]389export type DocType = CustomizableDocType | FixedDocType390 391export type DocumentDetailResponse = FullDocumentDetail392 393export const SEGMENT_STATUS_LIST = ['waiting', 'completed', 'error', 'indexing']394export type SegmentStatus = typeof SEGMENT_STATUS_LIST[number]395 396export type SegmentsQuery = {397  last_id?: string398  limit: number399  // status?: SegmentStatus400  hit_count_gte?: number401  keyword?: string402  enabled?: boolean403}404 405export type SegmentDetailModel = {406  id: string407  position: number408  document_id: string409  content: string410  word_count: number411  tokens: number412  keywords: string[]413  index_node_id: string414  index_node_hash: string415  hit_count: number416  enabled: boolean417  disabled_at: number418  disabled_by: string419  status: SegmentStatus420  created_by: string421  created_at: number422  indexing_at: number423  completed_at: number424  error: string | null425  stopped_at: number426  answer?: string427}428 429export type SegmentsResponse = {430  data: SegmentDetailModel[]431  has_more: boolean432  limit: number433  total: number434}435 436export type HitTestingRecord = {437  id: string438  content: string439  source: 'app' | 'hit_testing' | 'plugin'440  source_app_id: string441  created_by_role: 'account' | 'end_user'442  created_by: string443  created_at: number444}445 446export type HitTesting = {447  segment: Segment448  score: number449  tsne_position: TsnePosition450}451 452export type ExternalKnowledgeBaseHitTesting = {453  content: string454  title: string455  score: number456  metadata: {457    'x-amz-bedrock-kb-source-uri': string458    'x-amz-bedrock-kb-data-source-id': string459  }460}461 462export type Segment = {463  id: string464  document: Document465  content: string466  position: number467  word_count: number468  tokens: number469  keywords: string[]470  hit_count: number471  index_node_hash: string472}473 474export type Document = {475  id: string476  data_source_type: string477  name: string478  doc_type: DocType479}480 481export type HitTestingRecordsResponse = {482  data: HitTestingRecord[]483  has_more: boolean484  limit: number485  total: number486  page: number487}488 489export type TsnePosition = {490  x: number491  y: number492}493 494export type HitTestingResponse = {495  query: {496    content: string497    tsne_position: TsnePosition498  }499  records: Array<HitTesting>500}501 502export type ExternalKnowledgeBaseHitTestingResponse = {503  query: {504    content: string505  }506  records: Array<ExternalKnowledgeBaseHitTesting>507}508 509export type RelatedApp = {510  id: string511  name: string512  mode: AppMode513  icon_type: AppIconType | null514  icon: string515  icon_background: string516  icon_url: string517}518 519export type RelatedAppResponse = {520  data: Array<RelatedApp>521  total: number522}523 524export type SegmentUpdater = {525  content: string526  answer?: string527  keywords?: string[]528}529 530export enum DocForm {531  TEXT = 'text_model',532  QA = 'qa_model',533}534 535export type ErrorDocsResponse = {536  data: IndexingStatusResponse[]537  total: number538}539 540export type SelectedDatasetsMode = {541  allHighQuality: boolean542  allHighQualityVectorSearch: boolean543  allHighQualityFullTextSearch: boolean544  allEconomic: boolean545  mixtureHighQualityAndEconomic: boolean546  allInternal: boolean547  allExternal: boolean548  mixtureInternalAndExternal: boolean549  inconsistentEmbeddingModel: boolean550}551 552export enum WeightedScoreEnum {553  SemanticFirst = 'semantic_first',554  KeywordFirst = 'keyword_first',555  Customized = 'customized',556}557 558export enum RerankingModeEnum {559  RerankingModel = 'reranking_model',560  WeightedScore = 'weighted_score',561}562 563export const DEFAULT_WEIGHTED_SCORE = {564  allHighQualityVectorSearch: {565    semantic: 1.0,566    keyword: 0,567  },568  allHighQualityFullTextSearch: {569    semantic: 0,570    keyword: 1.0,571  },572  other: {573    semantic: 0.7,574    keyword: 0.3,575  },576}577