Underground-Digital/Workflow-Engine
0
1import json2import re3import uuid4from collections.abc import Mapping5from datetime import datetime6from enum import Enum7from typing import Any, Literal, Optional8 9import sqlalchemy as sa10from flask import request11from flask_login import UserMixin12from sqlalchemy import Float, func, text13from sqlalchemy.orm import Mapped, mapped_column14 15from configs import dify_config16from core.file import FILE_MODEL_IDENTITY, File, FileExtraConfig, FileTransferMethod, FileType17from core.file import helpers as file_helpers18from core.file.tool_file_parser import ToolFileParser19from extensions.ext_database import db20from libs.helper import generate_string21from models.enums import CreatedByRole22 23from .account import Account, Tenant24from .types import StringUUID25 26 27class DifySetup(db.Model):28 __tablename__ = "dify_setups"29 __table_args__ = (db.PrimaryKeyConstraint("version", name="dify_setup_pkey"),)30 31 version = db.Column(db.String(255), nullable=False)32 setup_at = db.Column(db.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)"))33 34 35class AppMode(str, Enum):36 COMPLETION = "completion"37 WORKFLOW = "workflow"38 CHAT = "chat"39 ADVANCED_CHAT = "advanced-chat"40 AGENT_CHAT = "agent-chat"41 CHANNEL = "channel"42 43 @classmethod44 def value_of(cls, value: str) -> "AppMode":45 """46 Get value of given mode.47 48 :param value: mode value49 :return: mode50 """51 for mode in cls:52 if mode.value == value:53 return mode54 raise ValueError(f"invalid mode value {value}")55 56 57class IconType(Enum):58 IMAGE = "image"59 EMOJI = "emoji"60 61 62class App(db.Model):63 __tablename__ = "apps"64 __table_args__ = (db.PrimaryKeyConstraint("id", name="app_pkey"), db.Index("app_tenant_id_idx", "tenant_id"))65 66 id = db.Column(StringUUID, server_default=db.text("uuid_generate_v4()"))67 tenant_id: Mapped[str] = db.Column(StringUUID, nullable=False)68 name = db.Column(db.String(255), nullable=False)69 description = db.Column(db.Text, nullable=False, server_default=db.text("''::character varying"))70 mode = db.Column(db.String(255), nullable=False)71 icon_type = db.Column(db.String(255), nullable=True)72 icon = db.Column(db.String(255))73 icon_background = db.Column(db.String(255))74 app_model_config_id = db.Column(StringUUID, nullable=True)75 workflow_id = db.Column(StringUUID, nullable=True)76 status = db.Column(db.String(255), nullable=False, server_default=db.text("'normal'::character varying"))77 enable_site = db.Column(db.Boolean, nullable=False)78 enable_api = db.Column(db.Boolean, nullable=False)79 api_rpm = db.Column(db.Integer, nullable=False, server_default=db.text("0"))80 api_rph = db.Column(db.Integer, nullable=False, server_default=db.text("0"))81 is_demo = db.Column(db.Boolean, nullable=False, server_default=db.text("false"))82 is_public = db.Column(db.Boolean, nullable=False, server_default=db.text("false"))83 is_universal = db.Column(db.Boolean, nullable=False, server_default=db.text("false"))84 tracing = db.Column(db.Text, nullable=True)85 max_active_requests = db.Column(db.Integer, nullable=True)86 created_by = db.Column(StringUUID, nullable=True)87 created_at = db.Column(db.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)"))88 updated_by = db.Column(StringUUID, nullable=True)89 updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)"))90 use_icon_as_answer_icon = db.Column(db.Boolean, nullable=False, server_default=db.text("false"))91 92 @property93 def desc_or_prompt(self):94 if self.description:95 return self.description96 else:97 app_model_config = self.app_model_config98 if app_model_config:99 return app_model_config.pre_prompt100 else:101 return ""102 103 @property104 def site(self):105 site = db.session.query(Site).filter(Site.app_id == self.id).first()106 return site107 108 @property109 def app_model_config(self):110 if self.app_model_config_id:111 return db.session.query(AppModelConfig).filter(AppModelConfig.id == self.app_model_config_id).first()112 113 return None114 115 @property116 def workflow(self) -> Optional["Workflow"]:117 if self.workflow_id:118 from .workflow import Workflow119 120 return db.session.query(Workflow).filter(Workflow.id == self.workflow_id).first()121 122 return None123 124 @property125 def api_base_url(self):126 return (dify_config.SERVICE_API_URL or request.host_url.rstrip("/")) + "/v1"127 128 @property129 def tenant(self):130 tenant = db.session.query(Tenant).filter(Tenant.id == self.tenant_id).first()131 return tenant132 133 @property134 def is_agent(self) -> bool:135 app_model_config = self.app_model_config136 if not app_model_config:137 return False138 if not app_model_config.agent_mode:139 return False140 if self.app_model_config.agent_mode_dict.get("enabled", False) and self.app_model_config.agent_mode_dict.get(141 "strategy", ""142 ) in {"function_call", "react"}:143 self.mode = AppMode.AGENT_CHAT.value144 db.session.commit()145 return True146 return False147 148 @property149 def mode_compatible_with_agent(self) -> str:150 if self.mode == AppMode.CHAT.value and self.is_agent:151 return AppMode.AGENT_CHAT.value152 153 return self.mode154 155 @property156 def deleted_tools(self) -> list:157 # get agent mode tools158 app_model_config = self.app_model_config159 if not app_model_config:160 return []161 if not app_model_config.agent_mode:162 return []163 agent_mode = app_model_config.agent_mode_dict164 tools = agent_mode.get("tools", [])165 166 provider_ids = []167 168 for tool in tools:169 keys = list(tool.keys())170 if len(keys) >= 4:171 provider_type = tool.get("provider_type", "")172 provider_id = tool.get("provider_id", "")173 if provider_type == "api":174 # check if provider id is a uuid string, if not, skip175 try:176 uuid.UUID(provider_id)177 except Exception:178 continue179 provider_ids.append(provider_id)180 181 if not provider_ids:182 return []183 184 api_providers = db.session.execute(185 text("SELECT id FROM tool_api_providers WHERE id IN :provider_ids"), {"provider_ids": tuple(provider_ids)}186 ).fetchall()187 188 deleted_tools = []189 current_api_provider_ids = [str(api_provider.id) for api_provider in api_providers]190 191 for tool in tools:192 keys = list(tool.keys())193 if len(keys) >= 4:194 provider_type = tool.get("provider_type", "")195 provider_id = tool.get("provider_id", "")196 if provider_type == "api" and provider_id not in current_api_provider_ids:197 deleted_tools.append(tool["tool_name"])198 199 return deleted_tools200 201 @property202 def tags(self):203 tags = (204 db.session.query(Tag)205 .join(TagBinding, Tag.id == TagBinding.tag_id)206 .filter(207 TagBinding.target_id == self.id,208 TagBinding.tenant_id == self.tenant_id,209 Tag.tenant_id == self.tenant_id,210 Tag.type == "app",211 )212 .all()213 )214 215 return tags or []216 217 218class AppModelConfig(db.Model):219 __tablename__ = "app_model_configs"220 __table_args__ = (db.PrimaryKeyConstraint("id", name="app_model_config_pkey"), db.Index("app_app_id_idx", "app_id"))221 222 id = db.Column(StringUUID, server_default=db.text("uuid_generate_v4()"))223 app_id = db.Column(StringUUID, nullable=False)224 provider = db.Column(db.String(255), nullable=True)225 model_id = db.Column(db.String(255), nullable=True)226 configs = db.Column(db.JSON, nullable=True)227 created_by = db.Column(StringUUID, nullable=True)228 created_at = db.Column(db.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)"))229 updated_by = db.Column(StringUUID, nullable=True)230 updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)"))231 opening_statement = db.Column(db.Text)232 suggested_questions = db.Column(db.Text)233 suggested_questions_after_answer = db.Column(db.Text)234 speech_to_text = db.Column(db.Text)235 text_to_speech = db.Column(db.Text)236 more_like_this = db.Column(db.Text)237 model = db.Column(db.Text)238 user_input_form = db.Column(db.Text)239 dataset_query_variable = db.Column(db.String(255))240 pre_prompt = db.Column(db.Text)241 agent_mode = db.Column(db.Text)242 sensitive_word_avoidance = db.Column(db.Text)243 retriever_resource = db.Column(db.Text)244 prompt_type = db.Column(db.String(255), nullable=False, server_default=db.text("'simple'::character varying"))245 chat_prompt_config = db.Column(db.Text)246 completion_prompt_config = db.Column(db.Text)247 dataset_configs = db.Column(db.Text)248 external_data_tools = db.Column(db.Text)249 file_upload = db.Column(db.Text)250 251 @property252 def app(self):253 app = db.session.query(App).filter(App.id == self.app_id).first()254 return app255 256 @property257 def model_dict(self) -> dict:258 return json.loads(self.model) if self.model else None259 260 @property261 def suggested_questions_list(self) -> list:262 return json.loads(self.suggested_questions) if self.suggested_questions else []263 264 @property265 def suggested_questions_after_answer_dict(self) -> dict:266 return (267 json.loads(self.suggested_questions_after_answer)268 if self.suggested_questions_after_answer269 else {"enabled": False}270 )271 272 @property273 def speech_to_text_dict(self) -> dict:274 return json.loads(self.speech_to_text) if self.speech_to_text else {"enabled": False}275 276 @property277 def text_to_speech_dict(self) -> dict:278 return json.loads(self.text_to_speech) if self.text_to_speech else {"enabled": False}279 280 @property281 def retriever_resource_dict(self) -> dict:282 return json.loads(self.retriever_resource) if self.retriever_resource else {"enabled": True}283 284 @property285 def annotation_reply_dict(self) -> dict:286 annotation_setting = (287 db.session.query(AppAnnotationSetting).filter(AppAnnotationSetting.app_id == self.app_id).first()288 )289 if annotation_setting:290 collection_binding_detail = annotation_setting.collection_binding_detail291 return {292 "id": annotation_setting.id,293 "enabled": True,294 "score_threshold": annotation_setting.score_threshold,295 "embedding_model": {296 "embedding_provider_name": collection_binding_detail.provider_name,297 "embedding_model_name": collection_binding_detail.model_name,298 },299 }300 301 else:302 return {"enabled": False}303 304 @property305 def more_like_this_dict(self) -> dict:306 return json.loads(self.more_like_this) if self.more_like_this else {"enabled": False}307 308 @property309 def sensitive_word_avoidance_dict(self) -> dict:310 return (311 json.loads(self.sensitive_word_avoidance)312 if self.sensitive_word_avoidance313 else {"enabled": False, "type": "", "configs": []}314 )315 316 @property317 def external_data_tools_list(self) -> list[dict]:318 return json.loads(self.external_data_tools) if self.external_data_tools else []319 320 @property321 def user_input_form_list(self) -> dict:322 return json.loads(self.user_input_form) if self.user_input_form else []323 324 @property325 def agent_mode_dict(self) -> dict:326 return (327 json.loads(self.agent_mode)328 if self.agent_mode329 else {"enabled": False, "strategy": None, "tools": [], "prompt": None}330 )331 332 @property333 def chat_prompt_config_dict(self) -> dict:334 return json.loads(self.chat_prompt_config) if self.chat_prompt_config else {}335 336 @property337 def completion_prompt_config_dict(self) -> dict:338 return json.loads(self.completion_prompt_config) if self.completion_prompt_config else {}339 340 @property341 def dataset_configs_dict(self) -> dict:342 if self.dataset_configs:343 dataset_configs = json.loads(self.dataset_configs)344 if "retrieval_model" not in dataset_configs:345 return {"retrieval_model": "single"}346 else:347 return dataset_configs348 return {349 "retrieval_model": "multiple",350 }351 352 @property353 def file_upload_dict(self) -> dict:354 return (355 json.loads(self.file_upload)356 if self.file_upload357 else {358 "image": {359 "enabled": False,360 "number_limits": 3,361 "detail": "high",362 "transfer_methods": ["remote_url", "local_file"],363 }364 }365 )366 367 def to_dict(self) -> dict:368 return {369 "opening_statement": self.opening_statement,370 "suggested_questions": self.suggested_questions_list,371 "suggested_questions_after_answer": self.suggested_questions_after_answer_dict,372 "speech_to_text": self.speech_to_text_dict,373 "text_to_speech": self.text_to_speech_dict,374 "retriever_resource": self.retriever_resource_dict,375 "annotation_reply": self.annotation_reply_dict,376 "more_like_this": self.more_like_this_dict,377 "sensitive_word_avoidance": self.sensitive_word_avoidance_dict,378 "external_data_tools": self.external_data_tools_list,379 "model": self.model_dict,380 "user_input_form": self.user_input_form_list,381 "dataset_query_variable": self.dataset_query_variable,382 "pre_prompt": self.pre_prompt,383 "agent_mode": self.agent_mode_dict,384 "prompt_type": self.prompt_type,385 "chat_prompt_config": self.chat_prompt_config_dict,386 "completion_prompt_config": self.completion_prompt_config_dict,387 "dataset_configs": self.dataset_configs_dict,388 "file_upload": self.file_upload_dict,389 }390 391 def from_model_config_dict(self, model_config: Mapping[str, Any]):392 self.opening_statement = model_config.get("opening_statement")393 self.suggested_questions = (394 json.dumps(model_config["suggested_questions"]) if model_config.get("suggested_questions") else None395 )396 self.suggested_questions_after_answer = (397 json.dumps(model_config["suggested_questions_after_answer"])398 if model_config.get("suggested_questions_after_answer")399 else None400 )401 self.speech_to_text = json.dumps(model_config["speech_to_text"]) if model_config.get("speech_to_text") else None402 self.text_to_speech = json.dumps(model_config["text_to_speech"]) if model_config.get("text_to_speech") else None403 self.more_like_this = json.dumps(model_config["more_like_this"]) if model_config.get("more_like_this") else None404 self.sensitive_word_avoidance = (405 json.dumps(model_config["sensitive_word_avoidance"])406 if model_config.get("sensitive_word_avoidance")407 else None408 )409 self.external_data_tools = (410 json.dumps(model_config["external_data_tools"]) if model_config.get("external_data_tools") else None411 )412 self.model = json.dumps(model_config["model"]) if model_config.get("model") else None413 self.user_input_form = (414 json.dumps(model_config["user_input_form"]) if model_config.get("user_input_form") else None415 )416 self.dataset_query_variable = model_config.get("dataset_query_variable")417 self.pre_prompt = model_config["pre_prompt"]418 self.agent_mode = json.dumps(model_config["agent_mode"]) if model_config.get("agent_mode") else None419 self.retriever_resource = (420 json.dumps(model_config["retriever_resource"]) if model_config.get("retriever_resource") else None421 )422 self.prompt_type = model_config.get("prompt_type", "simple")423 self.chat_prompt_config = (424 json.dumps(model_config.get("chat_prompt_config")) if model_config.get("chat_prompt_config") else None425 )426 self.completion_prompt_config = (427 json.dumps(model_config.get("completion_prompt_config"))428 if model_config.get("completion_prompt_config")429 else None430 )431 self.dataset_configs = (432 json.dumps(model_config.get("dataset_configs")) if model_config.get("dataset_configs") else None433 )434 self.file_upload = json.dumps(model_config.get("file_upload")) if model_config.get("file_upload") else None435 return self436 437 def copy(self):438 new_app_model_config = AppModelConfig(439 id=self.id,440 app_id=self.app_id,441 opening_statement=self.opening_statement,442 suggested_questions=self.suggested_questions,443 suggested_questions_after_answer=self.suggested_questions_after_answer,444 speech_to_text=self.speech_to_text,445 text_to_speech=self.text_to_speech,446 more_like_this=self.more_like_this,447 sensitive_word_avoidance=self.sensitive_word_avoidance,448 external_data_tools=self.external_data_tools,449 model=self.model,450 user_input_form=self.user_input_form,451 dataset_query_variable=self.dataset_query_variable,452 pre_prompt=self.pre_prompt,453 agent_mode=self.agent_mode,454 retriever_resource=self.retriever_resource,455 prompt_type=self.prompt_type,456 chat_prompt_config=self.chat_prompt_config,457 completion_prompt_config=self.completion_prompt_config,458 dataset_configs=self.dataset_configs,459 file_upload=self.file_upload,460 )461 462 return new_app_model_config463 464 465class RecommendedApp(db.Model):466 __tablename__ = "recommended_apps"467 __table_args__ = (468 db.PrimaryKeyConstraint("id", name="recommended_app_pkey"),469 db.Index("recommended_app_app_id_idx", "app_id"),470 db.Index("recommended_app_is_listed_idx", "is_listed", "language"),471 )472 473 id = db.Column(StringUUID, primary_key=True, server_default=db.text("uuid_generate_v4()"))474 app_id = db.Column(StringUUID, nullable=False)475 description = db.Column(db.JSON, nullable=False)476 copyright = db.Column(db.String(255), nullable=False)477 privacy_policy = db.Column(db.String(255), nullable=False)478 custom_disclaimer: Mapped[str] = mapped_column(sa.TEXT, default="")479 category = db.Column(db.String(255), nullable=False)480 position = db.Column(db.Integer, nullable=False, default=0)481 is_listed = db.Column(db.Boolean, nullable=False, default=True)482 install_count = db.Column(db.Integer, nullable=False, default=0)483 language = db.Column(db.String(255), nullable=False, server_default=db.text("'en-US'::character varying"))484 created_at = db.Column(db.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)"))485 updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)"))486 487 @property488 def app(self):489 app = db.session.query(App).filter(App.id == self.app_id).first()490 return app491 492 493class InstalledApp(db.Model):494 __tablename__ = "installed_apps"495 __table_args__ = (496 db.PrimaryKeyConstraint("id", name="installed_app_pkey"),497 db.Index("installed_app_tenant_id_idx", "tenant_id"),498 db.Index("installed_app_app_id_idx", "app_id"),499 db.UniqueConstraint("tenant_id", "app_id", name="unique_tenant_app"),500 )501 502 id = db.Column(StringUUID, server_default=db.text("uuid_generate_v4()"))503 tenant_id = db.Column(StringUUID, nullable=False)504 app_id = db.Column(StringUUID, nullable=False)505 app_owner_tenant_id = db.Column(StringUUID, nullable=False)506 position = db.Column(db.Integer, nullable=False, default=0)507 is_pinned = db.Column(db.Boolean, nullable=False, server_default=db.text("false"))508 last_used_at = db.Column(db.DateTime, nullable=True)509 created_at = db.Column(db.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)"))510 511 @property512 def app(self):513 app = db.session.query(App).filter(App.id == self.app_id).first()514 return app515 516 @property517 def tenant(self):518 tenant = db.session.query(Tenant).filter(Tenant.id == self.tenant_id).first()519 return tenant520 521 522class Conversation(db.Model):523 __tablename__ = "conversations"524 __table_args__ = (525 db.PrimaryKeyConstraint("id", name="conversation_pkey"),526 db.Index("conversation_app_from_user_idx", "app_id", "from_source", "from_end_user_id"),527 )528 529 id = db.Column(StringUUID, server_default=db.text("uuid_generate_v4()"))530 app_id = db.Column(StringUUID, nullable=False)531 app_model_config_id = db.Column(StringUUID, nullable=True)532 model_provider = db.Column(db.String(255), nullable=True)533 override_model_configs = db.Column(db.Text)534 model_id = db.Column(db.String(255), nullable=True)535 mode = db.Column(db.String(255), nullable=False)536 name = db.Column(db.String(255), nullable=False)537 summary = db.Column(db.Text)538 _inputs: Mapped[dict] = mapped_column("inputs", db.JSON)539 introduction = db.Column(db.Text)540 system_instruction = db.Column(db.Text)541 system_instruction_tokens = db.Column(db.Integer, nullable=False, server_default=db.text("0"))542 status = db.Column(db.String(255), nullable=False)543 invoke_from = db.Column(db.String(255), nullable=True)544 from_source = db.Column(db.String(255), nullable=False)545 from_end_user_id = db.Column(StringUUID)546 from_account_id = db.Column(StringUUID)547 read_at = db.Column(db.DateTime)548 read_account_id = db.Column(StringUUID)549 dialogue_count: Mapped[int] = mapped_column(default=0)550 created_at = db.Column(db.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)"))551 updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)"))552 553 messages = db.relationship("Message", backref="conversation", lazy="select", passive_deletes="all")554 message_annotations = db.relationship(555 "MessageAnnotation", backref="conversation", lazy="select", passive_deletes="all"556 )557 558 is_deleted = db.Column(db.Boolean, nullable=False, server_default=db.text("false"))559 560 @property561 def inputs(self):562 inputs = self._inputs.copy()563 for key, value in inputs.items():564 if isinstance(value, dict) and value.get("dify_model_identity") == FILE_MODEL_IDENTITY:565 inputs[key] = File.model_validate(value)566 elif isinstance(value, list) and all(567 isinstance(item, dict) and item.get("dify_model_identity") == FILE_MODEL_IDENTITY for item in value568 ):569 inputs[key] = [File.model_validate(item) for item in value]570 return inputs571 572 @inputs.setter573 def inputs(self, value: Mapping[str, Any]):574 inputs = dict(value)575 for k, v in inputs.items():576 if isinstance(v, File):577 inputs[k] = v.model_dump()578 elif isinstance(v, list) and all(isinstance(item, File) for item in v):579 inputs[k] = [item.model_dump() for item in v]580 self._inputs = inputs581 582 @property583 def model_config(self):584 model_config = {}585 if self.mode == AppMode.ADVANCED_CHAT.value:586 if self.override_model_configs:587 override_model_configs = json.loads(self.override_model_configs)588 model_config = override_model_configs589 else:590 if self.override_model_configs:591 override_model_configs = json.loads(self.override_model_configs)592 593 if "model" in override_model_configs:594 app_model_config = AppModelConfig()595 app_model_config = app_model_config.from_model_config_dict(override_model_configs)596 model_config = app_model_config.to_dict()597 else:598 model_config["configs"] = override_model_configs599 else:600 app_model_config = (601 db.session.query(AppModelConfig).filter(AppModelConfig.id == self.app_model_config_id).first()602 )603 604 model_config = app_model_config.to_dict()605 606 model_config["model_id"] = self.model_id607 model_config["provider"] = self.model_provider608 609 return model_config610 611 @property612 def summary_or_query(self):613 if self.summary:614 return self.summary615 else:616 first_message = self.first_message617 if first_message:618 return first_message.query619 else:620 return ""621 622 @property623 def annotated(self):624 return db.session.query(MessageAnnotation).filter(MessageAnnotation.conversation_id == self.id).count() > 0625 626 @property627 def annotation(self):628 return db.session.query(MessageAnnotation).filter(MessageAnnotation.conversation_id == self.id).first()629 630 @property631 def message_count(self):632 return db.session.query(Message).filter(Message.conversation_id == self.id).count()633 634 @property635 def user_feedback_stats(self):636 like = (637 db.session.query(MessageFeedback)638 .filter(639 MessageFeedback.conversation_id == self.id,640 MessageFeedback.from_source == "user",641 MessageFeedback.rating == "like",642 )643 .count()644 )645 646 dislike = (647 db.session.query(MessageFeedback)648 .filter(649 MessageFeedback.conversation_id == self.id,650 MessageFeedback.from_source == "user",651 MessageFeedback.rating == "dislike",652 )653 .count()654 )655 656 return {"like": like, "dislike": dislike}657 658 @property659 def admin_feedback_stats(self):660 like = (661 db.session.query(MessageFeedback)662 .filter(663 MessageFeedback.conversation_id == self.id,664 MessageFeedback.from_source == "admin",665 MessageFeedback.rating == "like",666 )667 .count()668 )669 670 dislike = (671 db.session.query(MessageFeedback)672 .filter(673 MessageFeedback.conversation_id == self.id,674 MessageFeedback.from_source == "admin",675 MessageFeedback.rating == "dislike",676 )677 .count()678 )679 680 return {"like": like, "dislike": dislike}681 682 @property683 def first_message(self):684 return db.session.query(Message).filter(Message.conversation_id == self.id).first()685 686 @property687 def app(self):688 return db.session.query(App).filter(App.id == self.app_id).first()689 690 @property691 def from_end_user_session_id(self):692 if self.from_end_user_id:693 end_user = db.session.query(EndUser).filter(EndUser.id == self.from_end_user_id).first()694 if end_user:695 return end_user.session_id696 697 return None698 699 @property700 def from_account_name(self):701 if self.from_account_id:702 account = db.session.query(Account).filter(Account.id == self.from_account_id).first()703 if account:704 return account.name705 706 return None707 708 @property709 def in_debug_mode(self):710 return self.override_model_configs is not None711 712 713class Message(db.Model):714 __tablename__ = "messages"715 __table_args__ = (716 db.PrimaryKeyConstraint("id", name="message_pkey"),717 db.Index("message_app_id_idx", "app_id", "created_at"),718 db.Index("message_conversation_id_idx", "conversation_id"),719 db.Index("message_end_user_idx", "app_id", "from_source", "from_end_user_id"),720 db.Index("message_account_idx", "app_id", "from_source", "from_account_id"),721 db.Index("message_workflow_run_id_idx", "conversation_id", "workflow_run_id"),722 )723 724 id = db.Column(StringUUID, server_default=db.text("uuid_generate_v4()"))725 app_id = db.Column(StringUUID, nullable=False)726 model_provider = db.Column(db.String(255), nullable=True)727 model_id = db.Column(db.String(255), nullable=True)728 override_model_configs = db.Column(db.Text)729 conversation_id = db.Column(StringUUID, db.ForeignKey("conversations.id"), nullable=False)730 _inputs: Mapped[dict] = mapped_column("inputs", db.JSON)731 query: Mapped[str] = db.Column(db.Text, nullable=False)732 message = db.Column(db.JSON, nullable=False)733 message_tokens = db.Column(db.Integer, nullable=False, server_default=db.text("0"))734 message_unit_price = db.Column(db.Numeric(10, 4), nullable=False)735 message_price_unit = db.Column(db.Numeric(10, 7), nullable=False, server_default=db.text("0.001"))736 answer: Mapped[str] = db.Column(db.Text, nullable=False)737 answer_tokens = db.Column(db.Integer, nullable=False, server_default=db.text("0"))738 answer_unit_price = db.Column(db.Numeric(10, 4), nullable=False)739 answer_price_unit = db.Column(db.Numeric(10, 7), nullable=False, server_default=db.text("0.001"))740 parent_message_id = db.Column(StringUUID, nullable=True)741 provider_response_latency = db.Column(db.Float, nullable=False, server_default=db.text("0"))742 total_price = db.Column(db.Numeric(10, 7))743 currency = db.Column(db.String(255), nullable=False)744 status = db.Column(db.String(255), nullable=False, server_default=db.text("'normal'::character varying"))745 error = db.Column(db.Text)746 message_metadata = db.Column(db.Text)747 invoke_from: Mapped[Optional[str]] = db.Column(db.String(255), nullable=True)748 from_source = db.Column(db.String(255), nullable=False)749 from_end_user_id: Mapped[Optional[str]] = db.Column(StringUUID)750 from_account_id: Mapped[Optional[str]] = db.Column(StringUUID)751 created_at = db.Column(db.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)"))752 updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)"))753 agent_based = db.Column(db.Boolean, nullable=False, server_default=db.text("false"))754 workflow_run_id = db.Column(StringUUID)755 756 @property757 def inputs(self):758 inputs = self._inputs.copy()759 for key, value in inputs.items():760 if isinstance(value, dict) and value.get("dify_model_identity") == FILE_MODEL_IDENTITY:761 inputs[key] = File.model_validate(value)762 elif isinstance(value, list) and all(763 isinstance(item, dict) and item.get("dify_model_identity") == FILE_MODEL_IDENTITY for item in value764 ):765 inputs[key] = [File.model_validate(item) for item in value]766 return inputs767 768 @inputs.setter769 def inputs(self, value: Mapping[str, Any]):770 inputs = dict(value)771 for k, v in inputs.items():772 if isinstance(v, File):773 inputs[k] = v.model_dump()774 elif isinstance(v, list) and all(isinstance(item, File) for item in v):775 inputs[k] = [item.model_dump() for item in v]776 self._inputs = inputs777 778 @property779 def re_sign_file_url_answer(self) -> str:780 if not self.answer:781 return self.answer782 783 pattern = r"\[!?.*?\]\((((http|https):\/\/.+)?\/files\/(tools\/)?[\w-]+.*?timestamp=.*&nonce=.*&sign=.*)\)"784 matches = re.findall(pattern, self.answer)785 786 if not matches:787 return self.answer788 789 urls = [match[0] for match in matches]790 791 # remove duplicate urls792 urls = list(set(urls))793 794 if not urls:795 return self.answer796 797 re_sign_file_url_answer = self.answer798 for url in urls:799 if "files/tools" in url:800 # get tool file id801 tool_file_id_pattern = r"\/files\/tools\/([\.\w-]+)?\?timestamp="802 result = re.search(tool_file_id_pattern, url)803 if not result:804 continue805 806 tool_file_id = result.group(1)807 808 # get extension809 if "." in tool_file_id:810 split_result = tool_file_id.split(".")811 extension = f".{split_result[-1]}"812 if len(extension) > 10:813 extension = ".bin"814 tool_file_id = split_result[0]815 else:816 extension = ".bin"817 818 if not tool_file_id:819 continue820 821 sign_url = ToolFileParser.get_tool_file_manager().sign_file(822 tool_file_id=tool_file_id, extension=extension823 )824 elif "file-preview" in url:825 # get upload file id826 upload_file_id_pattern = r"\/files\/([\w-]+)\/file-preview?\?timestamp="827 result = re.search(upload_file_id_pattern, url)828 if not result:829 continue830 831 upload_file_id = result.group(1)832 if not upload_file_id:833 continue834 sign_url = file_helpers.get_signed_file_url(upload_file_id)835 elif "image-preview" in url:836 # image-preview is deprecated, use file-preview instead837 upload_file_id_pattern = r"\/files\/([\w-]+)\/image-preview?\?timestamp="838 result = re.search(upload_file_id_pattern, url)839 if not result:840 continue841 upload_file_id = result.group(1)842 if not upload_file_id:843 continue844 sign_url = file_helpers.get_signed_file_url(upload_file_id)845 else:846 continue847 848 re_sign_file_url_answer = re_sign_file_url_answer.replace(url, sign_url)849 850 return re_sign_file_url_answer851 852 @property853 def user_feedback(self):854 feedback = (855 db.session.query(MessageFeedback)856 .filter(MessageFeedback.message_id == self.id, MessageFeedback.from_source == "user")857 .first()858 )859 return feedback860 861 @property862 def admin_feedback(self):863 feedback = (864 db.session.query(MessageFeedback)865 .filter(MessageFeedback.message_id == self.id, MessageFeedback.from_source == "admin")866 .first()867 )868 return feedback869 870 @property871 def feedbacks(self):872 feedbacks = db.session.query(MessageFeedback).filter(MessageFeedback.message_id == self.id).all()873 return feedbacks874 875 @property876 def annotation(self):877 annotation = db.session.query(MessageAnnotation).filter(MessageAnnotation.message_id == self.id).first()878 return annotation879 880 @property881 def annotation_hit_history(self):882 annotation_history = (883 db.session.query(AppAnnotationHitHistory).filter(AppAnnotationHitHistory.message_id == self.id).first()884 )885 if annotation_history:886 annotation = (887 db.session.query(MessageAnnotation)888 .filter(MessageAnnotation.id == annotation_history.annotation_id)889 .first()890 )891 return annotation892 return None893 894 @property895 def app_model_config(self):896 conversation = db.session.query(Conversation).filter(Conversation.id == self.conversation_id).first()897 if conversation:898 return (899 db.session.query(AppModelConfig).filter(AppModelConfig.id == conversation.app_model_config_id).first()900 )901 902 return None903 904 @property905 def in_debug_mode(self):906 return self.override_model_configs is not None907 908 @property909 def message_metadata_dict(self) -> dict:910 return json.loads(self.message_metadata) if self.message_metadata else {}911 912 @property913 def agent_thoughts(self):914 return (915 db.session.query(MessageAgentThought)916 .filter(MessageAgentThought.message_id == self.id)917 .order_by(MessageAgentThought.position.asc())918 .all()919 )920 921 @property922 def retriever_resources(self):923 return (924 db.session.query(DatasetRetrieverResource)925 .filter(DatasetRetrieverResource.message_id == self.id)926 .order_by(DatasetRetrieverResource.position.asc())927 .all()928 )929 930 @property931 def message_files(self):932 from factories import file_factory933 934 message_files = db.session.query(MessageFile).filter(MessageFile.message_id == self.id).all()935 current_app = db.session.query(App).filter(App.id == self.app_id).first()936 if not current_app:937 raise ValueError(f"App {self.app_id} not found")938 939 files: list[File] = []940 for message_file in message_files:941 if message_file.transfer_method == "local_file":942 if message_file.upload_file_id is None:943 raise ValueError(f"MessageFile {message_file.id} is a local file but has no upload_file_id")944 file = file_factory.build_from_mapping(945 mapping={946 "id": message_file.id,947 "upload_file_id": message_file.upload_file_id,948 "transfer_method": message_file.transfer_method,949 "type": message_file.type,950 },951 tenant_id=current_app.tenant_id,952 user_id=self.from_account_id or self.from_end_user_id or "",953 role=CreatedByRole(message_file.created_by_role),954 config=FileExtraConfig(),955 )956 elif message_file.transfer_method == "remote_url":957 if message_file.url is None:958 raise ValueError(f"MessageFile {message_file.id} is a remote url but has no url")959 file = file_factory.build_from_mapping(960 mapping={961 "id": message_file.id,962 "type": message_file.type,963 "transfer_method": message_file.transfer_method,964 "url": message_file.url,965 },966 tenant_id=current_app.tenant_id,967 user_id=self.from_account_id or self.from_end_user_id or "",968 role=CreatedByRole(message_file.created_by_role),969 config=FileExtraConfig(),970 )971 elif message_file.transfer_method == "tool_file":972 if message_file.upload_file_id is None:973 assert message_file.url is not None974 message_file.upload_file_id = message_file.url.split("/")[-1].split(".")[0]975 mapping = {976 "id": message_file.id,977 "type": message_file.type,978 "transfer_method": message_file.transfer_method,979 "tool_file_id": message_file.upload_file_id,980 }981 file = file_factory.build_from_mapping(982 mapping=mapping,983 tenant_id=current_app.tenant_id,984 user_id=self.from_account_id or self.from_end_user_id or "",985 role=CreatedByRole(message_file.created_by_role),986 config=FileExtraConfig(),987 )988 else:989 raise ValueError(990 f"MessageFile {message_file.id} has an invalid transfer_method {message_file.transfer_method}"991 )992 files.append(file)993 994 result = [995 {"belongs_to": message_file.belongs_to, **file.to_dict()}996 for (file, message_file) in zip(files, message_files)997 ]998 999 db.session.commit()1000 return result1001 1002 @property1003 def workflow_run(self):1004 if self.workflow_run_id:1005 from .workflow import WorkflowRun1006 1007 return db.session.query(WorkflowRun).filter(WorkflowRun.id == self.workflow_run_id).first()1008 1009 return None1010 1011 def to_dict(self) -> dict:1012 return {1013 "id": self.id,1014 "app_id": self.app_id,1015 "conversation_id": self.conversation_id,1016 "inputs": self.inputs,1017 "query": self.query,1018 "message": self.message,1019 "answer": self.answer,1020 "status": self.status,1021 "error": self.error,1022 "message_metadata": self.message_metadata_dict,1023 "from_source": self.from_source,1024 "from_end_user_id": self.from_end_user_id,1025 "from_account_id": self.from_account_id,1026 "created_at": self.created_at.isoformat(),1027 "updated_at": self.updated_at.isoformat(),1028 "agent_based": self.agent_based,1029 "workflow_run_id": self.workflow_run_id,1030 }1031 1032 @classmethod1033 def from_dict(cls, data: dict):1034 return cls(1035 id=data["id"],1036 app_id=data["app_id"],1037 conversation_id=data["conversation_id"],1038 inputs=data["inputs"],1039 query=data["query"],1040 message=data["message"],1041 answer=data["answer"],1042 status=data["status"],1043 error=data["error"],1044 message_metadata=json.dumps(data["message_metadata"]),1045 from_source=data["from_source"],1046 from_end_user_id=data["from_end_user_id"],1047 from_account_id=data["from_account_id"],1048 created_at=data["created_at"],1049 updated_at=data["updated_at"],1050 agent_based=data["agent_based"],1051 workflow_run_id=data["workflow_run_id"],1052 )1053 1054 1055class MessageFeedback(db.Model):1056 __tablename__ = "message_feedbacks"1057 __table_args__ = (1058 db.PrimaryKeyConstraint("id", name="message_feedback_pkey"),1059 db.Index("message_feedback_app_idx", "app_id"),1060 db.Index("message_feedback_message_idx", "message_id", "from_source"),1061 db.Index("message_feedback_conversation_idx", "conversation_id", "from_source", "rating"),1062 )1063 1064 id = db.Column(StringUUID, server_default=db.text("uuid_generate_v4()"))1065 app_id = db.Column(StringUUID, nullable=False)1066 conversation_id = db.Column(StringUUID, nullable=False)1067 message_id = db.Column(StringUUID, nullable=False)1068 rating = db.Column(db.String(255), nullable=False)1069 content = db.Column(db.Text)1070 from_source = db.Column(db.String(255), nullable=False)1071 from_end_user_id = db.Column(StringUUID)1072 from_account_id = db.Column(StringUUID)1073 created_at = db.Column(db.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)"))1074 updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)"))1075 1076 @property1077 def from_account(self):1078 account = db.session.query(Account).filter(Account.id == self.from_account_id).first()1079 return account1080 1081 1082class MessageFile(db.Model):1083 __tablename__ = "message_files"1084 __table_args__ = (1085 db.PrimaryKeyConstraint("id", name="message_file_pkey"),1086 db.Index("message_file_message_idx", "message_id"),1087 db.Index("message_file_created_by_idx", "created_by"),1088 )1089 1090 def __init__(1091 self,1092 *,1093 message_id: str,1094 type: FileType,1095 transfer_method: FileTransferMethod,1096 url: str | None = None,1097 belongs_to: Literal["user", "assistant"] | None = None,1098 upload_file_id: str | None = None,1099 created_by_role: CreatedByRole,1100 created_by: str,1101 ):1102 self.message_id = message_id1103 self.type = type1104 self.transfer_method = transfer_method1105 self.url = url1106 self.belongs_to = belongs_to1107 self.upload_file_id = upload_file_id1108 self.created_by_role = created_by_role.value1109 self.created_by = created_by1110 1111 id: Mapped[str] = db.Column(StringUUID, server_default=db.text("uuid_generate_v4()"))1112 message_id: Mapped[str] = db.Column(StringUUID, nullable=False)1113 type: Mapped[str] = db.Column(db.String(255), nullable=False)1114 transfer_method: Mapped[str] = db.Column(db.String(255), nullable=False)1115 url: Mapped[Optional[str]] = db.Column(db.Text, nullable=True)1116 belongs_to: Mapped[Optional[str]] = db.Column(db.String(255), nullable=True)1117 upload_file_id: Mapped[Optional[str]] = db.Column(StringUUID, nullable=True)1118 created_by_role: Mapped[str] = db.Column(db.String(255), nullable=False)1119 created_by: Mapped[str] = db.Column(StringUUID, nullable=False)1120 created_at: Mapped[datetime] = db.Column(1121 db.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)")1122 )1123 1124 1125class MessageAnnotation(db.Model):1126 __tablename__ = "message_annotations"1127 __table_args__ = (1128 db.PrimaryKeyConstraint("id", name="message_annotation_pkey"),1129 db.Index("message_annotation_app_idx", "app_id"),1130 db.Index("message_annotation_conversation_idx", "conversation_id"),1131 db.Index("message_annotation_message_idx", "message_id"),1132 )1133 1134 id = db.Column(StringUUID, server_default=db.text("uuid_generate_v4()"))1135 app_id = db.Column(StringUUID, nullable=False)1136 conversation_id = db.Column(StringUUID, db.ForeignKey("conversations.id"), nullable=True)1137 message_id = db.Column(StringUUID, nullable=True)1138 question = db.Column(db.Text, nullable=True)1139 content = db.Column(db.Text, nullable=False)1140 hit_count = db.Column(db.Integer, nullable=False, server_default=db.text("0"))1141 account_id = db.Column(StringUUID, nullable=False)1142 created_at = db.Column(db.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)"))1143 updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)"))1144 1145 @property1146 def account(self):1147 account = db.session.query(Account).filter(Account.id == self.account_id).first()1148 return account1149 1150 @property1151 def annotation_create_account(self):1152 account = db.session.query(Account).filter(Account.id == self.account_id).first()1153 return account1154 1155 1156class AppAnnotationHitHistory(db.Model):1157 __tablename__ = "app_annotation_hit_histories"1158 __table_args__ = (1159 db.PrimaryKeyConstraint("id", name="app_annotation_hit_histories_pkey"),1160 db.Index("app_annotation_hit_histories_app_idx", "app_id"),1161 db.Index("app_annotation_hit_histories_account_idx", "account_id"),1162 db.Index("app_annotation_hit_histories_annotation_idx", "annotation_id"),1163 db.Index("app_annotation_hit_histories_message_idx", "message_id"),1164 )1165 1166 id = db.Column(StringUUID, server_default=db.text("uuid_generate_v4()"))1167 app_id = db.Column(StringUUID, nullable=False)1168 annotation_id = db.Column(StringUUID, nullable=False)1169 source = db.Column(db.Text, nullable=False)1170 question = db.Column(db.Text, nullable=False)1171 account_id = db.Column(StringUUID, nullable=False)1172 created_at = db.Column(db.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)"))1173 score = db.Column(Float, nullable=False, server_default=db.text("0"))1174 message_id = db.Column(StringUUID, nullable=False)1175 annotation_question = db.Column(db.Text, nullable=False)1176 annotation_content = db.Column(db.Text, nullable=False)1177 1178 @property1179 def account(self):1180 account = (1181 db.session.query(Account)1182 .join(MessageAnnotation, MessageAnnotation.account_id == Account.id)1183 .filter(MessageAnnotation.id == self.annotation_id)1184 .first()1185 )1186 return account1187 1188 @property1189 def annotation_create_account(self):1190 account = db.session.query(Account).filter(Account.id == self.account_id).first()1191 return account1192 1193 1194class AppAnnotationSetting(db.Model):1195 __tablename__ = "app_annotation_settings"1196 __table_args__ = (1197 db.PrimaryKeyConstraint("id", name="app_annotation_settings_pkey"),1198 db.Index("app_annotation_settings_app_idx", "app_id"),1199 )1200