Underground-Digital/Workflow-Engine
0
1import re2import uuid3from json import dumps as json_dumps4from json import loads as json_loads5from json.decoder import JSONDecodeError6from typing import Optional7 8from requests import get9from yaml import YAMLError, safe_load10 11from core.tools.entities.common_entities import I18nObject12from core.tools.entities.tool_bundle import ApiToolBundle13from core.tools.entities.tool_entities import ApiProviderSchemaType, ToolParameter14from core.tools.errors import ToolApiSchemaError, ToolNotSupportedError, ToolProviderNotFoundError15 16 17class ApiBasedToolSchemaParser:18 @staticmethod19 def parse_openapi_to_tool_bundle(20 openapi: dict, extra_info: Optional[dict], warning: Optional[dict]21 ) -> list[ApiToolBundle]:22 warning = warning if warning is not None else {}23 extra_info = extra_info if extra_info is not None else {}24 25 # set description to extra_info26 extra_info["description"] = openapi["info"].get("description", "")27 28 if len(openapi["servers"]) == 0:29 raise ToolProviderNotFoundError("No server found in the openapi yaml.")30 31 server_url = openapi["servers"][0]["url"]32 33 # list all interfaces34 interfaces = []35 for path, path_item in openapi["paths"].items():36 methods = ["get", "post", "put", "delete", "patch", "head", "options", "trace"]37 for method in methods:38 if method in path_item:39 interfaces.append(40 {41 "path": path,42 "method": method,43 "operation": path_item[method],44 }45 )46 47 # get all parameters48 bundles = []49 for interface in interfaces:50 # convert parameters51 parameters = []52 if "parameters" in interface["operation"]:53 for parameter in interface["operation"]["parameters"]:54 tool_parameter = ToolParameter(55 name=parameter["name"],56 label=I18nObject(en_US=parameter["name"], zh_Hans=parameter["name"]),57 human_description=I18nObject(58 en_US=parameter.get("description", ""), zh_Hans=parameter.get("description", "")59 ),60 type=ToolParameter.ToolParameterType.STRING,61 required=parameter.get("required", False),62 form=ToolParameter.ToolParameterForm.LLM,63 llm_description=parameter.get("description"),64 default=parameter["schema"]["default"]65 if "schema" in parameter and "default" in parameter["schema"]66 else None,67 )68 69 # check if there is a type70 typ = ApiBasedToolSchemaParser._get_tool_parameter_type(parameter)71 if typ:72 tool_parameter.type = typ73 74 parameters.append(tool_parameter)75 # create tool bundle76 # check if there is a request body77 if "requestBody" in interface["operation"]:78 request_body = interface["operation"]["requestBody"]79 if "content" in request_body:80 for content_type, content in request_body["content"].items():81 # if there is a reference, get the reference and overwrite the content82 if "schema" not in content:83 continue84 85 if "$ref" in content["schema"]:86 # get the reference87 root = openapi88 reference = content["schema"]["$ref"].split("/")[1:]89 for ref in reference:90 root = root[ref]91 # overwrite the content92 interface["operation"]["requestBody"]["content"][content_type]["schema"] = root93 94 # parse body parameters95 if "schema" in interface["operation"]["requestBody"]["content"][content_type]:96 body_schema = interface["operation"]["requestBody"]["content"][content_type]["schema"]97 required = body_schema.get("required", [])98 properties = body_schema.get("properties", {})99 for name, property in properties.items():100 tool = ToolParameter(101 name=name,102 label=I18nObject(en_US=name, zh_Hans=name),103 human_description=I18nObject(104 en_US=property.get("description", ""), zh_Hans=property.get("description", "")105 ),106 type=ToolParameter.ToolParameterType.STRING,107 required=name in required,108 form=ToolParameter.ToolParameterForm.LLM,109 llm_description=property.get("description", ""),110 default=property.get("default", None),111 )112 113 # check if there is a type114 typ = ApiBasedToolSchemaParser._get_tool_parameter_type(property)115 if typ:116 tool.type = typ117 118 parameters.append(tool)119 120 # check if parameters is duplicated121 parameters_count = {}122 for parameter in parameters:123 if parameter.name not in parameters_count:124 parameters_count[parameter.name] = 0125 parameters_count[parameter.name] += 1126 for name, count in parameters_count.items():127 if count > 1:128 warning["duplicated_parameter"] = f"Parameter {name} is duplicated."129 130 # check if there is a operation id, use $path_$method as operation id if not131 if "operationId" not in interface["operation"]:132 # remove special characters like / to ensure the operation id is valid ^[a-zA-Z0-9_-]{1,64}$133 path = interface["path"]134 if interface["path"].startswith("/"):135 path = interface["path"][1:]136 # remove special characters like / to ensure the operation id is valid ^[a-zA-Z0-9_-]{1,64}$137 path = re.sub(r"[^a-zA-Z0-9_-]", "", path)138 if not path:139 path = str(uuid.uuid4())140 141 interface["operation"]["operationId"] = f'{path}_{interface["method"]}'142 143 bundles.append(144 ApiToolBundle(145 server_url=server_url + interface["path"],146 method=interface["method"],147 summary=interface["operation"]["description"]148 if "description" in interface["operation"]149 else interface["operation"].get("summary", None),150 operation_id=interface["operation"]["operationId"],151 parameters=parameters,152 author="",153 icon=None,154 openapi=interface["operation"],155 )156 )157 158 return bundles159 160 @staticmethod161 def _get_tool_parameter_type(parameter: dict) -> ToolParameter.ToolParameterType:162 parameter = parameter or {}163 typ = None164 if "type" in parameter:165 typ = parameter["type"]166 elif "schema" in parameter and "type" in parameter["schema"]:167 typ = parameter["schema"]["type"]168 169 if typ in {"integer", "number"}:170 return ToolParameter.ToolParameterType.NUMBER171 elif typ == "boolean":172 return ToolParameter.ToolParameterType.BOOLEAN173 elif typ == "string":174 return ToolParameter.ToolParameterType.STRING175 176 @staticmethod177 def parse_openapi_yaml_to_tool_bundle(178 yaml: str, extra_info: Optional[dict], warning: Optional[dict]179 ) -> list[ApiToolBundle]:180 """181 parse openapi yaml to tool bundle182 183 :param yaml: the yaml string184 :return: the tool bundle185 """186 warning = warning if warning is not None else {}187 extra_info = extra_info if extra_info is not None else {}188 189 openapi: dict = safe_load(yaml)190 if openapi is None:191 raise ToolApiSchemaError("Invalid openapi yaml.")192 return ApiBasedToolSchemaParser.parse_openapi_to_tool_bundle(openapi, extra_info=extra_info, warning=warning)193 194 @staticmethod195 def parse_swagger_to_openapi(swagger: dict, extra_info: Optional[dict], warning: Optional[dict]) -> dict:196 """197 parse swagger to openapi198 199 :param swagger: the swagger dict200 :return: the openapi dict201 """202 # convert swagger to openapi203 info = swagger.get("info", {"title": "Swagger", "description": "Swagger", "version": "1.0.0"})204 205 servers = swagger.get("servers", [])206 207 if len(servers) == 0:208 raise ToolApiSchemaError("No server found in the swagger yaml.")209 210 openapi = {211 "openapi": "3.0.0",212 "info": {213 "title": info.get("title", "Swagger"),214 "description": info.get("description", "Swagger"),215 "version": info.get("version", "1.0.0"),216 },217 "servers": swagger["servers"],218 "paths": {},219 "components": {"schemas": {}},220 }221 222 # check paths223 if "paths" not in swagger or len(swagger["paths"]) == 0:224 raise ToolApiSchemaError("No paths found in the swagger yaml.")225 226 # convert paths227 for path, path_item in swagger["paths"].items():228 openapi["paths"][path] = {}229 for method, operation in path_item.items():230 if "operationId" not in operation:231 raise ToolApiSchemaError(f"No operationId found in operation {method} {path}.")232 233 if ("summary" not in operation or len(operation["summary"]) == 0) and (234 "description" not in operation or len(operation["description"]) == 0235 ):236 warning["missing_summary"] = f"No summary or description found in operation {method} {path}."237 238 openapi["paths"][path][method] = {239 "operationId": operation["operationId"],240 "summary": operation.get("summary", ""),241 "description": operation.get("description", ""),242 "parameters": operation.get("parameters", []),243 "responses": operation.get("responses", {}),244 }245 246 if "requestBody" in operation:247 openapi["paths"][path][method]["requestBody"] = operation["requestBody"]248 249 # convert definitions250 for name, definition in swagger["definitions"].items():251 openapi["components"]["schemas"][name] = definition252 253 return openapi254 255 @staticmethod256 def parse_openai_plugin_json_to_tool_bundle(257 json: str, extra_info: Optional[dict], warning: Optional[dict]258 ) -> list[ApiToolBundle]:259 """260 parse openapi plugin yaml to tool bundle261 262 :param json: the json string263 :return: the tool bundle264 """265 warning = warning if warning is not None else {}266 extra_info = extra_info if extra_info is not None else {}267 268 try:269 openai_plugin = json_loads(json)270 api = openai_plugin["api"]271 api_url = api["url"]272 api_type = api["type"]273 except:274 raise ToolProviderNotFoundError("Invalid openai plugin json.")275 276 if api_type != "openapi":277 raise ToolNotSupportedError("Only openapi is supported now.")278 279 # get openapi yaml280 response = get(api_url, headers={"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "}, timeout=5)281 282 if response.status_code != 200:283 raise ToolProviderNotFoundError("cannot get openapi yaml from url.")284 285 return ApiBasedToolSchemaParser.parse_openapi_yaml_to_tool_bundle(286 response.text, extra_info=extra_info, warning=warning287 )288 289 @staticmethod290 def auto_parse_to_tool_bundle(291 content: str, extra_info: Optional[dict] = None, warning: Optional[dict] = None292 ) -> tuple[list[ApiToolBundle], str]:293 """294 auto parse to tool bundle295 296 :param content: the content297 :return: tools bundle, schema_type298 """299 warning = warning if warning is not None else {}300 extra_info = extra_info if extra_info is not None else {}301 302 content = content.strip()303 loaded_content = None304 json_error = None305 yaml_error = None306 307 try:308 loaded_content = json_loads(content)309 except JSONDecodeError as e:310 json_error = e311 312 if loaded_content is None:313 try:314 loaded_content = safe_load(content)315 except YAMLError as e:316 yaml_error = e317 if loaded_content is None:318 raise ToolApiSchemaError(319 f"Invalid api schema, schema is neither json nor yaml. json error: {str(json_error)},"320 f" yaml error: {str(yaml_error)}"321 )322 323 swagger_error = None324 openapi_error = None325 openapi_plugin_error = None326 schema_type = None327 328 try:329 openapi = ApiBasedToolSchemaParser.parse_openapi_to_tool_bundle(330 loaded_content, extra_info=extra_info, warning=warning331 )332 schema_type = ApiProviderSchemaType.OPENAPI.value333 return openapi, schema_type334 except ToolApiSchemaError as e:335 openapi_error = e336 337 # openai parse error, fallback to swagger338 try:339 converted_swagger = ApiBasedToolSchemaParser.parse_swagger_to_openapi(340 loaded_content, extra_info=extra_info, warning=warning341 )342 schema_type = ApiProviderSchemaType.SWAGGER.value343 return ApiBasedToolSchemaParser.parse_openapi_to_tool_bundle(344 converted_swagger, extra_info=extra_info, warning=warning345 ), schema_type346 except ToolApiSchemaError as e:347 swagger_error = e348 349 # swagger parse error, fallback to openai plugin350 try:351 openapi_plugin = ApiBasedToolSchemaParser.parse_openai_plugin_json_to_tool_bundle(352 json_dumps(loaded_content), extra_info=extra_info, warning=warning353 )354 return openapi_plugin, ApiProviderSchemaType.OPENAI_PLUGIN.value355 except ToolNotSupportedError as e:356 # maybe it's not plugin at all357 openapi_plugin_error = e358 359 raise ToolApiSchemaError(360 f"Invalid api schema, openapi error: {str(openapi_error)}, swagger error: {str(swagger_error)},"361 f" openapi plugin error: {str(openapi_plugin_error)}"362 )363