codekingpro/portable-devtools
114k
1from __future__ import annotations2 3import logging4import os5import sys6import warnings7from typing import (8 AbstractSet,9 Any,10 AsyncIterator,11 Awaitable,12 Callable,13 Collection,14 Dict,15 Iterator,16 List,17 Literal,18 Mapping,19 Optional,20 Set,21 Tuple,22 Union,23)24 25from langchain_core._api.deprecation import deprecated26from langchain_core.callbacks import (27 AsyncCallbackManagerForLLMRun,28 CallbackManagerForLLMRun,29)30from langchain_core.language_models.llms import BaseLLM, create_base_retry_decorator31from langchain_core.outputs import Generation, GenerationChunk, LLMResult32from langchain_core.utils import (33 get_from_dict_or_env,34 get_pydantic_field_names,35 pre_init,36)37from langchain_core.utils.pydantic import get_fields38from langchain_core.utils.utils import _build_model_kwargs39from pydantic import ConfigDict, Field, model_validator40 41from langchain_community.utils.openai import is_openai_v142 43logger = logging.getLogger(__name__)44 45 46def update_token_usage(47 keys: Set[str], response: Dict[str, Any], token_usage: Dict[str, Any]48) -> None:49 """Update token usage."""50 _keys_to_use = keys.intersection(response["usage"])51 for _key in _keys_to_use:52 if _key not in token_usage:53 token_usage[_key] = response["usage"][_key]54 else:55 token_usage[_key] += response["usage"][_key]56 57 58def _stream_response_to_generation_chunk(59 stream_response: Dict[str, Any],60) -> GenerationChunk:61 """Convert a stream response to a generation chunk."""62 if not stream_response["choices"]:63 return GenerationChunk(text="")64 return GenerationChunk(65 text=stream_response["choices"][0]["text"],66 generation_info=dict(67 finish_reason=stream_response["choices"][0].get("finish_reason", None),68 logprobs=stream_response["choices"][0].get("logprobs", None),69 ),70 )71 72 73def _update_response(response: Dict[str, Any], stream_response: Dict[str, Any]) -> None:74 """Update response from the stream response."""75 response["choices"][0]["text"] += stream_response["choices"][0]["text"]76 response["choices"][0]["finish_reason"] = stream_response["choices"][0].get(77 "finish_reason", None78 )79 response["choices"][0]["logprobs"] = stream_response["choices"][0]["logprobs"]80 81 82def _streaming_response_template() -> Dict[str, Any]:83 return {84 "choices": [85 {86 "text": "",87 "finish_reason": None,88 "logprobs": None,89 }90 ]91 }92 93 94def _create_retry_decorator(95 llm: Union[BaseOpenAI, OpenAIChat],96 run_manager: Optional[97 Union[AsyncCallbackManagerForLLMRun, CallbackManagerForLLMRun]98 ] = None,99) -> Callable[[Any], Any]:100 import openai101 102 errors = [103 openai.error.Timeout,104 openai.error.APIError,105 openai.error.APIConnectionError,106 openai.error.RateLimitError,107 openai.error.ServiceUnavailableError,108 ]109 return create_base_retry_decorator(110 error_types=errors, max_retries=llm.max_retries, run_manager=run_manager111 )112 113 114def completion_with_retry(115 llm: Union[BaseOpenAI, OpenAIChat],116 run_manager: Optional[CallbackManagerForLLMRun] = None,117 **kwargs: Any,118) -> Any:119 """Use tenacity to retry the completion call."""120 if is_openai_v1():121 return llm.client.create(**kwargs)122 123 retry_decorator = _create_retry_decorator(llm, run_manager=run_manager)124 125 @retry_decorator126 def _completion_with_retry(**kwargs: Any) -> Any:127 return llm.client.create(**kwargs)128 129 return _completion_with_retry(**kwargs)130 131 132async def acompletion_with_retry(133 llm: Union[BaseOpenAI, OpenAIChat],134 run_manager: Optional[AsyncCallbackManagerForLLMRun] = None,135 **kwargs: Any,136) -> Any:137 """Use tenacity to retry the async completion call."""138 if is_openai_v1():139 return await llm.async_client.create(**kwargs)140 141 retry_decorator = _create_retry_decorator(llm, run_manager=run_manager)142 143 @retry_decorator144 async def _completion_with_retry(**kwargs: Any) -> Any:145 # Use OpenAI's async api https://github.com/openai/openai-python#async-api146 return await llm.client.acreate(**kwargs)147 148 return await _completion_with_retry(**kwargs)149 150 151class BaseOpenAI(BaseLLM):152 """Base OpenAI large language model class."""153 154 @property155 def lc_secrets(self) -> Dict[str, str]:156 return {"openai_api_key": "OPENAI_API_KEY"}157 158 @classmethod159 def get_lc_namespace(cls) -> List[str]:160 """Get the namespace of the langchain object."""161 return ["langchain", "llms", "openai"]162 163 @property164 def lc_attributes(self) -> Dict[str, Any]:165 attributes: Dict[str, Any] = {}166 if self.openai_api_base:167 attributes["openai_api_base"] = self.openai_api_base168 169 if self.openai_organization:170 attributes["openai_organization"] = self.openai_organization171 172 if self.openai_proxy:173 attributes["openai_proxy"] = self.openai_proxy174 175 return attributes176 177 @classmethod178 def is_lc_serializable(cls) -> bool:179 return True180 181 client: Any = Field(default=None, exclude=True) #: :meta private:182 async_client: Any = Field(default=None, exclude=True) #: :meta private:183 model_name: str = Field(default="gpt-3.5-turbo-instruct", alias="model")184 """Model name to use."""185 temperature: float = 0.7186 """What sampling temperature to use."""187 max_tokens: int = 256188 """The maximum number of tokens to generate in the completion.189 -1 returns as many tokens as possible given the prompt and190 the models maximal context size."""191 top_p: float = 1192 """Total probability mass of tokens to consider at each step."""193 frequency_penalty: float = 0194 """Penalizes repeated tokens according to frequency."""195 presence_penalty: float = 0196 """Penalizes repeated tokens."""197 n: int = 1198 """How many completions to generate for each prompt."""199 best_of: int = 1200 """Generates best_of completions server-side and returns the "best"."""201 model_kwargs: Dict[str, Any] = Field(default_factory=dict)202 """Holds any model parameters valid for `create` call not explicitly specified."""203 # When updating this to use a SecretStr204 # Check for classes that derive from this class (as some of them205 # may assume openai_api_key is a str)206 openai_api_key: Optional[str] = Field(default=None, alias="api_key")207 """Automatically inferred from env var `OPENAI_API_KEY` if not provided."""208 openai_api_base: Optional[str] = Field(default=None, alias="base_url")209 """Base URL path for API requests, leave blank if not using a proxy or service 210 emulator."""211 openai_organization: Optional[str] = Field(default=None, alias="organization")212 """Automatically inferred from env var `OPENAI_ORG_ID` if not provided."""213 # to support explicit proxy for OpenAI214 openai_proxy: Optional[str] = None215 batch_size: int = 20216 """Batch size to use when passing multiple documents to generate."""217 request_timeout: Union[float, Tuple[float, float], Any, None] = Field(218 default=None, alias="timeout"219 )220 """Timeout for requests to OpenAI completion API. Can be float, httpx.Timeout or 221 None."""222 logit_bias: Optional[Dict[str, float]] = Field(default_factory=dict)223 """Adjust the probability of specific tokens being generated."""224 max_retries: int = 2225 """Maximum number of retries to make when generating."""226 streaming: bool = False227 """Whether to stream the results or not."""228 allowed_special: Union[Literal["all"], AbstractSet[str]] = set()229 """Set of special tokens that are allowed。"""230 disallowed_special: Union[Literal["all"], Collection[str]] = "all"231 """Set of special tokens that are not allowed。"""232 tiktoken_model_name: Optional[str] = None233 """The model name to pass to tiktoken when using this class. 234 Tiktoken is used to count the number of tokens in documents to constrain 235 them to be under a certain limit. By default, when set to None, this will 236 be the same as the embedding model name. However, there are some cases 237 where you may want to use this Embedding class with a model name not 238 supported by tiktoken. This can include when using Azure embeddings or 239 when using one of the many model providers that expose an OpenAI-like 240 API but with different models. In those cases, in order to avoid erroring 241 when tiktoken is called, you can specify a model name to use here."""242 default_headers: Union[Mapping[str, str], None] = None243 default_query: Union[Mapping[str, object], None] = None244 # Configure a custom httpx client. See the245 # [httpx documentation](https://www.python-httpx.org/api/#client) for more details.246 http_client: Union[Any, None] = None247 """Optional httpx.Client."""248 249 def __new__(cls, **data: Any) -> Union[OpenAIChat, BaseOpenAI]: # type: ignore[misc]250 """Initialize the OpenAI object."""251 model_name = data.get("model_name", "")252 if (253 model_name.startswith("gpt-3.5-turbo") or model_name.startswith("gpt-4")254 ) and "-instruct" not in model_name:255 warnings.warn(256 "You are trying to use a chat model. This way of initializing it is "257 "no longer supported. Instead, please use: "258 "`from langchain_community.chat_models import ChatOpenAI`"259 )260 return OpenAIChat(**data)261 return super().__new__(cls)262 263 model_config = ConfigDict(264 populate_by_name=True,265 )266 267 @model_validator(mode="before")268 @classmethod269 def build_extra(cls, values: Dict[str, Any]) -> Any:270 """Build extra kwargs from additional params that were passed in."""271 all_required_field_names = get_pydantic_field_names(cls)272 values = _build_model_kwargs(values, all_required_field_names)273 return values274 275 @pre_init276 def validate_environment(cls, values: Dict) -> Dict:277 """Validate that api key and python package exists in environment."""278 if values["n"] < 1:279 raise ValueError("n must be at least 1.")280 if values["streaming"] and values["n"] > 1:281 raise ValueError("Cannot stream results when n > 1.")282 if values["streaming"] and values["best_of"] > 1:283 raise ValueError("Cannot stream results when best_of > 1.")284 285 values["openai_api_key"] = get_from_dict_or_env(286 values, "openai_api_key", "OPENAI_API_KEY"287 )288 values["openai_api_base"] = values["openai_api_base"] or os.getenv(289 "OPENAI_API_BASE"290 )291 values["openai_proxy"] = get_from_dict_or_env(292 values,293 "openai_proxy",294 "OPENAI_PROXY",295 default="",296 )297 values["openai_organization"] = (298 values["openai_organization"]299 or os.getenv("OPENAI_ORG_ID")300 or os.getenv("OPENAI_ORGANIZATION")301 )302 try:303 import openai304 except ImportError:305 raise ImportError(306 "Could not import openai python package. "307 "Please install it with `pip install openai`."308 )309 310 if is_openai_v1():311 client_params = {312 "api_key": values["openai_api_key"],313 "organization": values["openai_organization"],314 "base_url": values["openai_api_base"],315 "timeout": values["request_timeout"],316 "max_retries": values["max_retries"],317 "default_headers": values["default_headers"],318 "default_query": values["default_query"],319 "http_client": values["http_client"],320 }321 if not values.get("client"):322 values["client"] = openai.OpenAI(**client_params).completions323 if not values.get("async_client"):324 values["async_client"] = openai.AsyncOpenAI(**client_params).completions325 elif not values.get("client"):326 values["client"] = openai.Completion327 else:328 pass329 330 return values331 332 @property333 def _default_params(self) -> Dict[str, Any]:334 """Get the default parameters for calling OpenAI API."""335 normal_params: Dict[str, Any] = {336 "temperature": self.temperature,337 "top_p": self.top_p,338 "frequency_penalty": self.frequency_penalty,339 "presence_penalty": self.presence_penalty,340 "n": self.n,341 "logit_bias": self.logit_bias,342 }343 344 if self.max_tokens is not None:345 normal_params["max_tokens"] = self.max_tokens346 if self.request_timeout is not None and not is_openai_v1():347 normal_params["request_timeout"] = self.request_timeout348 349 # Azure gpt-35-turbo doesn't support best_of350 # don't specify best_of if it is 1351 if self.best_of > 1:352 normal_params["best_of"] = self.best_of353 354 return {**normal_params, **self.model_kwargs}355 356 def _stream(357 self,358 prompt: str,359 stop: Optional[List[str]] = None,360 run_manager: Optional[CallbackManagerForLLMRun] = None,361 **kwargs: Any,362 ) -> Iterator[GenerationChunk]:363 params = {**self._invocation_params, **kwargs, "stream": True}364 self.get_sub_prompts(params, [prompt], stop) # this mutates params365 for stream_resp in completion_with_retry(366 self, prompt=prompt, run_manager=run_manager, **params367 ):368 if not isinstance(stream_resp, dict):369 stream_resp = stream_resp.dict()370 chunk = _stream_response_to_generation_chunk(stream_resp)371 if run_manager:372 run_manager.on_llm_new_token(373 chunk.text,374 chunk=chunk,375 verbose=self.verbose,376 logprobs=chunk.generation_info["logprobs"]377 if chunk.generation_info378 else None,379 )380 yield chunk381 382 async def _astream(383 self,384 prompt: str,385 stop: Optional[List[str]] = None,386 run_manager: Optional[AsyncCallbackManagerForLLMRun] = None,387 **kwargs: Any,388 ) -> AsyncIterator[GenerationChunk]:389 params = {**self._invocation_params, **kwargs, "stream": True}390 self.get_sub_prompts(params, [prompt], stop) # this mutates params391 async for stream_resp in await acompletion_with_retry(392 self, prompt=prompt, run_manager=run_manager, **params393 ):394 if not isinstance(stream_resp, dict):395 stream_resp = stream_resp.dict()396 chunk = _stream_response_to_generation_chunk(stream_resp)397 if run_manager:398 await run_manager.on_llm_new_token(399 chunk.text,400 chunk=chunk,401 verbose=self.verbose,402 logprobs=chunk.generation_info["logprobs"]403 if chunk.generation_info404 else None,405 )406 yield chunk407 408 def _generate(409 self,410 prompts: List[str],411 stop: Optional[List[str]] = None,412 run_manager: Optional[CallbackManagerForLLMRun] = None,413 **kwargs: Any,414 ) -> LLMResult:415 """Call out to OpenAI's endpoint with k unique prompts.416 417 Args:418 prompts: The prompts to pass into the model.419 stop: Optional list of stop words to use when generating.420 421 Returns:422 The full LLM output.423 424 Example:425 .. code-block:: python426 427 response = openai.generate(["Tell me a joke."])428 """429 # TODO: write a unit test for this430 params = self._invocation_params431 params = {**params, **kwargs}432 sub_prompts = self.get_sub_prompts(params, prompts, stop)433 choices = []434 token_usage: Dict[str, int] = {}435 # Get the token usage from the response.436 # Includes prompt, completion, and total tokens used.437 _keys = {"completion_tokens", "prompt_tokens", "total_tokens"}438 system_fingerprint: Optional[str] = None439 for _prompts in sub_prompts:440 if self.streaming:441 if len(_prompts) > 1:442 raise ValueError("Cannot stream results with multiple prompts.")443 444 generation: Optional[GenerationChunk] = None445 for chunk in self._stream(_prompts[0], stop, run_manager, **kwargs):446 if generation is None:447 generation = chunk448 else:449 generation += chunk450 assert generation is not None451 choices.append(452 {453 "text": generation.text,454 "finish_reason": generation.generation_info.get("finish_reason")455 if generation.generation_info456 else None,457 "logprobs": generation.generation_info.get("logprobs")458 if generation.generation_info459 else None,460 }461 )462 else:463 response = completion_with_retry(464 self, prompt=_prompts, run_manager=run_manager, **params465 )466 if not isinstance(response, dict):467 # V1 client returns the response in an PyDantic object instead of468 # dict. For the transition period, we deep convert it to dict.469 response = response.dict()470 471 choices.extend(response["choices"])472 update_token_usage(_keys, response, token_usage)473 if not system_fingerprint:474 system_fingerprint = response.get("system_fingerprint")475 return self.create_llm_result(476 choices,477 prompts,478 params,479 token_usage,480 system_fingerprint=system_fingerprint,481 )482 483 async def _agenerate(484 self,485 prompts: List[str],486 stop: Optional[List[str]] = None,487 run_manager: Optional[AsyncCallbackManagerForLLMRun] = None,488 **kwargs: Any,489 ) -> LLMResult:490 """Call out to OpenAI's endpoint async with k unique prompts."""491 params = self._invocation_params492 params = {**params, **kwargs}493 sub_prompts = self.get_sub_prompts(params, prompts, stop)494 choices = []495 token_usage: Dict[str, int] = {}496 # Get the token usage from the response.497 # Includes prompt, completion, and total tokens used.498 _keys = {"completion_tokens", "prompt_tokens", "total_tokens"}499 system_fingerprint: Optional[str] = None500 for _prompts in sub_prompts:501 if self.streaming:502 if len(_prompts) > 1:503 raise ValueError("Cannot stream results with multiple prompts.")504 505 generation: Optional[GenerationChunk] = None506 async for chunk in self._astream(507 _prompts[0], stop, run_manager, **kwargs508 ):509 if generation is None:510 generation = chunk511 else:512 generation += chunk513 assert generation is not None514 choices.append(515 {516 "text": generation.text,517 "finish_reason": generation.generation_info.get("finish_reason")518 if generation.generation_info519 else None,520 "logprobs": generation.generation_info.get("logprobs")521 if generation.generation_info522 else None,523 }524 )525 else:526 response = await acompletion_with_retry(527 self, prompt=_prompts, run_manager=run_manager, **params528 )529 if not isinstance(response, dict):530 response = response.dict()531 choices.extend(response["choices"])532 update_token_usage(_keys, response, token_usage)533 return self.create_llm_result(534 choices,535 prompts,536 params,537 token_usage,538 system_fingerprint=system_fingerprint,539 )540 541 def get_sub_prompts(542 self,543 params: Dict[str, Any],544 prompts: List[str],545 stop: Optional[List[str]] = None,546 ) -> List[List[str]]:547 """Get the sub prompts for llm call."""548 if stop is not None:549 if "stop" in params:550 raise ValueError("`stop` found in both the input and default params.")551 params["stop"] = stop552 if params["max_tokens"] == -1:553 if len(prompts) != 1:554 raise ValueError(555 "max_tokens set to -1 not supported for multiple inputs."556 )557 params["max_tokens"] = self.max_tokens_for_prompt(prompts[0])558 sub_prompts = [559 prompts[i : i + self.batch_size]560 for i in range(0, len(prompts), self.batch_size)561 ]562 return sub_prompts563 564 def create_llm_result(565 self,566 choices: Any,567 prompts: List[str],568 params: Dict[str, Any],569 token_usage: Dict[str, int],570 *,571 system_fingerprint: Optional[str] = None,572 ) -> LLMResult:573 """Create the LLMResult from the choices and prompts."""574 generations = []575 n = params.get("n", self.n)576 for i, _ in enumerate(prompts):577 sub_choices = choices[i * n : (i + 1) * n]578 generations.append(579 [580 Generation(581 text=choice["text"],582 generation_info=dict(583 finish_reason=choice.get("finish_reason"),584 logprobs=choice.get("logprobs"),585 ),586 )587 for choice in sub_choices588 ]589 )590 llm_output = {"token_usage": token_usage, "model_name": self.model_name}591 if system_fingerprint:592 llm_output["system_fingerprint"] = system_fingerprint593 return LLMResult(generations=generations, llm_output=llm_output)594 595 @property596 def _invocation_params(self) -> Dict[str, Any]:597 """Get the parameters used to invoke the model."""598 openai_creds: Dict[str, Any] = {}599 if not is_openai_v1():600 openai_creds.update(601 {602 "api_key": self.openai_api_key,603 "api_base": self.openai_api_base,604 "organization": self.openai_organization,605 }606 )607 if self.openai_proxy:608 import openai609 610 openai.proxy = {"http": self.openai_proxy, "https": self.openai_proxy}611 return {**openai_creds, **self._default_params}612 613 @property614 def _identifying_params(self) -> Mapping[str, Any]:615 """Get the identifying parameters."""616 return {**{"model_name": self.model_name}, **self._default_params}617 618 @property619 def _llm_type(self) -> str:620 """Return type of llm."""621 return "openai"622 623 def get_token_ids(self, text: str) -> List[int]:624 """Get the token IDs using the tiktoken package."""625 # tiktoken NOT supported for Python < 3.8626 if sys.version_info[1] < 8:627 return super().get_num_tokens(text)628 try:629 import tiktoken630 except ImportError:631 raise ImportError(632 "Could not import tiktoken python package. "633 "This is needed in order to calculate get_num_tokens. "634 "Please install it with `pip install tiktoken`."635 )636 637 model_name = self.tiktoken_model_name or self.model_name638 try:639 enc = tiktoken.encoding_for_model(model_name)640 except KeyError:641 logger.warning("Warning: model not found. Using cl100k_base encoding.")642 model = "cl100k_base"643 enc = tiktoken.get_encoding(model)644 645 return enc.encode(646 text,647 allowed_special=self.allowed_special,648 disallowed_special=self.disallowed_special,649 )650 651 @staticmethod652 def modelname_to_contextsize(modelname: str) -> int:653 """Calculate the maximum number of tokens possible to generate for a model.654 655 Args:656 modelname: The modelname we want to know the context size for.657 658 Returns:659 The maximum context size660 661 Example:662 .. code-block:: python663 664 max_tokens = openai.modelname_to_contextsize("gpt-3.5-turbo-instruct")665 """666 model_token_mapping = {667 "gpt-4o": 128_000,668 "gpt-4o-2024-05-13": 128_000,669 "gpt-4": 8192,670 "gpt-4-0314": 8192,671 "gpt-4-0613": 8192,672 "gpt-4-32k": 32768,673 "gpt-4-32k-0314": 32768,674 "gpt-4-32k-0613": 32768,675 "gpt-3.5-turbo": 4096,676 "gpt-3.5-turbo-0301": 4096,677 "gpt-3.5-turbo-0613": 4096,678 "gpt-3.5-turbo-16k": 16385,679 "gpt-3.5-turbo-16k-0613": 16385,680 "gpt-3.5-turbo-instruct": 4096,681 "text-ada-001": 2049,682 "ada": 2049,683 "text-babbage-001": 2040,684 "babbage": 2049,685 "text-curie-001": 2049,686 "curie": 2049,687 "davinci": 2049,688 "text-davinci-003": 4097,689 "text-davinci-002": 4097,690 "code-davinci-002": 8001,691 "code-davinci-001": 8001,692 "code-cushman-002": 2048,693 "code-cushman-001": 2048,694 }695 696 # handling finetuned models697 if "ft-" in modelname:698 modelname = modelname.split(":")[0]699 700 context_size = model_token_mapping.get(modelname, None)701 702 if context_size is None:703 raise ValueError(704 f"Unknown model: {modelname}. Please provide a valid OpenAI model name."705 "Known models are: " + ", ".join(model_token_mapping.keys())706 )707 708 return context_size709 710 @property711 def max_context_size(self) -> int:712 """Get max context size for this model."""713 return self.modelname_to_contextsize(self.model_name)714 715 def max_tokens_for_prompt(self, prompt: str) -> int:716 """Calculate the maximum number of tokens possible to generate for a prompt.717 718 Args:719 prompt: The prompt to pass into the model.720 721 Returns:722 The maximum number of tokens to generate for a prompt.723 724 Example:725 .. code-block:: python726 727 max_tokens = openai.max_tokens_for_prompt("Tell me a joke.")728 """729 num_tokens = self.get_num_tokens(prompt)730 return self.max_context_size - num_tokens731 732 733@deprecated(since="0.0.10", removal="1.0", alternative_import="langchain_openai.OpenAI")734class OpenAI(BaseOpenAI):735 """OpenAI large language models.736 737 To use, you should have the ``openai`` python package installed, and the738 environment variable ``OPENAI_API_KEY`` set with your API key.739 740 Any parameters that are valid to be passed to the openai.create call can be passed741 in, even if not explicitly saved on this class.742 743 Example:744 .. code-block:: python745 746 from langchain_community.llms import OpenAI747 openai = OpenAI(model_name="gpt-3.5-turbo-instruct")748 """749 750 @classmethod751 def get_lc_namespace(cls) -> List[str]:752 """Get the namespace of the langchain object."""753 return ["langchain", "llms", "openai"]754 755 @property756 def _invocation_params(self) -> Dict[str, Any]:757 return {**{"model": self.model_name}, **super()._invocation_params}758 759 760@deprecated(761 since="0.0.10", removal="1.0", alternative_import="langchain_openai.AzureOpenAI"762)763class AzureOpenAI(BaseOpenAI):764 """Azure-specific OpenAI large language models.765 766 To use, you should have the ``openai`` python package installed, and the767 environment variable ``OPENAI_API_KEY`` set with your API key.768 769 Any parameters that are valid to be passed to the openai.create call can be passed770 in, even if not explicitly saved on this class.771 772 Example:773 .. code-block:: python774 775 from langchain_community.llms import AzureOpenAI776 777 openai = AzureOpenAI(model_name="gpt-3.5-turbo-instruct")778 """779 780 azure_endpoint: Union[str, None] = None781 """Your Azure endpoint, including the resource.782 783 Automatically inferred from env var `AZURE_OPENAI_ENDPOINT` if not provided.784 785 Example: `https://example-resource.azure.openai.com/`786 """787 deployment_name: Union[str, None] = Field(default=None, alias="azure_deployment")788 """A model deployment. 789 790 If given sets the base client URL to include `/deployments/{azure_deployment}`.791 Note: this means you won't be able to use non-deployment endpoints.792 """793 openai_api_version: str = Field(default="", alias="api_version")794 """Automatically inferred from env var `OPENAI_API_VERSION` if not provided."""795 openai_api_key: Union[str, None] = Field(default=None, alias="api_key")796 """Automatically inferred from env var `AZURE_OPENAI_API_KEY` if not provided."""797 azure_ad_token: Union[str, None] = None798 """Your Azure Active Directory token.799 800 Automatically inferred from env var `AZURE_OPENAI_AD_TOKEN` if not provided.801 802 For more: 803 https://www.microsoft.com/en-us/security/business/identity-access/microsoft-entra-id.804 """805 azure_ad_token_provider: Union[Callable[[], str], None] = None806 """A function that returns an Azure Active Directory token.807 808 Will be invoked on every sync request. For async requests,809 will be invoked if `azure_ad_async_token_provider` is not provided.810 """811 azure_ad_async_token_provider: Union[Callable[[], Awaitable[str]], None] = None812 """A function that returns an Azure Active Directory token.813 814 Will be invoked on every async request.815 """816 openai_api_type: str = ""817 """Legacy, for openai<1.0.0 support."""818 validate_base_url: bool = True819 """For backwards compatibility. If legacy val openai_api_base is passed in, try to 820 infer if it is a base_url or azure_endpoint and update accordingly.821 """822 823 @classmethod824 def get_lc_namespace(cls) -> List[str]:825 """Get the namespace of the langchain object."""826 return ["langchain", "llms", "openai"]827 828 @pre_init829 def validate_environment(cls, values: Dict) -> Dict:830 """Validate that api key and python package exists in environment."""831 if values["n"] < 1:832 raise ValueError("n must be at least 1.")833 if values["streaming"] and values["n"] > 1:834 raise ValueError("Cannot stream results when n > 1.")835 if values["streaming"] and values["best_of"] > 1:836 raise ValueError("Cannot stream results when best_of > 1.")837 838 # Check OPENAI_KEY for backwards compatibility.839 # TODO: Remove OPENAI_API_KEY support to avoid possible conflict when using840 # other forms of azure credentials.841 values["openai_api_key"] = (842 values["openai_api_key"]843 or os.getenv("AZURE_OPENAI_API_KEY")844 or os.getenv("OPENAI_API_KEY")845 )846 847 values["azure_endpoint"] = values["azure_endpoint"] or os.getenv(848 "AZURE_OPENAI_ENDPOINT"849 )850 values["azure_ad_token"] = values["azure_ad_token"] or os.getenv(851 "AZURE_OPENAI_AD_TOKEN"852 )853 values["openai_api_base"] = values["openai_api_base"] or os.getenv(854 "OPENAI_API_BASE"855 )856 values["openai_proxy"] = get_from_dict_or_env(857 values,858 "openai_proxy",859 "OPENAI_PROXY",860 default="",861 )862 values["openai_organization"] = (863 values["openai_organization"]864 or os.getenv("OPENAI_ORG_ID")865 or os.getenv("OPENAI_ORGANIZATION")866 )867 values["openai_api_version"] = values["openai_api_version"] or os.getenv(868 "OPENAI_API_VERSION"869 )870 values["openai_api_type"] = get_from_dict_or_env(871 values, "openai_api_type", "OPENAI_API_TYPE", default="azure"872 )873 try:874 import openai875 except ImportError:876 raise ImportError(877 "Could not import openai python package. "878 "Please install it with `pip install openai`."879 )880 if is_openai_v1():881 # For backwards compatibility. Before openai v1, no distinction was made882 # between azure_endpoint and base_url (openai_api_base).883 openai_api_base = values["openai_api_base"]884 if openai_api_base and values["validate_base_url"]:885 if "/openai" not in openai_api_base:886 values["openai_api_base"] = (887 values["openai_api_base"].rstrip("/") + "/openai"888 )889 warnings.warn(890 "As of openai>=1.0.0, Azure endpoints should be specified via "891 f"the `azure_endpoint` param not `openai_api_base` "892 f"(or alias `base_url`). Updating `openai_api_base` from "893 f"{openai_api_base} to {values['openai_api_base']}."894 )895 if values["deployment_name"]:896 warnings.warn(897 "As of openai>=1.0.0, if `deployment_name` (or alias "898 "`azure_deployment`) is specified then "899 "`openai_api_base` (or alias `base_url`) should not be. "900 "Instead use `deployment_name` (or alias `azure_deployment`) "901 "and `azure_endpoint`."902 )903 if values["deployment_name"] not in values["openai_api_base"]:904 warnings.warn(905 "As of openai>=1.0.0, if `openai_api_base` "906 "(or alias `base_url`) is specified it is expected to be "907 "of the form "908 "https://example-resource.azure.openai.com/openai/deployments/example-deployment. " # noqa: E501909 f"Updating {openai_api_base} to "910 f"{values['openai_api_base']}."911 )912 values["openai_api_base"] += (913 "/deployments/" + values["deployment_name"]914 )915 values["deployment_name"] = None916 client_params = {917 "api_version": values["openai_api_version"],918 "azure_endpoint": values["azure_endpoint"],919 "azure_deployment": values["deployment_name"],920 "api_key": values["openai_api_key"],921 "azure_ad_token": values["azure_ad_token"],922 "azure_ad_token_provider": values["azure_ad_token_provider"],923 "organization": values["openai_organization"],924 "base_url": values["openai_api_base"],925 "timeout": values["request_timeout"],926 "max_retries": values["max_retries"],927 "default_headers": {928 **(values["default_headers"] or {}),929 "User-Agent": "langchain-comm-python-azure-openai",930 },931 "default_query": values["default_query"],932 "http_client": values["http_client"],933 }934 values["client"] = openai.AzureOpenAI(**client_params).completions935 936 azure_ad_async_token_provider = values["azure_ad_async_token_provider"]937 938 if azure_ad_async_token_provider:939 client_params["azure_ad_token_provider"] = azure_ad_async_token_provider940 941 values["async_client"] = openai.AsyncAzureOpenAI(942 **client_params943 ).completions944 945 else:946 values["client"] = openai.Completion947 948 return values949 950 @property951 def _identifying_params(self) -> Mapping[str, Any]:952 return {953 **{"deployment_name": self.deployment_name},954 **super()._identifying_params,955 }956 957 @property958 def _invocation_params(self) -> Dict[str, Any]:959 if is_openai_v1():960 openai_params = {"model": self.deployment_name}961 else:962 openai_params = {963 "engine": self.deployment_name,964 "api_type": self.openai_api_type,965 "api_version": self.openai_api_version,966 }967 return {**openai_params, **super()._invocation_params}968 969 @property970 def _llm_type(self) -> str:971 """Return type of llm."""972 return "azure"973 974 @property975 def lc_attributes(self) -> Dict[str, Any]:976 return {977 "openai_api_type": self.openai_api_type,978 "openai_api_version": self.openai_api_version,979 }980 981 982@deprecated(983 since="0.0.1",984 removal="1.0",985 alternative_import="langchain_openai.ChatOpenAI",986)987class OpenAIChat(BaseLLM):988 """OpenAI Chat large language models.989 990 To use, you should have the ``openai`` python package installed, and the991 environment variable ``OPENAI_API_KEY`` set with your API key.992 993 Any parameters that are valid to be passed to the openai.create call can be passed994 in, even if not explicitly saved on this class.995 996 Example:997 .. code-block:: python998 999 from langchain_community.llms import OpenAIChat1000 openaichat = OpenAIChat(model_name="gpt-3.5-turbo")1001 """1002 1003 client: Any = Field(default=None, exclude=True) #: :meta private:1004 async_client: Any = Field(default=None, exclude=True) #: :meta private:1005 model_name: str = "gpt-3.5-turbo"1006 """Model name to use."""1007 model_kwargs: Dict[str, Any] = Field(default_factory=dict)1008 """Holds any model parameters valid for `create` call not explicitly specified."""1009 # When updating this to use a SecretStr1010 # Check for classes that derive from this class (as some of them1011 # may assume openai_api_key is a str)1012 openai_api_key: Optional[str] = Field(default=None, alias="api_key")1013 """Automatically inferred from env var `OPENAI_API_KEY` if not provided."""1014 openai_api_base: Optional[str] = Field(default=None, alias="base_url")1015 """Base URL path for API requests, leave blank if not using a proxy or service 1016 emulator."""1017 # to support explicit proxy for OpenAI1018 openai_proxy: Optional[str] = None1019 max_retries: int = 61020 """Maximum number of retries to make when generating."""1021 prefix_messages: List = Field(default_factory=list)1022 """Series of messages for Chat input."""1023 streaming: bool = False1024 """Whether to stream the results or not."""1025 allowed_special: Union[Literal["all"], AbstractSet[str]] = set()1026 """Set of special tokens that are allowed。"""1027 disallowed_special: Union[Literal["all"], Collection[str]] = "all"1028 """Set of special tokens that are not allowed。"""1029 1030 @model_validator(mode="before")1031 @classmethod1032 def build_extra(cls, values: Dict[str, Any]) -> Any:1033 """Build extra kwargs from additional params that were passed in."""1034 all_required_field_names = {field.alias for field in get_fields(cls).values()}1035 1036 extra = values.get("model_kwargs", {})1037 for field_name in list(values):1038 if field_name not in all_required_field_names:1039 if field_name in extra:1040 raise ValueError(f"Found {field_name} supplied twice.")1041 extra[field_name] = values.pop(field_name)1042 values["model_kwargs"] = extra1043 return values1044 1045 @pre_init1046 def validate_environment(cls, values: Dict) -> Dict:1047 """Validate that api key and python package exists in environment."""1048 openai_api_key = get_from_dict_or_env(1049 values, "openai_api_key", "OPENAI_API_KEY"1050 )1051 openai_api_base = get_from_dict_or_env(1052 values,1053 "openai_api_base",1054 "OPENAI_API_BASE",1055 default="",1056 )1057 openai_proxy = get_from_dict_or_env(1058 values,1059 "openai_proxy",1060 "OPENAI_PROXY",1061 default="",1062 )1063 openai_organization = get_from_dict_or_env(1064 values, "openai_organization", "OPENAI_ORGANIZATION", default=""1065 )1066 try:1067 import openai1068 1069 openai.api_key = openai_api_key1070 if openai_api_base:1071 openai.api_base = openai_api_base1072 if openai_organization:1073 openai.organization = openai_organization1074 if openai_proxy:1075 openai.proxy = {"http": openai_proxy, "https": openai_proxy}1076 except ImportError:1077 raise ImportError(1078 "Could not import openai python package. "1079 "Please install it with `pip install openai`."1080 )1081 try:1082 values["client"] = openai.ChatCompletion1083 except AttributeError:1084 raise ValueError(1085 "`openai` has no `ChatCompletion` attribute, this is likely "1086 "due to an old version of the openai package. Try upgrading it "1087 "with `pip install --upgrade openai`."1088 )1089 warnings.warn(1090 "You are trying to use a chat model. This way of initializing it is "1091 "no longer supported. Instead, please use: "1092 "`from langchain_community.chat_models import ChatOpenAI`"1093 )1094 return values1095 1096 @property1097 def _default_params(self) -> Dict[str, Any]:1098 """Get the default parameters for calling OpenAI API."""1099 return self.model_kwargs1100 1101 def _get_chat_params(1102 self, prompts: List[str], stop: Optional[List[str]] = None1103 ) -> Tuple:1104 if len(prompts) > 1:1105 raise ValueError(1106 f"OpenAIChat currently only supports single prompt, got {prompts}"1107 )1108 messages = self.prefix_messages + [{"role": "user", "content": prompts[0]}]1109 params: Dict[str, Any] = {**{"model": self.model_name}, **self._default_params}1110 if stop is not None:1111 if "stop" in params:1112 raise ValueError("`stop` found in both the input and default params.")1113 params["stop"] = stop1114 if params.get("max_tokens") == -1:1115 # for ChatGPT api, omitting max_tokens is equivalent to having no limit1116 del params["max_tokens"]1117 return messages, params1118 1119 def _stream(1120 self,1121 prompt: str,1122 stop: Optional[List[str]] = None,1123 run_manager: Optional[CallbackManagerForLLMRun] = None,1124 **kwargs: Any,1125 ) -> Iterator[GenerationChunk]:1126 messages, params = self._get_chat_params([prompt], stop)1127 params = {**params, **kwargs, "stream": True}1128 for stream_resp in completion_with_retry(1129 self, messages=messages, run_manager=run_manager, **params1130 ):1131 if not isinstance(stream_resp, dict):1132 stream_resp = stream_resp.dict()1133 token = stream_resp["choices"][0]["delta"].get("content", "")1134 chunk = GenerationChunk(text=token)1135 if run_manager:1136 run_manager.on_llm_new_token(token, chunk=chunk)1137 yield chunk1138 1139 async def _astream(1140 self,1141 prompt: str,1142 stop: Optional[List[str]] = None,1143 run_manager: Optional[AsyncCallbackManagerForLLMRun] = None,1144 **kwargs: Any,1145 ) -> AsyncIterator[GenerationChunk]:1146 messages, params = self._get_chat_params([prompt], stop)1147 params = {**params, **kwargs, "stream": True}1148 async for stream_resp in await acompletion_with_retry(1149 self, messages=messages, run_manager=run_manager, **params1150 ):1151 if not isinstance(stream_resp, dict):1152 stream_resp = stream_resp.dict()1153 token = stream_resp["choices"][0]["delta"].get("content", "")1154 chunk = GenerationChunk(text=token)1155 if run_manager:1156 await run_manager.on_llm_new_token(token, chunk=chunk)1157 yield chunk1158 1159 def _generate(1160 self,1161 prompts: List[str],1162 stop: Optional[List[str]] = None,1163 run_manager: Optional[CallbackManagerForLLMRun] = None,1164 **kwargs: Any,1165 ) -> LLMResult:1166 if self.streaming:1167 generation: Optional[GenerationChunk] = None1168 for chunk in self._stream(prompts[0], stop, run_manager, **kwargs):1169 if generation is None:1170 generation = chunk1171 else:1172 generation += chunk1173 assert generation is not None1174 return LLMResult(generations=[[generation]])1175 1176 messages, params = self._get_chat_params(prompts, stop)1177 params = {**params, **kwargs}1178 full_response = completion_with_retry(1179 self, messages=messages, run_manager=run_manager, **params1180 )1181 if not isinstance(full_response, dict):1182 full_response = full_response.dict()1183 llm_output = {1184 "token_usage": full_response["usage"],1185 "model_name": self.model_name,1186 }1187 return LLMResult(1188 generations=[1189 [Generation(text=full_response["choices"][0]["message"]["content"])]1190 ],1191 llm_output=llm_output,1192 )1193 1194 async def _agenerate(1195 self,1196 prompts: List[str],1197 stop: Optional[List[str]] = None,1198 run_manager: Optional[AsyncCallbackManagerForLLMRun] = None,1199 **kwargs: Any,1200 ) -> LLMResult: