itsanan/codebert-embed-crewai-base
091
1---2language:3- en4license: apache-2.05tags:6- sentence-transformers7- sentence-similarity8- feature-extraction9- dense10- generated_from_trainer11- dataset_size:90012- loss:MatryoshkaLoss13- loss:MultipleNegativesRankingLoss14base_model: microsoft/codebert-base15widget:16- source_sentence: Explain the test_code_docs_search_tool logic17 sentences:18 - "def test_anthropic_call_with_interceptor_tracks_requests(self) -> None:\n \19 \ \"\"\"Test that interceptor tracks Anthropic API requests.\"\"\"\n \20 \ interceptor = AnthropicTestInterceptor()\n llm = LLM(model=\"anthropic/claude-3-5-haiku-20241022\"\21 , interceptor=interceptor)\n\n # Make a simple completion call\n \22 \ result = llm.call(\n messages=[{\"role\": \"user\", \"content\"\23 : \"Say 'Hello World' and nothing else\"}]\n )\n\n # Verify custom\24 \ headers were added\n for request in interceptor.outbound_calls:\n \25 \ assert \"X-Anthropic-Interceptor\" in request.headers\n assert\26 \ request.headers[\"X-Anthropic-Interceptor\"] == \"anthropic-test-value\"\n \27 \ assert \"X-Request-ID\" in request.headers\n assert request.headers[\"\28 X-Request-ID\"] == \"test-request-456\"\n\n # Verify response was tracked\n\29 \ for response in interceptor.inbound_calls:\n assert \"X-Response-Tracked\"\30 \ in response.headers\n assert response.headers[\"X-Response-Tracked\"\31 ] == \"true\"\n\n # Verify result is valid\n assert result is not\32 \ None\n assert isinstance(result, str)\n assert len(result) > 0"33 - "def on_inbound(self, message: httpx.Response) -> httpx.Response:\n \"\"\34 \"Pass through inbound response.\n\n Args:\n message: The inbound\35 \ response.\n\n Returns:\n The response unchanged.\n \36 \ \"\"\"\n return message"37 - "def test_code_docs_search_tool(mock_adapter):\n mock_adapter.query.return_value\38 \ = \"test documentation\"\n\n docs_url = \"https://crewai.com/any-docs-url\"\39 \n search_query = \"test documentation\"\n tool = CodeDocsSearchTool(docs_url=docs_url,\40 \ adapter=mock_adapter)\n result = tool._run(search_query=search_query)\n \41 \ assert \"test documentation\" in result\n mock_adapter.add.assert_called_once_with(docs_url,\42 \ data_type=DataType.DOCS_SITE)\n mock_adapter.query.assert_called_once_with(\n\43 \ search_query, similarity_threshold=0.6, limit=5\n )\n\n mock_adapter.query.reset_mock()\n\44 \ mock_adapter.add.reset_mock()\n\n tool = CodeDocsSearchTool(adapter=mock_adapter)\n\45 \ result = tool._run(docs_url=docs_url, search_query=search_query)\n assert\46 \ \"test documentation\" in result\n mock_adapter.add.assert_called_once_with(docs_url,\47 \ data_type=DataType.DOCS_SITE)\n mock_adapter.query.assert_called_once_with(\n\48 \ search_query, similarity_threshold=0.6, limit=5\n )"49- source_sentence: Explain the test_openai_get_client_params_with_base_url_priority50 logic51 sentences:52 - "def test_openai_get_client_params_with_base_url_priority():\n \"\"\"\n \53 \ Test that base_url takes priority over api_base in _get_client_params\n \"\54 \"\"\n llm = OpenAICompletion(\n model=\"gpt-4o\",\n base_url=\"\55 https://priority.openai.com/v1\",\n api_base=\"https://fallback.openai.com/v1\"\56 ,\n )\n client_params = llm._get_client_params()\n assert client_params[\"\57 base_url\"] == \"https://priority.openai.com/v1\""58 - "def get_context_window_size(self) -> int:\n \"\"\"Get the context window\59 \ size for the LLM.\n\n Returns:\n The number of tokens/characters\60 \ the model can handle.\n \"\"\"\n # Default implementation - subclasses\61 \ should override with model-specific values\n return DEFAULT_CONTEXT_WINDOW_SIZE"62 - "def _inject_date_to_task(self, task: Task) -> None:\n \"\"\"Inject the\63 \ current date into the task description if inject_date is enabled.\"\"\"\n \64 \ if self.inject_date:\n from datetime import datetime\n\n \65 \ try:\n valid_format_codes = [\n \"\66 %Y\",\n \"%m\",\n \"%d\",\n \67 \ \"%H\",\n \"%M\",\n \"%S\",\n \68 \ \"%B\",\n \"%b\",\n \69 \ \"%A\",\n \"%a\",\n ]\n is_valid\70 \ = any(code in self.date_format for code in valid_format_codes)\n\n \71 \ if not is_valid:\n raise ValueError(f\"Invalid date\72 \ format: {self.date_format}\")\n\n current_date = datetime.now().strftime(self.date_format)\n\73 \ task.description += f\"\\n\\nCurrent Date: {current_date}\"\n\74 \ except Exception as e:\n self._logger.log(\"warning\"\75 , f\"Failed to inject date: {e!s}\")"76- source_sentence: How to implement async _get_connection?77 sentences:78 - "def mock_env():\n with patch.dict(os.environ, {\"CREWAI_PERSONAL_ACCESS_TOKEN\"\79 : \"test_token\"}):\n os.environ.pop(\"CREWAI_PLUS_URL\", None)\n \80 \ yield"81 - "async def _get_connection(self) -> SnowflakeConnection:\n \"\"\"Get a\82 \ connection from the pool or create a new one.\"\"\"\n if self._pool_lock\83 \ is None:\n raise RuntimeError(\"Pool lock not initialized\")\n \84 \ if self._connection_pool is None:\n raise RuntimeError(\"Connection\85 \ pool not initialized\")\n async with self._pool_lock:\n if\86 \ not self._connection_pool:\n conn = await asyncio.get_event_loop().run_in_executor(\n\87 \ self._thread_pool, self._create_connection\n \88 \ )\n self._connection_pool.append(conn)\n return\89 \ self._connection_pool.pop()"90 - "async def _arun(self, selector: str, thread_id: str = \"default\", **kwargs)\91 \ -> str:\n \"\"\"Use the async tool.\"\"\"\n try:\n \92 \ # Get the current page\n page = await self.get_async_page(thread_id)\n\93 \n # Click on the element\n selector_effective = self._selector_effective(selector=selector)\n\94 \ from playwright.async_api import TimeoutError as PlaywrightTimeoutError\n\95 \n try:\n await page.click(\n selector_effective,\n\96 \ strict=self.playwright_strict,\n timeout=self.playwright_timeout,\n\97 \ )\n except PlaywrightTimeoutError:\n \98 \ return f\"Unable to click on element '{selector}'\"\n except Exception\99 \ as click_error:\n return f\"Unable to click on element '{selector}':\100 \ {click_error!s}\"\n\n return f\"Clicked element '{selector}'\"\n\101 \ except Exception as e:\n return f\"Error clicking on element:\102 \ {e!s}\""103- source_sentence: Example usage of test_personal_access_token_from_environment104 sentences:105 - "async def close(self):\n return None"106 - "def test_structured_state_persistence(tmp_path):\n \"\"\"Test persistence\107 \ with Pydantic model state.\"\"\"\n db_path = os.path.join(tmp_path, \"test_flows.db\"\108 )\n persistence = SQLiteFlowPersistence(db_path)\n\n class StructuredFlow(Flow[TestState]):\n\109 \ initial_state = TestState\n\n @start()\n @persist(persistence)\n\110 \ def count_up(self):\n self.state.counter += 1\n \111 \ self.state.message = f\"Count is {self.state.counter}\"\n\n # Run flow and\112 \ verify state changes are saved\n flow = StructuredFlow(persistence=persistence)\n\113 \ flow.kickoff()\n\n # Load and verify state\n saved_state = persistence.load_state(flow.state.id)\n\114 \ assert saved_state is not None\n assert saved_state[\"counter\"] == 1\n\115 \ assert saved_state[\"message\"] == \"Count is 1\""116 - "def test_personal_access_token_from_environment(tool):\n assert tool.personal_access_token\117 \ == \"test_token\""118- source_sentence: Best practices for handle_a2a_polling_started119 sentences:120 - "def external_supported_storages() -> dict[str, Any]:\n return {\n \121 \ \"mem0\": ExternalMemory._configure_mem0,\n }"122 - "def handle_a2a_polling_started(\n self,\n task_id: str,\n \123 \ polling_interval: float,\n endpoint: str,\n ) -> None:\n \124 \ \"\"\"Handle A2A polling started event with panel display.\"\"\"\n content\125 \ = Text()\n content.append(\"A2A Polling Started\\n\", style=\"cyan bold\"\126 )\n content.append(\"Task ID: \", style=\"white\")\n content.append(f\"\127 {task_id[:8]}...\\n\", style=\"cyan\")\n content.append(\"Interval: \"\128 , style=\"white\")\n content.append(f\"{polling_interval}s\\n\", style=\"\129 cyan\")\n\n self.print_panel(content, \"⏳ A2A Polling\", \"cyan\")"130 - "def test_agent_with_knowledge_sources_generate_search_query():\n content =\131 \ \"Brandon's favorite color is red and he likes Mexican food.\"\n string_source\132 \ = StringKnowledgeSource(content=content)\n\n with (\n patch(\"crewai.knowledge\"\133 ) as mock_knowledge,\n patch(\n \"crewai.knowledge.storage.knowledge_storage.KnowledgeStorage\"\134 \n ) as mock_knowledge_storage,\n patch(\n \"crewai.knowledge.source.base_knowledge_source.KnowledgeStorage\"\135 \n ) as mock_base_knowledge_storage,\n patch(\"crewai.rag.chromadb.client.ChromaDBClient\"\136 ) as mock_chromadb,\n ):\n mock_knowledge_instance = mock_knowledge.return_value\n\137 \ mock_knowledge_instance.sources = [string_source]\n mock_knowledge_instance.query.return_value\138 \ = [{\"content\": content}]\n\n mock_storage_instance = mock_knowledge_storage.return_value\n\139 \ mock_storage_instance.sources = [string_source]\n mock_storage_instance.query.return_value\140 \ = [{\"content\": content}]\n mock_storage_instance.save.return_value\141 \ = None\n\n mock_chromadb_instance = mock_chromadb.return_value\n \142 \ mock_chromadb_instance.add_documents.return_value = None\n\n mock_base_knowledge_storage.return_value\143 \ = mock_storage_instance\n\n agent = Agent(\n role=\"Information\144 \ Agent with extensive role description that is longer than 80 characters\",\n\145 \ goal=\"Provide information based on knowledge sources\",\n \146 \ backstory=\"You have access to specific knowledge sources.\",\n \147 \ llm=LLM(model=\"gpt-4o-mini\"),\n knowledge_sources=[string_source],\n\148 \ )\n\n task = Task(\n description=\"What is Brandon's\149 \ favorite color?\",\n expected_output=\"The answer to the question,\150 \ in a format like this: `{{name: str, favorite_color: str}}`\",\n \151 \ agent=agent,\n )\n\n crew = Crew(agents=[agent], tasks=[task])\n\152 \ result = crew.kickoff()\n\n # Updated assertion to check the JSON\153 \ content\n assert \"Brandon\" in str(agent.knowledge_search_query)\n \154 \ assert \"favorite color\" in str(agent.knowledge_search_query)\n\n \155 \ assert \"red\" in result.raw.lower()"156pipeline_tag: sentence-similarity157library_name: sentence-transformers158metrics:159- cosine_accuracy@1160- cosine_accuracy@3161- cosine_accuracy@5162- cosine_accuracy@10163- cosine_precision@1164- cosine_precision@3165- cosine_precision@5166- cosine_precision@10167- cosine_recall@1168- cosine_recall@3169- cosine_recall@5170- cosine_recall@10171- cosine_ndcg@10172- cosine_mrr@10173- cosine_map@100174model-index:175- name: CodeBERT Fine-tuned on CrewAI176 results:177 - task:178 type: information-retrieval179 name: Information Retrieval180 dataset:181 name: dim 768182 type: dim_768183 metrics:184 - type: cosine_accuracy@1185 value: 0.57186 name: Cosine Accuracy@1187 - type: cosine_accuracy@3188 value: 0.57189 name: Cosine Accuracy@3190 - type: cosine_accuracy@5191 value: 0.57192 name: Cosine Accuracy@5193 - type: cosine_accuracy@10194 value: 0.65195 name: Cosine Accuracy@10196 - type: cosine_precision@1197 value: 0.57198 name: Cosine Precision@1199 - type: cosine_precision@3200 value: 0.57201 name: Cosine Precision@3202 - type: cosine_precision@5203 value: 0.57204 name: Cosine Precision@5205 - type: cosine_precision@10206 value: 0.325207 name: Cosine Precision@10208 - type: cosine_recall@1209 value: 0.11399999999999996210 name: Cosine Recall@1211 - type: cosine_recall@3212 value: 0.3420000000000001213 name: Cosine Recall@3214 - type: cosine_recall@5215 value: 0.57216 name: Cosine Recall@5217 - type: cosine_recall@10218 value: 0.65219 name: Cosine Recall@10220 - type: cosine_ndcg@10221 value: 0.6132795614223119222 name: Cosine Ndcg@10223 - type: cosine_mrr@10224 value: 0.5833333333333334225 name: Cosine Mrr@10226 - type: cosine_map@100227 value: 0.6323349876959563228 name: Cosine Map@100229 - task:230 type: information-retrieval231 name: Information Retrieval232 dataset:233 name: dim 512234 type: dim_512235 metrics:236 - type: cosine_accuracy@1237 value: 0.56238 name: Cosine Accuracy@1239 - type: cosine_accuracy@3240 value: 0.56241 name: Cosine Accuracy@3242 - type: cosine_accuracy@5243 value: 0.56244 name: Cosine Accuracy@5245 - type: cosine_accuracy@10246 value: 0.68247 name: Cosine Accuracy@10248 - type: cosine_precision@1249 value: 0.56250 name: Cosine Precision@1251 - type: cosine_precision@3252 value: 0.56253 name: Cosine Precision@3254 - type: cosine_precision@5255 value: 0.56256 name: Cosine Precision@5257 - type: cosine_precision@10258 value: 0.34259 name: Cosine Precision@10260 - type: cosine_recall@1261 value: 0.11199999999999999262 name: Cosine Recall@1263 - type: cosine_recall@3264 value: 0.336265 name: Cosine Recall@3266 - type: cosine_recall@5267 value: 0.56268 name: Cosine Recall@5269 - type: cosine_recall@10270 value: 0.68271 name: Cosine Recall@10272 - type: cosine_ndcg@10273 value: 0.6249193421334678274 name: Cosine Ndcg@10275 - type: cosine_mrr@10276 value: 0.5799999999999998277 name: Cosine Mrr@10278 - type: cosine_map@100279 value: 0.6328444860345127280 name: Cosine Map@100281 - task:282 type: information-retrieval283 name: Information Retrieval284 dataset:285 name: dim 256286 type: dim_256287 metrics:288 - type: cosine_accuracy@1289 value: 0.54290 name: Cosine Accuracy@1291 - type: cosine_accuracy@3292 value: 0.54293 name: Cosine Accuracy@3294 - type: cosine_accuracy@5295 value: 0.54296 name: Cosine Accuracy@5297 - type: cosine_accuracy@10298 value: 0.67299 name: Cosine Accuracy@10300 - type: cosine_precision@1301 value: 0.54302 name: Cosine Precision@1303 - type: cosine_precision@3304 value: 0.54305 name: Cosine Precision@3306 - type: cosine_precision@5307 value: 0.54308 name: Cosine Precision@5309 - type: cosine_precision@10310 value: 0.335311 name: Cosine Precision@10312 - type: cosine_recall@1313 value: 0.10799999999999997314 name: Cosine Recall@1315 - type: cosine_recall@3316 value: 0.324317 name: Cosine Recall@3318 - type: cosine_recall@5319 value: 0.54320 name: Cosine Recall@5321 - type: cosine_recall@10322 value: 0.67323 name: Cosine Recall@10324 - type: cosine_ndcg@10325 value: 0.6103292873112568326 name: Cosine Ndcg@10327 - type: cosine_mrr@10328 value: 0.5616666666666664329 name: Cosine Mrr@10330 - type: cosine_map@100331 value: 0.622676615058847332 name: Cosine Map@100333 - task:334 type: information-retrieval335 name: Information Retrieval336 dataset:337 name: dim 128338 type: dim_128339 metrics:340 - type: cosine_accuracy@1341 value: 0.47342 name: Cosine Accuracy@1343 - type: cosine_accuracy@3344 value: 0.47345 name: Cosine Accuracy@3346 - type: cosine_accuracy@5347 value: 0.47348 name: Cosine Accuracy@5349 - type: cosine_accuracy@10350 value: 0.58351 name: Cosine Accuracy@10352 - type: cosine_precision@1353 value: 0.47354 name: Cosine Precision@1355 - type: cosine_precision@3356 value: 0.47357 name: Cosine Precision@3358 - type: cosine_precision@5359 value: 0.47360 name: Cosine Precision@5361 - type: cosine_precision@10362 value: 0.29363 name: Cosine Precision@10364 - type: cosine_recall@1365 value: 0.09399999999999999366 name: Cosine Recall@1367 - type: cosine_recall@3368 value: 0.28200000000000003369 name: Cosine Recall@3370 - type: cosine_recall@5371 value: 0.47372 name: Cosine Recall@5373 - type: cosine_recall@10374 value: 0.58375 name: Cosine Recall@10376 - type: cosine_ndcg@10377 value: 0.5295093969556788378 name: Cosine Ndcg@10379 - type: cosine_mrr@10380 value: 0.48833333333333323381 name: Cosine Mrr@10382 - type: cosine_map@100383 value: 0.5581789904714569384 name: Cosine Map@100385 - task:386 type: information-retrieval387 name: Information Retrieval388 dataset:389 name: dim 64390 type: dim_64391 metrics:392 - type: cosine_accuracy@1393 value: 0.5394 name: Cosine Accuracy@1395 - type: cosine_accuracy@3396 value: 0.5397 name: Cosine Accuracy@3398 - type: cosine_accuracy@5399 value: 0.5400 name: Cosine Accuracy@5401 - type: cosine_accuracy@10402 value: 0.6403 name: Cosine Accuracy@10404 - type: cosine_precision@1405 value: 0.5406 name: Cosine Precision@1407 - type: cosine_precision@3408 value: 0.5409 name: Cosine Precision@3410 - type: cosine_precision@5411 value: 0.5412 name: Cosine Precision@5413 - type: cosine_precision@10414 value: 0.3415 name: Cosine Precision@10416 - type: cosine_recall@1417 value: 0.1418 name: Cosine Recall@1419 - type: cosine_recall@3420 value: 0.3421 name: Cosine Recall@3422 - type: cosine_recall@5423 value: 0.5424 name: Cosine Recall@5425 - type: cosine_recall@10426 value: 0.6427 name: Cosine Recall@10428 - type: cosine_ndcg@10429 value: 0.5540994517778899430 name: Cosine Ndcg@10431 - type: cosine_mrr@10432 value: 0.5166666666666665433 name: Cosine Mrr@10434 - type: cosine_map@100435 value: 0.5748485156077728436 name: Cosine Map@100437---438 439# CodeBERT Fine-tuned on CrewAI440 441This is a [sentence-transformers](https://www.SBERT.net) model finetuned from [microsoft/codebert-base](https://huggingface.co/microsoft/codebert-base). It maps sentences & paragraphs to a 768-dimensional dense vector space and can be used for semantic textual similarity, semantic search, paraphrase mining, text classification, clustering, and more.442 443## Model Details444 445### Model Description446- **Model Type:** Sentence Transformer447- **Base model:** [microsoft/codebert-base](https://huggingface.co/microsoft/codebert-base) <!-- at revision 3b0952feddeffad0063f274080e3c23d75e7eb39 -->448- **Maximum Sequence Length:** 512 tokens449- **Output Dimensionality:** 768 dimensions450- **Similarity Function:** Cosine Similarity451<!-- - **Training Dataset:** Unknown -->452- **Language:** en453- **License:** apache-2.0454 455### Model Sources456 457- **Documentation:** [Sentence Transformers Documentation](https://sbert.net)458- **Repository:** [Sentence Transformers on GitHub](https://github.com/huggingface/sentence-transformers)459- **Hugging Face:** [Sentence Transformers on Hugging Face](https://huggingface.co/models?library=sentence-transformers)460 461### Full Model Architecture462 463```464SentenceTransformer(465 (0): Transformer({'max_seq_length': 512, 'do_lower_case': False, 'architecture': 'RobertaModel'})466 (1): Pooling({'word_embedding_dimension': 768, 'pooling_mode_cls_token': False, 'pooling_mode_mean_tokens': True, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False, 'pooling_mode_weightedmean_tokens': False, 'pooling_mode_lasttoken': False, 'include_prompt': True})467)468```469 470## Usage471 472### Direct Usage (Sentence Transformers)473 474First install the Sentence Transformers library:475 476```bash477pip install -U sentence-transformers478```479 480Then you can load this model and run inference.481```python482from sentence_transformers import SentenceTransformer483 484# Download from the 🤗 Hub485model = SentenceTransformer("itsanan/codebert-embed-crewai-base")486# Run inference487sentences = [488 'Best practices for handle_a2a_polling_started',489 'def handle_a2a_polling_started(\n self,\n task_id: str,\n polling_interval: float,\n endpoint: str,\n ) -> None:\n """Handle A2A polling started event with panel display."""\n content = Text()\n content.append("A2A Polling Started\\n", style="cyan bold")\n content.append("Task ID: ", style="white")\n content.append(f"{task_id[:8]}...\\n", style="cyan")\n content.append("Interval: ", style="white")\n content.append(f"{polling_interval}s\\n", style="cyan")\n\n self.print_panel(content, "⏳ A2A Polling", "cyan")',490 'def test_agent_with_knowledge_sources_generate_search_query():\n content = "Brandon\'s favorite color is red and he likes Mexican food."\n string_source = StringKnowledgeSource(content=content)\n\n with (\n patch("crewai.knowledge") as mock_knowledge,\n patch(\n "crewai.knowledge.storage.knowledge_storage.KnowledgeStorage"\n ) as mock_knowledge_storage,\n patch(\n "crewai.knowledge.source.base_knowledge_source.KnowledgeStorage"\n ) as mock_base_knowledge_storage,\n patch("crewai.rag.chromadb.client.ChromaDBClient") as mock_chromadb,\n ):\n mock_knowledge_instance = mock_knowledge.return_value\n mock_knowledge_instance.sources = [string_source]\n mock_knowledge_instance.query.return_value = [{"content": content}]\n\n mock_storage_instance = mock_knowledge_storage.return_value\n mock_storage_instance.sources = [string_source]\n mock_storage_instance.query.return_value = [{"content": content}]\n mock_storage_instance.save.return_value = None\n\n mock_chromadb_instance = mock_chromadb.return_value\n mock_chromadb_instance.add_documents.return_value = None\n\n mock_base_knowledge_storage.return_value = mock_storage_instance\n\n agent = Agent(\n role="Information Agent with extensive role description that is longer than 80 characters",\n goal="Provide information based on knowledge sources",\n backstory="You have access to specific knowledge sources.",\n llm=LLM(model="gpt-4o-mini"),\n knowledge_sources=[string_source],\n )\n\n task = Task(\n description="What is Brandon\'s favorite color?",\n expected_output="The answer to the question, in a format like this: `{{name: str, favorite_color: str}}`",\n agent=agent,\n )\n\n crew = Crew(agents=[agent], tasks=[task])\n result = crew.kickoff()\n\n # Updated assertion to check the JSON content\n assert "Brandon" in str(agent.knowledge_search_query)\n assert "favorite color" in str(agent.knowledge_search_query)\n\n assert "red" in result.raw.lower()',491]492embeddings = model.encode(sentences)493print(embeddings.shape)494# [3, 768]495 496# Get the similarity scores for the embeddings497similarities = model.similarity(embeddings, embeddings)498print(similarities)499# tensor([[1.0000, 0.7350, 0.6480],500# [0.7350, 1.0000, 0.8133],501# [0.6480, 0.8133, 1.0000]])502```503 504<!--505### Direct Usage (Transformers)506 507<details><summary>Click to see the direct usage in Transformers</summary>508 509</details>510-->511 512<!--513### Downstream Usage (Sentence Transformers)514 515You can finetune this model on your own dataset.516 517<details><summary>Click to expand</summary>518 519</details>520-->521 522<!--523### Out-of-Scope Use524 525*List how the model may foreseeably be misused and address what users ought not to do with the model.*526-->527 528## Evaluation529 530### Metrics531 532#### Information Retrieval533 534* Dataset: `dim_768`535* Evaluated with [<code>InformationRetrievalEvaluator</code>](https://sbert.net/docs/package_reference/sentence_transformer/evaluation.html#sentence_transformers.evaluation.InformationRetrievalEvaluator) with these parameters:536 ```json537 {538 "truncate_dim": 768539 }540 ```541 542| Metric | Value |543|:--------------------|:-----------|544| cosine_accuracy@1 | 0.57 |545| cosine_accuracy@3 | 0.57 |546| cosine_accuracy@5 | 0.57 |547| cosine_accuracy@10 | 0.65 |548| cosine_precision@1 | 0.57 |549| cosine_precision@3 | 0.57 |550| cosine_precision@5 | 0.57 |551| cosine_precision@10 | 0.325 |552| cosine_recall@1 | 0.114 |553| cosine_recall@3 | 0.342 |554| cosine_recall@5 | 0.57 |555| cosine_recall@10 | 0.65 |556| **cosine_ndcg@10** | **0.6133** |557| cosine_mrr@10 | 0.5833 |558| cosine_map@100 | 0.6323 |559 560#### Information Retrieval561 562* Dataset: `dim_512`563* Evaluated with [<code>InformationRetrievalEvaluator</code>](https://sbert.net/docs/package_reference/sentence_transformer/evaluation.html#sentence_transformers.evaluation.InformationRetrievalEvaluator) with these parameters:564 ```json565 {566 "truncate_dim": 512567 }568 ```569 570| Metric | Value |571|:--------------------|:-----------|572| cosine_accuracy@1 | 0.56 |573| cosine_accuracy@3 | 0.56 |574| cosine_accuracy@5 | 0.56 |575| cosine_accuracy@10 | 0.68 |576| cosine_precision@1 | 0.56 |577| cosine_precision@3 | 0.56 |578| cosine_precision@5 | 0.56 |579| cosine_precision@10 | 0.34 |580| cosine_recall@1 | 0.112 |581| cosine_recall@3 | 0.336 |582| cosine_recall@5 | 0.56 |583| cosine_recall@10 | 0.68 |584| **cosine_ndcg@10** | **0.6249** |585| cosine_mrr@10 | 0.58 |586| cosine_map@100 | 0.6328 |587 588#### Information Retrieval589 590* Dataset: `dim_256`591* Evaluated with [<code>InformationRetrievalEvaluator</code>](https://sbert.net/docs/package_reference/sentence_transformer/evaluation.html#sentence_transformers.evaluation.InformationRetrievalEvaluator) with these parameters:592 ```json593 {594 "truncate_dim": 256595 }596 ```597 598| Metric | Value |599|:--------------------|:-----------|600| cosine_accuracy@1 | 0.54 |601| cosine_accuracy@3 | 0.54 |602| cosine_accuracy@5 | 0.54 |603| cosine_accuracy@10 | 0.67 |604| cosine_precision@1 | 0.54 |605| cosine_precision@3 | 0.54 |606| cosine_precision@5 | 0.54 |607| cosine_precision@10 | 0.335 |608| cosine_recall@1 | 0.108 |609| cosine_recall@3 | 0.324 |610| cosine_recall@5 | 0.54 |611| cosine_recall@10 | 0.67 |612| **cosine_ndcg@10** | **0.6103** |613| cosine_mrr@10 | 0.5617 |614| cosine_map@100 | 0.6227 |615 616#### Information Retrieval617 618* Dataset: `dim_128`619* Evaluated with [<code>InformationRetrievalEvaluator</code>](https://sbert.net/docs/package_reference/sentence_transformer/evaluation.html#sentence_transformers.evaluation.InformationRetrievalEvaluator) with these parameters:620 ```json621 {622 "truncate_dim": 128623 }624 ```625 626| Metric | Value |627|:--------------------|:-----------|628| cosine_accuracy@1 | 0.47 |629| cosine_accuracy@3 | 0.47 |630| cosine_accuracy@5 | 0.47 |631| cosine_accuracy@10 | 0.58 |632| cosine_precision@1 | 0.47 |633| cosine_precision@3 | 0.47 |634| cosine_precision@5 | 0.47 |635| cosine_precision@10 | 0.29 |636| cosine_recall@1 | 0.094 |637| cosine_recall@3 | 0.282 |638| cosine_recall@5 | 0.47 |639| cosine_recall@10 | 0.58 |640| **cosine_ndcg@10** | **0.5295** |641| cosine_mrr@10 | 0.4883 |642| cosine_map@100 | 0.5582 |643 644#### Information Retrieval645 646* Dataset: `dim_64`647* Evaluated with [<code>InformationRetrievalEvaluator</code>](https://sbert.net/docs/package_reference/sentence_transformer/evaluation.html#sentence_transformers.evaluation.InformationRetrievalEvaluator) with these parameters:648 ```json649 {650 "truncate_dim": 64651 }652 ```653 654| Metric | Value |655|:--------------------|:-----------|656| cosine_accuracy@1 | 0.5 |657| cosine_accuracy@3 | 0.5 |658| cosine_accuracy@5 | 0.5 |659| cosine_accuracy@10 | 0.6 |660| cosine_precision@1 | 0.5 |661| cosine_precision@3 | 0.5 |662| cosine_precision@5 | 0.5 |663| cosine_precision@10 | 0.3 |664| cosine_recall@1 | 0.1 |665| cosine_recall@3 | 0.3 |666| cosine_recall@5 | 0.5 |667| cosine_recall@10 | 0.6 |668| **cosine_ndcg@10** | **0.5541** |669| cosine_mrr@10 | 0.5167 |670| cosine_map@100 | 0.5748 |671 672<!--673## Bias, Risks and Limitations674 675*What are the known or foreseeable issues stemming from this model? You could also flag here known failure cases or weaknesses of the model.*676-->677 678<!--679### Recommendations680 681*What are recommendations with respect to the foreseeable issues? For example, filtering explicit content.*682-->683 684## Training Details685 686### Training Dataset687 688#### Unnamed Dataset689 690* Size: 900 training samples691* Columns: <code>anchor</code> and <code>positive</code>692* Approximate statistics based on the first 900 samples:693 | | anchor | positive |694 |:--------|:-----------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------|695 | type | string | string |696 | details | <ul><li>min: 6 tokens</li><li>mean: 13.96 tokens</li><li>max: 141 tokens</li></ul> | <ul><li>min: 20 tokens</li><li>mean: 254.94 tokens</li><li>max: 512 tokens</li></ul> |697* Samples:698 | anchor | positive |699 |:---------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|700 | <code>Example usage of DeeplyNestedFlow</code> | <code>class DeeplyNestedFlow(Flow):<br> @start()<br> def a(self):<br> execution_order.append("a")<br><br> @start()<br> def b(self):<br> execution_order.append("b")<br><br> @start()<br> def c(self):<br> execution_order.append("c")<br><br> @start()<br> def d(self):<br> execution_order.append("d")<br><br> # Nested: (a AND b) OR (c AND d)<br> @listen(or_(and_(a, b), and_(c, d)))<br> def result(self):<br> execution_order.append("result")</code> |701 | <code>Explain the test_agent_with_knowledge_sources_generate_search_query logic</code> | <code>def test_agent_with_knowledge_sources_generate_search_query():<br> content = "Brandon's favorite color is red and he likes Mexican food."<br> string_source = StringKnowledgeSource(content=content)<br><br> with (<br> patch("crewai.knowledge") as mock_knowledge,<br> patch(<br> "crewai.knowledge.storage.knowledge_storage.KnowledgeStorage"<br> ) as mock_knowledge_storage,<br> patch(<br> "crewai.knowledge.source.base_knowledge_source.KnowledgeStorage"<br> ) as mock_base_knowledge_storage,<br> patch("crewai.rag.chromadb.client.ChromaDBClient") as mock_chromadb,<br> ):<br> mock_knowledge_instance = mock_knowledge.return_value<br> mock_knowledge_instance.sources = [string_source]<br> mock_knowledge_instance.query.return_value = [{"content": content}]<br><br> mock_storage_instance = mock_knowledge_storage.return_value<br> mock_storage_instance.sources = [string_source]<br> mock_storage_instance.query.return_value = [{"content": content}]...</code> |702 | <code>Example usage of agent</code> | <code>def agent(self) -> Agent \| None:<br> """Get the current agent associated with this memory."""<br> return self._agent</code> |703* Loss: [<code>MatryoshkaLoss</code>](https://sbert.net/docs/package_reference/sentence_transformer/losses.html#matryoshkaloss) with these parameters:704 ```json705 {706 "loss": "MultipleNegativesRankingLoss",707 "matryoshka_dims": [708 768,709 512,710 256,711 128,712 64713 ],714 "matryoshka_weights": [715 1,716 1,717 1,718 1,719 1720 ],721 "n_dims_per_step": -1722 }723 ```724 725### Training Hyperparameters726#### Non-Default Hyperparameters727 728- `eval_strategy`: epoch729- `per_device_train_batch_size`: 4730- `per_device_eval_batch_size`: 4731- `gradient_accumulation_steps`: 16732- `learning_rate`: 2e-05733- `num_train_epochs`: 4734- `lr_scheduler_type`: cosine735- `warmup_ratio`: 0.1736- `fp16`: True737- `load_best_model_at_end`: True738- `optim`: adamw_torch739- `batch_sampler`: no_duplicates740 741#### All Hyperparameters742<details><summary>Click to expand</summary>743 744- `overwrite_output_dir`: False745- `do_predict`: False746- `eval_strategy`: epoch747- `prediction_loss_only`: True748- `per_device_train_batch_size`: 4749- `per_device_eval_batch_size`: 4750- `per_gpu_train_batch_size`: None751- `per_gpu_eval_batch_size`: None752- `gradient_accumulation_steps`: 16753- `eval_accumulation_steps`: None754- `torch_empty_cache_steps`: None755- `learning_rate`: 2e-05756- `weight_decay`: 0.0757- `adam_beta1`: 0.9758- `adam_beta2`: 0.999759- `adam_epsilon`: 1e-08760- `max_grad_norm`: 1.0761- `num_train_epochs`: 4762- `max_steps`: -1763- `lr_scheduler_type`: cosine764- `lr_scheduler_kwargs`: None765- `warmup_ratio`: 0.1766- `warmup_steps`: 0767- `log_level`: passive768- `log_level_replica`: warning769- `log_on_each_node`: True770- `logging_nan_inf_filter`: True771- `save_safetensors`: True772- `save_on_each_node`: False773- `save_only_model`: False774- `restore_callback_states_from_checkpoint`: False775- `no_cuda`: False776- `use_cpu`: False777- `use_mps_device`: False778- `seed`: 42779- `data_seed`: None780- `jit_mode_eval`: False781- `bf16`: False782- `fp16`: True783- `fp16_opt_level`: O1784- `half_precision_backend`: auto785- `bf16_full_eval`: False786- `fp16_full_eval`: False787- `tf32`: None788- `local_rank`: 0789- `ddp_backend`: None790- `tpu_num_cores`: None791- `tpu_metrics_debug`: False792- `debug`: []793- `dataloader_drop_last`: False794- `dataloader_num_workers`: 0795- `dataloader_prefetch_factor`: None796- `past_index`: -1797- `disable_tqdm`: False798- `remove_unused_columns`: True799- `label_names`: None800- `load_best_model_at_end`: True801- `ignore_data_skip`: False802- `fsdp`: []803- `fsdp_min_num_params`: 0804- `fsdp_config`: {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}805- `fsdp_transformer_layer_cls_to_wrap`: None806- `accelerator_config`: {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}807- `parallelism_config`: None808- `deepspeed`: None809- `label_smoothing_factor`: 0.0810- `optim`: adamw_torch811- `optim_args`: None812- `adafactor`: False813- `group_by_length`: False814- `length_column_name`: length815- `project`: huggingface816- `trackio_space_id`: trackio817- `ddp_find_unused_parameters`: None818- `ddp_bucket_cap_mb`: None819- `ddp_broadcast_buffers`: False820- `dataloader_pin_memory`: True821- `dataloader_persistent_workers`: False822- `skip_memory_metrics`: True823- `use_legacy_prediction_loop`: False824- `push_to_hub`: False825- `resume_from_checkpoint`: None826- `hub_model_id`: None827- `hub_strategy`: every_save828- `hub_private_repo`: None829- `hub_always_push`: False830- `hub_revision`: None831- `gradient_checkpointing`: False832- `gradient_checkpointing_kwargs`: None833- `include_inputs_for_metrics`: False834- `include_for_metrics`: []835- `eval_do_concat_batches`: True836- `fp16_backend`: auto837- `push_to_hub_model_id`: None838- `push_to_hub_organization`: None839- `mp_parameters`: 840- `auto_find_batch_size`: False841- `full_determinism`: False842- `torchdynamo`: None843- `ray_scope`: last844- `ddp_timeout`: 1800845- `torch_compile`: False846- `torch_compile_backend`: None847- `torch_compile_mode`: None848- `include_tokens_per_second`: False849- `include_num_input_tokens_seen`: no850- `neftune_noise_alpha`: None851- `optim_target_modules`: None852- `batch_eval_metrics`: False853- `eval_on_start`: False854- `use_liger_kernel`: False855- `liger_kernel_config`: None856- `eval_use_gather_object`: False857- `average_tokens_across_devices`: True858- `prompts`: None859- `batch_sampler`: no_duplicates860- `multi_dataset_batch_sampler`: proportional861- `router_mapping`: {}862- `learning_rate_mapping`: {}863 864</details>865 866### Training Logs867| Epoch | Step | Training Loss | dim_768_cosine_ndcg@10 | dim_512_cosine_ndcg@10 | dim_256_cosine_ndcg@10 | dim_128_cosine_ndcg@10 | dim_64_cosine_ndcg@10 |868|:-------:|:------:|:-------------:|:----------------------:|:----------------------:|:----------------------:|:----------------------:|:---------------------:|869| 0.7111 | 10 | 7.1051 | - | - | - | - | - |870| 1.0 | 15 | - | 0.1170 | 0.06 | 0.0608 | 0.0825 | 0.0762 |871| 1.3556 | 20 | 6.4716 | - | - | - | - | - |872| 2.0 | 30 | 5.4463 | 0.1879 | 0.1770 | 0.1625 | 0.1816 | 0.1987 |873| 2.7111 | 40 | 3.7856 | - | - | - | - | - |874| 3.0 | 45 | - | 0.4987 | 0.5133 | 0.4587 | 0.4249 | 0.4425 |875| 3.3556 | 50 | 2.4942 | - | - | - | - | - |876| **4.0** | **60** | **1.71** | **0.6133** | **0.6249** | **0.6103** | **0.5295** | **0.5541** |877 878* The bold row denotes the saved checkpoint.879 880### Framework Versions881- Python: 3.12.12882- Sentence Transformers: 5.2.2883- Transformers: 4.57.6884- PyTorch: 2.9.0+cu126885- Accelerate: 1.12.0886- Datasets: 4.0.0887- Tokenizers: 0.22.2888 889## Citation890 891### BibTeX892 893#### Sentence Transformers894```bibtex895@inproceedings{reimers-2019-sentence-bert,896 title = "Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks",897 author = "Reimers, Nils and Gurevych, Iryna",898 booktitle = "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing",899 month = "11",900 year = "2019",901 publisher = "Association for Computational Linguistics",902 url = "https://arxiv.org/abs/1908.10084",903}904```905 906#### MatryoshkaLoss907```bibtex908@misc{kusupati2024matryoshka,909 title={Matryoshka Representation Learning},910 author={Aditya Kusupati and Gantavya Bhatt and Aniket Rege and Matthew Wallingford and Aditya Sinha and Vivek Ramanujan and William Howard-Snyder and Kaifeng Chen and Sham Kakade and Prateek Jain and Ali Farhadi},911 year={2024},912 eprint={2205.13147},913 archivePrefix={arXiv},914 primaryClass={cs.LG}915}916```917 918#### MultipleNegativesRankingLoss919```bibtex920@misc{henderson2017efficient,921 title={Efficient Natural Language Response Suggestion for Smart Reply},922 author={Matthew Henderson and Rami Al-Rfou and Brian Strope and Yun-hsuan Sung and Laszlo Lukacs and Ruiqi Guo and Sanjiv Kumar and Balint Miklos and Ray Kurzweil},923 year={2017},924 eprint={1705.00652},925 archivePrefix={arXiv},926 primaryClass={cs.CL}927}928```929 930<!--931## Glossary932 933*Clearly define terms in order to be accessible across audiences.*934-->935 936<!--937## Model Card Authors938 939*Lists the people who create the model card, providing recognition and accountability for the detailed work that goes into its construction.*940-->941 942<!--943## Model Card Contact944 945*Provides a way for people who have updates to the Model Card, suggestions, or questions, to contact the Model Card authors.*946-->