SahilCodevally/codevally-vision-language-action
0
1"""2Object Mapper — Synonym normalization for bridging YOLO labels3and natural-language user commands.4 5YOLO may detect "cup" while the user says "mug". This module provides6a mapping layer and fuzzy-match helper to reconcile the two vocabularies.7"""8 9import logging10from difflib import SequenceMatcher11 12logger = logging.getLogger(__name__)13 14# ---------------------------------------------------------------------------15# Synonym Map: user-language → canonical YOLO label16# ---------------------------------------------------------------------------17SYNONYM_MAP: dict[str, str] = {18 # ---------------------------------------------------------------19 # COCO has 80 classes. This map bridges common user vocabulary20 # to the actual YOLO/COCO label strings.21 # ---------------------------------------------------------------22 23 # person24 "human": "person",25 "worker": "person",26 "operator": "person",27 "man": "person",28 "woman": "person",29 "people": "person",30 "employee": "person",31 "technician": "person",32 33 # bicycle34 "bike": "bicycle",35 "cycle": "bicycle",36 37 # car38 "automobile": "car",39 "vehicle": "car",40 "sedan": "car",41 42 # motorcycle43 "motorbike": "motorcycle",44 "scooter": "motorcycle",45 46 # truck47 "van": "truck",48 "lorry": "truck",49 "pickup": "truck",50 51 # boat52 "ship": "boat",53 "vessel": "boat",54 "canoe": "boat",55 56 # traffic light57 "signal": "traffic light",58 "stoplight": "traffic light",59 60 # backpack61 "bag": "backpack",62 "rucksack": "backpack",63 "knapsack": "backpack",64 65 # umbrella66 "parasol": "umbrella",67 68 # handbag69 "purse": "handbag",70 "clutch": "handbag",71 72 # suitcase (toolboxes, cases look like suitcases to YOLO)73 "briefcase": "suitcase",74 "toolbox": "suitcase",75 "case": "suitcase",76 "luggage": "suitcase",77 "tool box": "suitcase",78 79 # bottle80 "water bottle": "bottle",81 "flask": "bottle",82 "jug": "bottle",83 "canteen": "bottle",84 "container": "bottle",85 86 # wine glass87 "goblet": "wine glass",88 "chalice": "wine glass",89 "glass": "wine glass",90 91 # cup92 "mug": "cup",93 "teacup": "cup",94 "coffee cup": "cup",95 "tumbler": "cup",96 97 # fork98 "prong": "fork",99 100 # knife (wrenches, tools often detected as knife-like)101 "blade": "knife",102 "wrench": "knife",103 "spanner": "knife",104 "screwdriver": "knife",105 "tool": "knife",106 107 # spoon108 "ladle": "spoon",109 "scoop": "spoon",110 111 # bowl112 "tray": "bowl",113 "dish": "bowl",114 "plate": "bowl",115 "bin": "bowl",116 "basket": "bowl",117 118 # banana / apple / orange (fruits)119 "fruit": "apple",120 121 # sandwich122 "sub": "sandwich",123 "burger": "sandwich",124 "wrap": "sandwich",125 126 # pizza127 "pie": "pizza",128 129 # cake130 "pastry": "cake",131 "dessert": "cake",132 133 # chair134 "seat": "chair",135 "stool": "chair",136 "office chair": "chair",137 138 # couch139 "sofa": "couch",140 "loveseat": "couch",141 "settee": "couch",142 143 # bed144 "mattress": "bed",145 "cot": "bed",146 147 # dining table148 "desk": "dining table",149 "table": "dining table",150 "workbench": "dining table",151 "bench": "dining table",152 "counter": "dining table",153 154 # tv155 "monitor": "tv",156 "screen": "tv",157 "display": "tv",158 "television": "tv",159 160 # laptop161 "notebook": "laptop",162 "computer": "laptop",163 "macbook": "laptop",164 165 # mouse166 "trackpad": "mouse",167 168 # remote169 "controller": "remote",170 "remote control": "remote",171 172 # keyboard173 "keypad": "keyboard",174 175 # cell phone176 "mobile": "cell phone",177 "phone": "cell phone",178 "smartphone": "cell phone",179 "iphone": "cell phone",180 "handset": "cell phone",181 "scanner": "cell phone",182 183 # microwave184 "microwave oven": "microwave",185 186 # oven187 "stove": "oven",188 "furnace": "oven",189 190 # refrigerator191 "fridge": "refrigerator",192 "cooler": "refrigerator",193 "freezer": "refrigerator",194 195 # book196 "notebook paper": "book",197 "manual": "book",198 "textbook": "book",199 "binder": "book",200 201 # clock202 "watch": "clock",203 "timer": "clock",204 205 # vase206 "pot": "vase",207 "planter": "vase",208 "jar": "vase",209 210 # scissors211 "cutter": "scissors",212 "shears": "scissors",213 "clipper": "scissors",214 "pliers": "scissors",215 "wire cutter": "scissors",216 217 # teddy bear218 "stuffed animal": "teddy bear",219 "plush": "teddy bear",220 "toy": "teddy bear",221 222 # potted plant223 "plant": "potted plant",224 "houseplant": "potted plant",225 "flower": "potted plant",226 227 # -----------------------------------------------------------228 # Domain-specific aliases for industrial / demo contexts229 # -----------------------------------------------------------230 "damaged part": "knife",231 "defective part": "knife",232 "component": "knife",233 "metal part": "knife",234 "part": "knife",235 "piece": "knife",236 237 "red bin": "bowl",238 "blue bin": "bowl",239 "sorting bin": "bowl",240 "reject bin": "bowl",241 "accept bin": "bowl",242 243 "circuit board": "keyboard",244 "pcb": "keyboard",245 "board": "keyboard",246 247 "hard hat": "sports ball",248 "helmet": "sports ball",249 "safety hat": "sports ball",250 251 "gloves": "backpack",252 "safety gloves": "backpack",253 254 "pallet": "suitcase",255 "crate": "suitcase",256 "carton": "suitcase",257 "box": "suitcase",258 259 "hand truck": "bicycle",260 "dolly": "bicycle",261 "cart": "bicycle",262 263 "forklift": "truck",264 265 "clipboard": "book",266 "blueprint": "book",267 268 "stethoscope": "tie",269 "syringe": "toothbrush",270 "safety goggles": "wine glass",271 "goggles": "wine glass",272 273 "oil can": "bottle",274 "spray can": "bottle",275 "battery": "suitcase",276 "car battery": "suitcase",277 278 "multimeter": "cell phone",279 "meter": "cell phone",280 "soldering iron": "knife",281 "magnifying lamp": "clock",282}283 284 285 286def normalize_label(label: str) -> str:287 """288 Normalize a label using the synonym map.289 290 Args:291 label: Raw label string (from user command or YOLO).292 293 Returns:294 The canonical label if a synonym exists, otherwise the295 original label lowercased and stripped.296 """297 cleaned = label.strip().lower()298 return SYNONYM_MAP.get(cleaned, cleaned)299 300 301def _similarity(a: str, b: str) -> float:302 """Return a 0-1 similarity score between two strings."""303 return SequenceMatcher(None, a.lower(), b.lower()).ratio()304 305 306def find_best_match(307 query: str,308 detected_labels: list[str],309 threshold: float = 0.4,310) -> str | None:311 """312 Find the best matching detected label for a user query term.313 314 Strategy:315 1. Exact match after normalization.316 2. Substring containment check.317 3. Fuzzy similarity above *threshold*.318 319 Args:320 query: The term from the user command (e.g. "mug").321 detected_labels: Labels returned by the vision detector.322 threshold: Minimum similarity score for fuzzy matching.323 324 Returns:325 The best matching label, or ``None`` if no match is found.326 """327 normalized_query = normalize_label(query)328 normalized_detected = {normalize_label(lbl): lbl for lbl in detected_labels}329 330 # 1. Exact match on normalized labels331 if normalized_query in normalized_detected:332 logger.info(333 "Exact match: '%s' → '%s'",334 query,335 normalized_detected[normalized_query],336 )337 return normalized_detected[normalized_query]338 339 # 2. Substring containment340 for norm_lbl, orig_lbl in normalized_detected.items():341 if normalized_query in norm_lbl or norm_lbl in normalized_query:342 logger.info(343 "Substring match: '%s' → '%s'", query, orig_lbl344 )345 return orig_lbl346 347 # 3. Fuzzy matching348 best_score = 0.0349 best_label: str | None = None350 for norm_lbl, orig_lbl in normalized_detected.items():351 score = _similarity(normalized_query, norm_lbl)352 if score > best_score:353 best_score = score354 best_label = orig_lbl355 356 if best_label and best_score >= threshold:357 logger.info(358 "Fuzzy match: '%s' → '%s' (score=%.2f)",359 query,360 best_label,361 best_score,362 )363 return best_label364 365 logger.warning("No match found for query '%s'", query)366 return None367