OpenTSLab/SciTS
SciTS: Scientific Time Series Understanding and Generation with LLMs This repository contains the official dataset for SciTS: Scientific Time Series Understanding and Generation with LLMs (ICLR 2026). SciTS is a large-scale benchmark designed to evaluate the capabilities of large language models on complex scientific time series data. It spans 12 scientific disciplines, 43 distinct tasks, and includes 54,023 instances. Dataset Structure The benchmark is organized… See the full description on the dataset page: https://huggingface.co/datasets/OpenTSLab/SciTS.
53.5k
1import json2from pathlib import Path3 4import numpy as np5import fire6import h5py7from sklearn.metrics import accuracy_score, mean_absolute_error, f1_score8 9from infer_eval_utils import read_time_series_data, concat_base_path, non_zero_rel_mae, DATASET_TO_TASK10 11 12class Runner(object):13 14 def multitask_classification(self,15 infer_path: str = "",16 gts: list = [],17 preds: list = []):18 tasks = gts[0].keys()19 output_fpath = Path(20 infer_path).parent / f"results/{Path(infer_path).stem}.json"21 output_fpath.parent.mkdir(parents=True, exist_ok=True)22 res_dict = {}23 success = 024 fail = 025 for task in tasks:26 correct_count = 027 for gt, pred in zip(gts, preds):28 try:29 if pred[task] == gt[task][0] or pred[task].lower(30 ) == gt[task][0]:31 correct_count += 132 success += 133 except:34 fail += 135 acc = correct_count / len(gts)36 print(f"Accuracy for {task}: {acc}")37 class_f1s = []38 task_gts = [gt[task][0] for gt in gts]39 labels = list(set(task_gts))40 for label in labels:41 # Find all samples with this true label42 true_indices = [43 i for i, gt in enumerate(task_gts) if gt == label44 ]45 if len(true_indices) == 0:46 recall = 0.047 else:48 correct_predictions = 049 for idx in true_indices:50 pred = preds[idx][task]51 gt = task_gts[idx]52 if pred == gt or pred.lower() == gt:53 correct_predictions += 154 55 recall = correct_predictions / len(true_indices)56 57 pred_indices = [58 i for i, pred in enumerate(preds)59 if pred[task].lower() == label or pred[task] == label60 ]61 if len(pred_indices) == 0:62 precision = 0.063 else:64 correct_predictions = 065 for idx in pred_indices:66 pred = preds[idx][task]67 gt = task_gts[idx]68 if pred == gt or pred.lower() == gt:69 correct_predictions += 170 precision = correct_predictions / len(pred_indices)71 72 f1 = (2 * recall * precision) / (recall + precision + 1e-6)73 class_f1s.append(f1)74 75 res_dict[task] = {76 "acc": acc,77 "f1": np.mean(class_f1s),78 "success": success,79 "fail": fail,80 "success_rate": success / (success + fail)81 }82 83 res_dict["overall"] = {84 "f1": np.mean([r["f1"] for r in res_dict.values()]),85 "acc": np.mean([r["acc"] for r in res_dict.values()]),86 }87 88 with open(output_fpath, "w") as writer:89 json.dump(res_dict, writer, indent=4)90 writer.write("\n")91 92 def multichoice_classification(self,93 infer_path: str = "",94 gts: list = [],95 preds: list = []):96 all_labels = set()97 for gt in gts:98 if isinstance(gt, list):99 all_labels.update(gt)100 else:101 all_labels.add(gt)102 103 all_labels = sorted(list(all_labels))104 105 # Convert ground truth and predictions to multi-label format106 y_true_multilabel = []107 y_pred_multilabel = []108 success = 0109 fail = 0110 for gt, pred in zip(gts, preds):111 # Process ground truth112 if isinstance(gt, list):113 gt_labels = gt114 else:115 gt_labels = [gt]116 117 # Process predictions118 while '\n\n' in pred:119 pred = pred.replace('\n\n', '\n')120 pred_labels = [x.strip() for x in pred.split("\n")]121 122 # Convert to binary vectors123 gt_binary = [124 1 if label in gt_labels else 0 for label in all_labels125 ]126 pred_binary = []127 for label in all_labels:128 is_found = False129 for pred_label in pred_labels:130 if pred_label == label or pred_label.lower() == label:131 is_found = True132 break133 if is_found:134 pred_binary.append(1)135 else:136 pred_binary.append(0)137 138 y_true_multilabel.append(gt_binary)139 y_pred_multilabel.append(pred_binary)140 141 y_true_multilabel = np.array(y_true_multilabel)142 y_pred_multilabel = np.array(y_pred_multilabel)143 144 # Calculate F1 score for each class145 f1_scores = []146 for i, label in enumerate(all_labels):147 f1 = f1_score(y_true_multilabel[:, i],148 y_pred_multilabel[:, i],149 zero_division=0)150 f1_scores.append(f1)151 print(f"F1 score for class {label}: {f1:.4f}")152 153 # Calculate mean of F1 scores154 macro_f1 = np.mean(f1_scores)155 print(f"Macro F1 score (mean of all classes): {macro_f1:.4f}")156 157 # Save results158 output_fpath = Path(159 infer_path).parent / f"results/{Path(infer_path).stem}.json"160 output_fpath.parent.mkdir(parents=True, exist_ok=True)161 162 results = {163 "macro_f1": macro_f1,164 "per_class_f1": dict(zip(all_labels, f1_scores))165 }166 167 with open(output_fpath, "w") as writer:168 json.dump(results, writer, indent=4)169 writer.write("\n")170 171 def classification(self, infer_path: str = ""):172 gts, preds = [], []173 with open(infer_path, "r") as f:174 for line in f:175 item = json.loads(line)176 if "id" not in item:177 continue178 gts.append(item["ground_truth"])179 preds.append(item["output"])180 181 if any(isinstance(gt, list) for gt in gts):182 return self.multichoice_classification(infer_path, gts, preds)183 184 if isinstance(preds[0], dict):185 return self.multitask_classification(infer_path, gts, preds)186 # Custom comparison function: consider both exact match and case-insensitive match187 # This is because LLM outputs sometimes capitalize the first letter to follow English grammar188 correct_count = 0189 for gt, pred in zip(gts, preds):190 if pred == gt or pred.lower() == gt:191 correct_count += 1192 193 acc = correct_count / len(gts)194 print(f"Accuracy: {acc}")195 196 # Using the same case-insensitive matching as accuracy calculation197 labels = list(set(gts))198 199 # Manually calculate recall for each class200 class_recalls = []201 class_precisions = []202 class_f1s = []203 for label in labels:204 # Find all samples with this true label205 true_indices = [i for i, gt in enumerate(gts) if gt == label]206 if len(true_indices) == 0:207 recall = 0.0208 class_recalls.append(0.0)209 else:210 # Calculate recall for this label211 correct_predictions = 0212 for idx in true_indices:213 pred = preds[idx]214 gt = gts[idx]215 if pred == gt or pred.lower() == gt:216 correct_predictions += 1217 218 recall = correct_predictions / len(true_indices)219 class_recalls.append(recall)220 221 pred_indices = [222 i for i, pred in enumerate(preds)223 if pred.lower() == label or pred == label224 ]225 if len(pred_indices) == 0:226 precision = 0.0227 class_precisions.append(0.0)228 else:229 correct_predictions = 0230 for idx in pred_indices:231 pred = preds[idx]232 gt = gts[idx]233 if pred == gt or pred.lower() == gt:234 correct_predictions += 1235 236 precision = correct_predictions / len(pred_indices)237 class_precisions.append(precision)238 239 f1 = (2 * recall * precision) / (recall + precision + 1e-6)240 class_f1s.append(f1)241 242 output_fpath = Path(243 infer_path).parent / f"results/{Path(infer_path).stem}.json"244 output_fpath.parent.mkdir(parents=True, exist_ok=True)245 with open(output_fpath, "w") as writer:246 json.dump({247 "acc": acc,248 "f1": np.mean(class_f1s),249 },250 writer,251 indent=4)252 writer.write("\n")253 254 def mcq(self, infer_path: str = ""):255 gts, preds = [], []256 with open(infer_path, "r") as f:257 for line in f:258 item = json.loads(line)259 if "id" not in item:260 continue261 gts.append(item["ground_truth"])262 preds.append(item["output"])263 264 # Custom comparison function: consider both exact match and case-insensitive match265 # This is because LLM outputs sometimes capitalize the first letter to follow English grammar266 correct_count = 0267 for gt, pred in zip(gts, preds):268 if pred == gt or pred.lower() == gt:269 correct_count += 1270 271 acc = correct_count / len(gts)272 print(f"Accuracy: {acc}")273 274 # Calculate recall with custom matching logic275 # Using the same case-insensitive matching as accuracy calculation276 try:277 labels = list(set(gts))278 279 # Manually calculate recall for each class280 class_recalls = []281 for label in labels:282 # Find all samples with this true label283 true_indices = [i for i, gt in enumerate(gts) if gt == label]284 if len(true_indices) == 0:285 class_recalls.append(0.0)286 continue287 288 # Calculate recall for this label (using lenient matching)289 correct_predictions = 0290 for idx in true_indices:291 pred = preds[idx]292 gt = gts[idx]293 if pred == gt or pred.lower() == gt:294 correct_predictions += 1295 296 recall = correct_predictions / len(true_indices)297 class_recalls.append(recall)298 299 # Calculate macro average recall300 weighted_recall = np.mean(class_recalls)301 print(f"Weighted Recall: {weighted_recall}")302 303 output_fpath = Path(304 infer_path).parent / f"results/{Path(infer_path).stem}.json"305 output_fpath.parent.mkdir(parents=True, exist_ok=True)306 with open(output_fpath, "w") as writer:307 json.dump({308 "acc": acc,309 "uar": weighted_recall310 },311 writer,312 indent=4)313 writer.write("\n")314 315 except Exception as e:316 print(f"Error calculating Weighted Recall: {e}")317 print(318 "Possible reasons: labels are not numeric or contain non-numeric labels"319 )320 321 def anomaly_detection(self, infer_path: str = ""):322 gts, preds = [], []323 success, fail = 0, 0324 with open(infer_path, "r") as f:325 for line in f:326 item = json.loads(line)327 if "id" not in item:328 continue329 330 if item["output"].lower() == "yes":331 preds.append(True)332 elif item["output"].lower() == "no":333 preds.append(False)334 else:335 fail += 1336 continue337 success += 1338 gts.append(item["ground_truth"])339 340 correct_count = 0341 for gt, pred in zip(gts, preds):342 if pred == gt:343 correct_count += 1344 print(f"Success: {success}, Fail: {fail}")345 acc = correct_count / len(gts)346 print(f"Accuracy: {acc}")347 348 f1 = f1_score(gts, preds)349 print(f"F1 Score: {f1}")350 output_fpath = Path(351 infer_path).parent / f"results/{Path(infer_path).stem}.json"352 output_fpath.parent.mkdir(parents=True, exist_ok=True)353 with open(output_fpath, "w") as writer:354 json.dump({355 "acc": acc,356 "f1": f1,357 "success": success,358 "fail": fail359 },360 writer,361 indent=4)362 writer.write("\n")363 364 def forecasting(self, infer_path: str = ""):365 gt_arrs = []366 pred_arrs = []367 success = 0368 fail = 0369 with h5py.File(infer_path, "r") as f:370 base_path = Path(f["base_path"][()].decode("utf-8"))371 for id in f.keys():372 try:373 if id not in [374 "base_path", "dataset_name", "domain", "task",375 "scene"376 ]:377 gt_path = concat_base_path(378 base_path,379 f[id]["gt_path"][()].decode("utf-8").strip("/"))380 gt_data = read_time_series_data(gt_path)381 gt_data = np.array(gt_data, dtype=np.float32)382 pred = f[id]["pred_result"][()]383 384 if pred.shape != gt_data.shape:385 raise ValueError(386 f"Pred shape {pred.shape} does not match gt shape {gt_data.shape}"387 )388 gt_arrs.append(gt_data.reshape(-1))389 pred_arrs.append(pred.reshape(-1))390 success += 1391 except Exception as e:392 print(f"Error processing {id}: {e}")393 fail += 1394 395 if len(gt_arrs) == 0:396 mae = "N/A"397 rel_mae = "N/A"398 else:399 gt_arrs = np.concatenate(gt_arrs)400 pred_arrs = np.concatenate(pred_arrs)401 402 # mse = mean_squared_error(gt_arrs, pred_arrs)403 mae = mean_absolute_error(gt_arrs, pred_arrs)404 rel_mae = non_zero_rel_mae(gt_arrs, pred_arrs)405 print(406 f"MAE: {mae}, REL_MAE: {rel_mae}, Success: {success}, Fail: {fail}"407 )408 output_fpath = Path(409 infer_path).parent / f"results/{Path(infer_path).stem}.json"410 output_fpath.parent.mkdir(parents=True, exist_ok=True)411 with open(output_fpath, "w") as writer:412 json.dump(413 {414 "rel_mae": rel_mae,415 "mae": mae,416 "success": success,417 "fail": fail,418 "success_rate": success / (success + fail)419 },420 writer,421 indent=4)422 writer.write("\n")423 424 def synthesize(self, infer_path: str = ""):425 return self.forecasting(infer_path)426 427 def imputation(self, infer_path: str = ""):428 gt_arrs = []429 pred_arrs = []430 success = 0431 fail = 0432 with h5py.File(infer_path, "r") as f:433 base_path = Path(f["base_path"][()].decode("utf-8"))434 for id in f.keys():435 try:436 if id not in [437 "base_path", "dataset_name", "domain", "task",438 "scene"439 ]:440 # gt_path = base_path / f[id]["gt_path"][441 # ()].decode("utf-8")442 gt_path = concat_base_path(443 base_path,444 f[id]["gt_path"][()].decode("utf-8").strip("/"))445 gt_data = read_time_series_data(gt_path)446 447 # input_path = base_path / f[id]["input_ts_path"][448 # ()].decode("utf-8")449 input_path = concat_base_path(450 base_path, f[id]["input_ts_path"][(451 )].decode("utf-8").strip("/"))452 input_data = read_time_series_data(input_path)453 454 pred_indices = np.where(input_data == "X")[0]455 pred = f[id]["pred_result"][()]456 457 pred = pred[pred_indices]458 gt_data = gt_data[pred_indices]459 if len(pred) != len(gt_data):460 length_mismatch += 1461 else:462 success += 1463 if len(pred) < len(gt_data):464 pred = pred[:len(gt_data)]465 if len(pred) > len(gt_data):466 gt_data = gt_data[:len(pred)]467 gt_arrs.append(gt_data)468 pred_arrs.append(pred)469 # success += 1470 except Exception as e:471 print(f"Error processing {id}: {e}")472 fail += 1473 474 gt_arrs = np.concatenate(gt_arrs)475 pred_arrs = np.concatenate(pred_arrs)476 # mse = mean_squared_error(gt_arrs, pred_arrs)477 rel_mae = non_zero_rel_mae(gt_arrs, pred_arrs)478 mae = mean_absolute_error(gt_arrs, pred_arrs)479 480 print(481 f"REL_MAE: {rel_mae}, MAE: {mae}, Success: {success}, Fail: {fail}"482 )483 output_fpath = Path(484 infer_path).parent / f"results/{Path(infer_path).stem}.json"485 output_fpath.parent.mkdir(parents=True, exist_ok=True)486 with open(output_fpath, "w") as writer:487 json.dump(488 {489 "rel_mae": rel_mae,490 "mae": mae,491 "success": success,492 "fail": fail,493 "success_rate": success / (success + fail)494 },495 writer,496 indent=4)497 writer.write("\n")498 499 def event_detection(self, infer_path: str = ""):500 event_gts, event_preds = [], []501 seq_length = None502 success = 0503 total = 0504 timestamp_gts, timestamp_preds = [], []505 with open(infer_path, "r") as f:506 for line in f:507 item = json.loads(line)508 if "id" not in item:509 seq_length = item["seq_length"]510 continue511 event_gt = item["ground_truth"]["contain"]512 event_gts.append(1 if event_gt else 0)513 514 if "\n" in item["output"]:515 while '\n\n' in item["output"]:516 item["output"] = item["output"].replace('\n\n', '\n')517 event_pred, *timestamps = item["output"].split("\n")518 else:519 event_pred = item["output"]520 timestamps = None521 event_preds.append(1 if event_pred.lower() == "yes" else 0)522 523 if event_gt:524 if "start_time" in item["ground_truth"]:525 gt_timestamps = [item["ground_truth"]["start_time"]]526 elif "start_time_p" in item["ground_truth"]:527 gt_timestamps = [528 item["ground_truth"]["start_time_p"],529 item["ground_truth"]["start_time_s"]530 ]531 532 if timestamps is None:533 pass534 else:535 try:536 assert len(timestamps) == len(gt_timestamps)537 for pred_timestamp, gt_timestamp in zip(538 timestamps, gt_timestamps):539 pred_timestamp = eval(pred_timestamp)540 timestamp_preds.append(pred_timestamp)541 timestamp_gts.append(gt_timestamp)542 success += 1543 except Exception as e:544 print(str(e))545 total += 1546 547 event_acc = accuracy_score(event_gts, event_preds)548 event_f1 = f1_score(event_gts, event_preds)549 timestamp_gts = np.array(timestamp_gts)550 timestamp_preds = np.array(timestamp_preds)551 mape = non_zero_rel_mae(timestamp_gts, timestamp_preds)552 output_fpath = Path(553 infer_path).parent / f"results/{Path(infer_path).stem}.json"554 output_fpath.parent.mkdir(parents=True, exist_ok=True)555 with open(output_fpath, "w") as writer:556 json.dump(557 {558 "acc": event_acc,559 "f1": event_f1,560 "mape": mape,561 "success_rate": success / total562 },563 writer,564 indent=4)565 writer.write("\n")566 print({567 "acc": event_acc,568 "f1": event_f1,569 "mape": mape,570 "success_rate": success / total571 })572 573 def evaluate(self, infer_dir: str):574 for infer_path in Path(infer_dir).glob("*"):575 if infer_path.is_dir():576 continue577 dataset_id = infer_path.stem578 task = DATASET_TO_TASK[dataset_id]579 print(f"evaluating {dataset_id} ...")580 getattr(self, task)(infer_path)581 582 583if __name__ == "__main__":584 fire.Fire(Runner)585