ZZoutian/figure2data-database-v2
figure2data databasev2 — 40K v6 合成科研图表数据集（最终交付版） 生成日期：2026-09-17 生成器：generator 1.6.0 / dataset_generation_revision v6（含两次 hotfix） 规模：40,000 样本（10 图族 / 33 亚型；area、matrix 冻结不生成，set_relation 已删除） 目录结构 databasev2/ ├── figure2data.sqlite3 # 主数据库（2.0 GB：40,000 samples / 280,000 documents） ├── schema/ # sqlite schema ├── shards/ # 数据资产（shard = (样本序号-1)//1000） │ └── shard_000 .. shard_039/ │ ├── images/ # PNG… See the full description on the dataset page: https://huggingface.co/datasets/ZZoutian/figure2data-database-v2.
06.3k
1"""§52.8 Duplicate Test — 注入相同 image → exact duplicate gate 捕获。"""2 3from __future__ import annotations4 5import json6 7from databasev1.build import cmd_generate, cmd_plan, load_config8from databasev1.storage import paths as P9from databasev1.storage.staging import read_completed10from databasev1.validation.duplicate_gate import check11 12 13def _seed_staging(tmp_staging, n=6):14 cfg = dict(load_config())15 cfg["target_n"] = 4016 cmd_plan(cfg)17 stats = cmd_generate(cfg, shard_ids=[0], limit=n, workers=2)18 assert stats["generated"] == n19 return cfg20 21 22def test_no_duplicates_in_fresh_run(tmp_staging):23 _seed_staging(tmp_staging, 6)24 report = check([0])25 assert report["errors"] == []26 assert report["duplicate_sample_ids"] == {}27 assert report["duplicate_image_sha256"] == {}28 assert report["n_records"] == 629 30 31def test_injected_duplicate_image_is_caught(tmp_staging):32 _seed_staging(tmp_staging, 6)33 recs = read_completed(0)34 # 注入：把 recs[1] 的 image 复制为 recs[3] 的 image（相同 SHA 记录）35 img1 = P.image_path(0, recs[1]["sample_id"])36 img3 = P.image_path(0, recs[3]["sample_id"])37 img3.write_bytes(img1.read_bytes())38 # 同步 completed 记录的 sha39 lines = []40 for line in P.shard_completed_path(0).read_text().splitlines():41 rec = json.loads(line)42 if rec["sample_id"] == recs[3]["sample_id"]:43 rec["image_sha256"] = recs[1]["image_sha256"]44 lines.append(json.dumps(rec, ensure_ascii=False, sort_keys=True))45 P.shard_completed_path(0).write_text("\n".join(lines) + "\n")46 47 report = check([0])48 assert report["duplicate_image_sha256"], "duplicate image SHA must be detected"49 assert recs[1]["image_sha256"] in report["duplicate_image_sha256"]50 assert report["errors"], "gate must flag duplicates as errors"51 52 53def test_injected_duplicate_sample_id_is_caught(tmp_staging):54 _seed_staging(tmp_staging, 5)55 recs = read_completed(0)56 # 注入一条重复 sample_id 记录57 with open(P.shard_completed_path(0), "a", encoding="utf-8") as f:58 dup = dict(recs[0])59 f.write(json.dumps(dup, ensure_ascii=False, sort_keys=True) + "\n")60 report = check([0])61 assert report["duplicate_sample_ids"], "duplicate sample_id must be detected"62 assert recs[0]["sample_id"] in report["duplicate_sample_ids"]63