Datasetpublic

Ian100/ProbeScout-features

ProbeScout features Frozen SigLIP features for Stanford Cars, HICO-DET and CelebA: float32 global embeddings [N, 768], float16 patch tokens [N, 196, 768], and records that map each row to an image ID. Total size: 81.10 GB. Use the ProbeScout setup guide and dataset-specific download commands. Download only the dataset you need. Place its dataset/ contents under the code repository's dataset/ directory, preserving paths and row order. These features skip extraction for training… See the full description on the dataset page: https://huggingface.co/datasets/Ian100/ProbeScout-features.

sourceHugging Faceupdated 24d agoView on Hugging Face

0likes356downloads

rootrestore_patch_features.py

restore_patch_features.py49 linesDownload Raw Back to root

1"""Reassemble uploaded byte shards into the exact original NPY file."""
2import argparse,hashlib,json,shutil
3from pathlib import Path
4
5def sha(path):
6    h=hashlib.sha256()
7    with path.open('rb') as stream:
8        while data:=stream.read(8*1024**2): h.update(data)
9    return h.hexdigest()
10
11def main():
12    parser=argparse.ArgumentParser(description=__doc__)
13    parser.add_argument('--root',type=Path,default=Path(__file__).resolve().parent)
14    args=parser.parse_args();root=args.root.resolve()
15    manifest=json.loads((root/'asset_manifest.json').read_text(encoding='utf-8'))
16    for item in manifest.get('reassemble',[]):
17        target=(root/item['path']).resolve()
18        if not target.is_relative_to(root):raise ValueError('Path escapes asset root')
19        target.parent.mkdir(parents=True,exist_ok=True)
20        if target.exists():
21            if target.stat().st_size!=item['bytes'] or sha(target)!=item['sha256']:
22                raise ValueError('Existing output differs; refusing overwrite')
23            print('Already verified:',item['path']);continue
24        if shutil.disk_usage(target.parent).free<item['bytes']:
25            raise RuntimeError('Insufficient space for the restored NPY; shards are retained')
26        temp=target.with_name(target.name+'.restoring')
27        digest=hashlib.sha256();written=0
28        try:
29            with temp.open('xb') as output:
30                for part in item['parts']:
31                    if part['byte_offset']!=written:raise ValueError('Non-contiguous shards')
32                    source=(root/part['path']).resolve()
33                    if not source.is_relative_to(root) or source.stat().st_size!=part['bytes']:
34                        raise ValueError('Missing or invalid shard')
35                    part_hash=hashlib.sha256()
36                    with source.open('rb') as stream:
37                        while data:=stream.read(8*1024**2):
38                            output.write(data);part_hash.update(data);digest.update(data);written+=len(data)
39                    if part_hash.hexdigest()!=part['sha256']:raise ValueError('Shard checksum mismatch')
40            if written!=item['bytes'] or digest.hexdigest()!=item['sha256']:
41                raise ValueError('Restored NPY checksum mismatch')
42            temp.rename(target)
43        except Exception:
44            # Retain partial output for inspection; never remove existing user files.
45            raise
46        print('Restored and verified:',item['path'])
47
48if __name__=='__main__':main()
49