HumanBehaviorAtlas/human_behavior_atlas

from datasets import load_dataset

# Stream without downloading everything
ds = load_dataset("HumanBehaviorAtlas/human_behavior_atlas", split="train", streaming=True)
sample = next(iter(ds))

# Load a subset
ds_100 = load_dataset("HumanBehaviorAtlas/human_behavior_atlas", split="train[:100]")

import io
import soundfile as sf

sample = ds_100[0]

# Audio is raw bytes — decode with soundfile or torchaudio
if sample["audios"]:
    audio_data, sr = sf.read(io.BytesIO(sample["audios"][0]))

# Video is raw bytes — decode with decord, opencv, or write to temp file
if sample["videos"]:
    video_bytes = sample["videos"][0]

{
  "problem": "<audio>
Don't forget a jacket.
The above is a speech recording along with the transcript from a clinical context. What emotion is the speaker expressing? Answer with one word from the following: anger, disgust, fear, happy, neutral, sad",
  "answer": "sad",
  "images": [],
  "videos": [],
  "audios": ["..."],
  "dataset": "cremad",
  "modality_signature": "text_audio",
  "task": "emotion_cls",
  "class_label": "sad"
}

@inproceedings{
ong2026human,
title={Human Behavior Atlas: Benchmarking Unified Psychological And Social Behavior Understanding},
author={Keane Ong and Wei Dai and Carol Li and Dewei Feng and Hengzhi Li and Jingyao Wu and Jiaee Cheong and Rui Mao and Gianmarco Mengaldo and Erik Cambria and Paul Pu Liang},
booktitle={The Fourteenth International Conference on Learning Representations},
year={2026},
url={https://openreview.net/forum?id=ZKE23BBvlQ}
}

@article{ong2026omnisapiens,
  title={Omnisapiens: A foundation model for social behavior processing via heterogeneity-aware relative policy optimization},
  author={Ong, Keane and Boughorbel, Sabri and Xiao, Luwei and Ekbote, Chanakya and Dai, Wei and Qu, Ao and Wu, Jingyao and Mao, Rui and Hoque, Ehsan and Cambria, Erik and others},
  journal={arXiv preprint arXiv:2602.10635},
  year={2026}
}