Yourify/structured-data-formatter

1import os2import logging3from datasets import Dataset, load_dataset4from datetime import datetime5 6logger = logging.getLogger(__name__)7DATASET_REPO = "yourify/project-knowledge"8HF_TOKEN = os.getenv("HF_TOKEN")9 10def load_persistent_context(limit: int = 20):11    """Load latest context from HF Dataset"""12    try:13        ds = load_dataset(DATASET_REPO, split="train")14        data = ds.to_list()[-limit:]15        return [item["text"] for item in data if "text" in item]16    except Exception as e:17        logger.warning(f"Load persistence failed: {e}")18        return []19 20def save_to_persistent(text: str, metadata: dict = None):21    """Append new entry to persistent dataset"""22    if not text or not text.strip():23        return False24    try:25        existing = load_persistent_context(limit=100)26        new_entry = {27            "text": text.strip(),28            "timestamp": datetime.now().isoformat(),29            "metadata": metadata or {}30        }31        all_data = [{"text": t} for t in existing] + [new_entry]32        new_ds = Dataset.from_list(all_data)33        new_ds.push_to_hub(DATASET_REPO, private=True, token=HF_TOKEN)34        logger.info(f"✅ Saved to persistent dataset. Total entries: {len(all_data)}")35        return True36    except Exception as e:37        logger.error(f"Save to dataset failed: {e}")38        return False