Decoder2704/python-chatbot-jay

1#!/usr/bin/env python32"""3Build final_chatbot_data.csv from Questions.csv and AnswersV2.csv.4 5Parity with Chatbot-V5.ipynb: same reads, renames, filter, merge, drops, renames,6and BeautifulSoup HTML stripping (no TF-IDF / UI).7 8Default inputs: Questions.csv from repo pybot_data/, AnswersV2.csv from backend/data/.9Default output: backend/data/final_chatbot_data.csv10 11Paths resolve from this script file location (not the process working directory).12"""13 14from __future__ import annotations15 16import argparse17import logging18import sys19from pathlib import Path20 21import pandas as pd22from bs4 import BeautifulSoup23 24_BACKEND_DIR = Path(__file__).resolve().parent.parent25if str(_BACKEND_DIR) not in sys.path:26    sys.path.insert(0, str(_BACKEND_DIR))27 28from app.paths import DATA_DIR, PYBOT_DATA_DIR, ensure_data_and_artifacts_dirs29 30DEFAULT_QUESTIONS_PATH = PYBOT_DATA_DIR / "Questions.csv"31DEFAULT_ANSWERS_V2_PATH = DATA_DIR / "AnswersV2.csv"32DEFAULT_OUTPUT_PATH = DATA_DIR / "final_chatbot_data.csv"33 34CSV_ENCODING = "latin-1"35 36DROP_AFTER_MERGE = [37    "OwnerUserId_x",38    "CreationDate_x",39    "Score_x",40    "OwnerUserId_y",41    "CreationDate_y",42]43 44logging.basicConfig(45    level=logging.INFO,46    format="%(levelname)s: %(message)s",47)48logger = logging.getLogger(__name__)49 50 51def read_questions(path: Path) -> pd.DataFrame:52    logger.info("Loading Questions from %s", path)53    df = pd.read_csv(path, encoding=CSV_ENCODING)54    logger.info("Questions: %s rows, %s columns", f"{len(df):,}", len(df.columns))55    return df56 57 58def read_answers_v2(path: Path) -> pd.DataFrame:59    logger.info("Loading AnswersV2 from %s", path)60    df = pd.read_csv(path, encoding=CSV_ENCODING)61    logger.info("AnswersV2: %s rows, %s columns", f"{len(df):,}", len(df.columns))62    return df63 64 65def rename_answer_keys(an: pd.DataFrame) -> pd.DataFrame:66    """Notebook cell 4: ParentId -> QId."""67    out = an.rename(columns={"ParentId": "QId"})68    return out69 70 71def rename_ids(q: pd.DataFrame, an: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame]:72    """Notebook cell 5: Questions Id -> QId; Answers Id -> AId."""73    q = q.rename(columns={"Id": "QId"})74    an = an.rename(columns={"Id": "AId"})75    return q, an76 77 78def filter_answers_score_gt_five(an: pd.DataFrame) -> pd.DataFrame:79    """Notebook cell 6: an = an[an['Score']>5]."""80    before = len(an)81    out = an[an["Score"] > 5]82    logger.info("Filtered answers with Score > 5: %s -> %s rows", f"{before:,}", f"{len(out):,}")83    return out84 85 86def merge_on_qid(q: pd.DataFrame, an: pd.DataFrame) -> pd.DataFrame:87    """Notebook cell 8: df = q.merge(an, on='QId')."""88    df = q.merge(an, on="QId")89    logger.info("Merged on QId: %s rows", f"{len(df):,}")90    return df91 92 93def drop_columns(df: pd.DataFrame) -> pd.DataFrame:94    """Notebook cell 9."""95    missing = [c for c in DROP_AFTER_MERGE if c not in df.columns]96    if missing:97        raise ValueError(f"Expected merge columns missing before drop: {missing}")98    return df.drop(columns=DROP_AFTER_MERGE)99 100 101def rename_body_columns(df: pd.DataFrame) -> pd.DataFrame:102    """Notebook cell 10."""103    return df.rename(104        columns={"Body_x": "Question", "Body_y": "Answer", "Score_y": "Score"}105    )106 107 108def strip_html_question_answer(df: pd.DataFrame) -> pd.DataFrame:109    """Notebook cell 12 (same order: Answer, then Question)."""110    out = df.copy()111    out["Answer"] = out["Answer"].apply(lambda x: BeautifulSoup(x).get_text())112    out["Question"] = out["Question"].apply(lambda x: BeautifulSoup(x).get_text())113    return out114 115 116def save_final(df: pd.DataFrame, path: Path) -> None:117    path.parent.mkdir(parents=True, exist_ok=True)118    df.to_csv(path, index=False)119    logger.info("Wrote %s rows to %s", f"{len(df):,}", path)120 121 122def build_final_dataset(q: pd.DataFrame, an: pd.DataFrame) -> pd.DataFrame:123    """Full notebook data pipeline (cells 4–12)."""124    an = rename_answer_keys(an)125    q, an = rename_ids(q, an)126    an = filter_answers_score_gt_five(an)127    df = merge_on_qid(q, an)128    df = drop_columns(df)129    df = rename_body_columns(df)130    df = strip_html_question_answer(df)131    return df132 133 134def parse_args() -> argparse.Namespace:135    p = argparse.ArgumentParser(136        description="Build final_chatbot_data.csv (Chatbot-V5.ipynb data flow).",137    )138    p.add_argument(139        "--questions",140        type=Path,141        default=DEFAULT_QUESTIONS_PATH,142        help=f"Path to Questions.csv (default: {DEFAULT_QUESTIONS_PATH})",143    )144    p.add_argument(145        "--answers-v2",146        type=Path,147        default=DEFAULT_ANSWERS_V2_PATH,148        help=f"Path to AnswersV2.csv (default: {DEFAULT_ANSWERS_V2_PATH})",149    )150    p.add_argument(151        "--output",152        type=Path,153        default=DEFAULT_OUTPUT_PATH,154        help=f"Output CSV path (default: {DEFAULT_OUTPUT_PATH})",155    )156    return p.parse_args()157 158 159def main() -> int:160    ensure_data_and_artifacts_dirs()161    args = parse_args()162    questions_path = args.questions.resolve()163    answers_path = args.answers_v2.resolve()164    output_path = args.output.resolve()165 166    if not questions_path.is_file():167        print(f"ERROR: Questions.csv not found: {questions_path}", file=sys.stderr)168        return 1169    if not answers_path.is_file():170        print(f"ERROR: AnswersV2.csv not found: {answers_path}", file=sys.stderr)171        return 1172 173    try:174        q = read_questions(questions_path)175        an = read_answers_v2(answers_path)176        df = build_final_dataset(q, an)177        save_final(df, output_path)178    except UnicodeDecodeError as exc:179        print(f"ERROR: CSV decode failed (expected {CSV_ENCODING!r}): {exc}", file=sys.stderr)180        return 1181    except OSError as exc:182        print(f"ERROR: file read/write failed: {exc}", file=sys.stderr)183        return 1184    except ValueError as exc:185        print(f"ERROR: {exc}", file=sys.stderr)186        return 1187    except Exception as exc:188        logger.exception("Build failed: %s", exc)189        return 1190 191    print(f"input (Questions):  {questions_path}")192    print(f"input (AnswersV2):  {answers_path}")193    print(f"output:              {output_path}")194    print(f"rows:                {len(df):,}")195    print(f"columns:             {list(df.columns)}")196    return 0197 198 199if __name__ == "__main__":200    raise SystemExit(main())201