hussain2010/Geospatial_Cleaning_Preprocessing_Wrangling_FeatureEngineering

1import streamlit as st2import geopandas as gpd3import pandas as pd4import pyarrow.parquet as pq5from huggingface_hub import hf_hub_download6import warnings7 8# Suppress specific RuntimeWarnings (USECOLS)9warnings.filterwarnings("ignore", category=RuntimeWarning, message=".*USECOLS.*")10 11# Set Streamlit Page Configuration12st.set_page_config(page_title="Optimized GADM Data Processing", layout="wide")13 14# Hugging Face Credentials15hf_api_key = st.secrets["World_Map_Dataset_Cleaning"]16repo_id = "hussain2010/World_Map_Files"17filename = "gadm_410.gpkg"18 19# Streamlit App Title20st.title("🌍 Optimized GADM Geospatial Data Processing")21 22# Step 1: Download File from Hugging Face23st.subheader("📥 Downloading Dataset from Hugging Face")24try:25    dataset_path = hf_hub_download(26        repo_id=repo_id, filename=filename, token=hf_api_key, repo_type="dataset"27    )28    st.success("✅ Dataset downloaded successfully!")29except Exception as e:30    st.error(f"⚠️ Error downloading dataset: {e}")31    st.stop()32 33# Step 2: Efficiently Load the Dataset34st.subheader("📍 Loading Dataset Efficiently")35try:36    # Load only metadata (no geometry) for filtering37    metadata_df = gpd.read_file(dataset_path, layer=0, usecols=["GID_0", "NAME_0"]).drop_duplicates()38 39    # Let user select a country first40    selected_country = st.selectbox("🌎 Select a Country", metadata_df["NAME_0"].unique())41 42    # Load only selected country data (efficient filtering)43    gdf = gpd.read_file(dataset_path, layer=0, where=f"NAME_0 = '{selected_country}'")44 45    # Drop invalid geometries46    gdf = gdf[gdf.is_valid]47 48    # Convert CRS to EPSG:4326 if needed49    if gdf.crs and gdf.crs.to_string() != "EPSG:4326":50        gdf = gdf.to_crs("EPSG:4326")51 52    # Convert geometry to WKT to avoid PyArrow serialization errors53    gdf["geometry_wkt"] = gdf["geometry"].apply(lambda geom: geom.wkt if geom else None)54 55    # Drop original geometry column to reduce memory usage56    gdf.drop(columns=["geometry"], inplace=True)57 58    # Save to memory-efficient Parquet format59    optimized_file = "optimized_gadm.parquet"60    gdf.to_parquet(optimized_file, engine="pyarrow")61 62    # Show optimized dataset63    st.write("✔ Memory-efficient dataset preview:")64    st.dataframe(gdf.head(100))  # Show only first 100 rows65 66    # Provide Download Option67    with open(optimized_file, "rb") as file:68        st.download_button("📥 Download Optimized Geospatial Data", data=file, file_name="optimized_gadm.parquet", mime="application/octet-stream")69 70except Exception as e:71    st.error(f"⚠️ Error processing file: {e}")72