evalstate/diffusers-pr-api
0
1from __future__ import annotations2 3import json4import urllib.error5import urllib.request6from datetime import UTC, datetime, timedelta7from typing import Any8 9from slop_farmer.config import resolve_github_token10from slop_farmer.data.http import urlopen_with_retry11 12GRAPHQL_URL = "https://api.github.com/graphql"13 14PROFILE_QUERY = """15query UserActivityProfile($login: String!, $from: DateTime!, $to: DateTime!) {16 rateLimit { cost remaining resetAt }17 viewer {18 login19 organizations(first: 100) {20 totalCount21 nodes {22 login23 name24 }25 }26 }27 user(login: $login) {28 login29 name30 createdAt31 organizations(first: 100) {32 totalCount33 nodes {34 login35 name36 }37 }38 starredRepositories(first: 100, orderBy: {field: STARRED_AT, direction: DESC}) {39 totalCount40 nodes {41 nameWithOwner42 stargazerCount43 owner {44 login45 }46 }47 }48 contributionsCollection(from: $from, to: $to) {49 contributionCalendar {50 totalContributions51 weeks {52 contributionDays {53 date54 contributionCount55 }56 }57 }58 totalIssueContributions59 totalPullRequestContributions60 pullRequestContributionsByRepository(maxRepositories: 10) {61 repository { nameWithOwner }62 contributions { totalCount }63 }64 issueContributionsByRepository(maxRepositories: 10) {65 repository { nameWithOwner }66 contributions { totalCount }67 }68 }69 }70}71""".strip()72 73SEARCH_PRS_QUERY = """74query SearchPullRequests($query: String!, $cursor: String) {75 rateLimit { cost remaining resetAt }76 search(type: ISSUE, query: $query, first: 100, after: $cursor) {77 issueCount78 pageInfo { hasNextPage endCursor }79 nodes {80 ... on PullRequest {81 number82 state83 merged84 createdAt85 updatedAt86 repository { nameWithOwner stargazerCount }87 }88 }89 }90}91""".strip()92 93SEARCH_ISSUES_QUERY = """94query SearchIssues($query: String!, $cursor: String) {95 rateLimit { cost remaining resetAt }96 search(type: ISSUE, query: $query, first: 100, after: $cursor) {97 issueCount98 pageInfo { hasNextPage endCursor }99 nodes {100 ... on Issue {101 number102 state103 createdAt104 updatedAt105 repository { nameWithOwner }106 }107 }108 }109}110""".strip()111 112 113def summarize_user(login: str, days: int, repo: str | None) -> dict[str, Any]:114 now = datetime.now(tz=UTC)115 start = (now - timedelta(days=days)).replace(microsecond=0)116 from_iso = start.isoformat().replace("+00:00", "Z")117 from_date = start.date().isoformat()118 to_iso = now.replace(microsecond=0).isoformat().replace("+00:00", "Z")119 120 _user_activity_log(f"user enrichment {login}: fetching profile")121 profile, _errors = _post_graphql(122 PROFILE_QUERY, {"login": login, "from": from_iso, "to": to_iso}123 )124 viewer = profile["viewer"]125 user = profile["user"]126 if user is None:127 raise RuntimeError(f"unknown user {login!r}")128 129 contributions = user["contributionsCollection"]130 calendar = contributions["contributionCalendar"]131 repo_term = f" repo:{repo}" if repo else ""132 pr_query = f"author:{login} is:pr created:>={from_date}{repo_term} sort:created-desc"133 open_pr_query = (134 f"author:{login} is:pr is:open created:>={from_date}{repo_term} sort:created-desc"135 )136 issue_query = f"author:{login} is:issue created:>={from_date}{repo_term} sort:created-desc"137 138 pr_count, prs, inaccessible_pr_nodes = _search_all(pr_query, prs=True, label=f"{login} prs")139 open_pr_count, open_prs, inaccessible_open_pr_nodes = _search_all(140 open_pr_query, prs=True, label=f"{login} open-prs"141 )142 issue_count, issues, inaccessible_issue_nodes = _search_all(143 issue_query, prs=False, label=f"{login} issues"144 )145 146 merged_prs = [pr for pr in prs if pr.get("merged")]147 closed_unmerged_prs = [pr for pr in prs if pr.get("state") == "CLOSED" and not pr.get("merged")]148 still_open_prs = [pr for pr in prs if pr.get("state") == "OPEN"]149 open_pr_repos = sorted(150 {pr["repository"]["nameWithOwner"] for pr in open_prs if pr.get("repository")}151 )152 pr_repos = sorted({pr["repository"]["nameWithOwner"] for pr in prs if pr.get("repository")})153 issue_repos = sorted(154 {issue["repository"]["nameWithOwner"] for issue in issues if issue.get("repository")}155 )156 starred = user.get("starredRepositories") or {}157 starred_nodes = [row for row in starred.get("nodes") or [] if isinstance(row, dict)]158 non_self_starred = [159 row160 for row in starred_nodes161 if ((row.get("owner") or {}).get("login") or "").casefold() != login.casefold()162 ]163 164 recent_pr_repo_stars = [165 int((pr.get("repository") or {}).get("stargazerCount") or 0) for pr in prs166 ]167 merged_pr_repo_stars = [168 int((pr.get("repository") or {}).get("stargazerCount") or 0) for pr in merged_prs169 ]170 closed_unmerged_pr_repo_stars = [171 int((pr.get("repository") or {}).get("stargazerCount") or 0) for pr in closed_unmerged_prs172 ]173 open_pr_repo_stars = [174 int((pr.get("repository") or {}).get("stargazerCount") or 0) for pr in open_prs175 ]176 repo_owners = sorted(177 {178 repo_name.split("/", 1)[0]179 for repo_name in pr_repos + issue_repos + open_pr_repos180 if "/" in repo_name and repo_name.split("/", 1)[0].casefold() != login.casefold()181 }182 )183 184 public_org_rows = _public_orgs(login)185 target_listed_org_rows = [186 {"login": row["login"], "name": row.get("name")}187 for row in (user.get("organizations") or {}).get("nodes") or []188 if isinstance(row, dict) and row.get("login")189 ]190 viewer_listed_org_rows = [191 {"login": row["login"], "name": row.get("name")}192 for row in (viewer.get("organizations") or {}).get("nodes") or []193 if isinstance(row, dict) and row.get("login")194 ]195 viewer_is_target = viewer.get("login", "").casefold() == login.casefold()196 auth_membership_rows = _authenticated_memberships() if viewer_is_target else []197 direct_membership_checks = _check_viewer_org_membership(repo_owners) if viewer_is_target else []198 199 public_org_logins = {row["login"] for row in public_org_rows}200 listed_org_logins = {row["login"] for row in target_listed_org_rows}201 auth_membership_logins = {row["login"] for row in auth_membership_rows}202 directly_confirmed_orgs = sorted(203 row["login"] for row in direct_membership_checks if row.get("viewer_is_a_member")204 )205 listing_endpoints_partial = any(206 org not in public_org_logins | listed_org_logins | auth_membership_logins207 for org in directly_confirmed_orgs208 )209 210 created_at = datetime.fromisoformat(user["createdAt"].replace("Z", "+00:00"))211 account_age_days = (now - created_at).days212 213 return {214 "login": user["login"],215 "name": user.get("name"),216 "repo_scope": repo,217 "window": {"days": days, "from": from_iso, "to": to_iso},218 "account": {"created_at": user["createdAt"], "age_days": account_age_days},219 "activity": {220 "visible_contributions_total": calendar["totalContributions"],221 **_contribution_calendar_summary(calendar["weeks"]),222 "authored_issues": contributions["totalIssueContributions"],223 "authored_pull_requests": contributions["totalPullRequestContributions"],224 "visible_authored_issue_count": len(issues),225 "visible_authored_pr_count": len(prs),226 "visible_open_pr_count": len(open_prs),227 "search_authored_issue_count": issue_count,228 "search_authored_pr_count": pr_count,229 "search_open_pr_count": open_pr_count,230 "inaccessible_issue_nodes": inaccessible_issue_nodes,231 "inaccessible_pr_nodes": inaccessible_pr_nodes,232 "inaccessible_open_pr_nodes": inaccessible_open_pr_nodes,233 "distinct_repos_with_authored_prs": len(pr_repos),234 "distinct_repos_with_authored_issues": len(issue_repos),235 "distinct_repos_with_open_prs": len(open_pr_repos),236 "open_pr_count": len(open_prs),237 "merged_pr_count": len(merged_prs),238 "closed_unmerged_pr_count": len(closed_unmerged_prs),239 "still_open_pr_count": len(still_open_prs),240 "merged_pr_rate": _rate(len(merged_prs), len(prs)),241 "closed_unmerged_pr_rate": _rate(len(closed_unmerged_prs), len(prs)),242 "still_open_pr_rate": _rate(len(still_open_prs), len(prs)),243 },244 "stars": {245 "starred_repositories_total": starred.get("totalCount"),246 "visible_starred_repositories_returned": len(starred_nodes),247 "visible_non_self_starred_repositories": len(non_self_starred),248 "visible_non_self_starred_examples": [249 row["nameWithOwner"] for row in non_self_starred[:10]250 ],251 "recent_pr_repo_star_distribution": _star_distribution(recent_pr_repo_stars),252 "recent_merged_pr_repo_star_distribution": _star_distribution(merged_pr_repo_stars),253 "recent_closed_unmerged_pr_repo_star_distribution": _star_distribution(254 closed_unmerged_pr_repo_stars255 ),256 "recent_open_pr_repo_star_distribution": _star_distribution(open_pr_repo_stars),257 },258 "top_repositories": {259 "pull_requests": [260 {261 "repo": row["repository"]["nameWithOwner"],262 "count": row["contributions"]["totalCount"],263 }264 for row in contributions["pullRequestContributionsByRepository"]265 ],266 "issues": [267 {268 "repo": row["repository"]["nameWithOwner"],269 "count": row["contributions"]["totalCount"],270 }271 for row in contributions["issueContributionsByRepository"]272 ],273 "open_pr_repositories": open_pr_repos,274 },275 "organization_membership": {276 "target_is_viewer": viewer_is_target,277 "public_orgs": public_org_rows,278 "graphql_target_listed_orgs": target_listed_org_rows,279 "graphql_viewer_listed_orgs": viewer_listed_org_rows,280 "authenticated_memberships": auth_membership_rows,281 "checked_recent_repo_owner_orgs": direct_membership_checks,282 "visibility": {283 "listing_endpoints_partial": listing_endpoints_partial,284 "public_org_count": len(public_org_rows),285 "graphql_target_listed_org_count": len(target_listed_org_rows),286 "authenticated_membership_count": len(auth_membership_rows),287 "directly_confirmed_membership_count": len(directly_confirmed_orgs),288 "directly_confirmed_memberships": directly_confirmed_orgs,289 },290 },291 }292 293 294def _post_graphql(295 query: str,296 variables: dict[str, Any],297 *,298 allow_partial: bool = False,299) -> tuple[dict[str, Any], list[dict[str, Any]]]:300 token = resolve_github_token()301 if not token:302 raise RuntimeError("missing GITHUB_TOKEN/GRAPHQL_TOKEN/GH_TOKEN")303 body = json.dumps({"query": query, "variables": variables}).encode()304 request = urllib.request.Request(305 GRAPHQL_URL,306 data=body,307 headers={308 "Authorization": f"bearer {token}",309 "User-Agent": "slop-farmer",310 "Content-Type": "application/json",311 "Accept": "application/json",312 },313 method="POST",314 )315 try:316 with urlopen_with_retry(317 request,318 timeout=120,319 log=_user_activity_log,320 label="GitHub GraphQL user activity",321 ) as response:322 payload = json.load(response)323 except urllib.error.HTTPError as exc: # pragma: no cover - live network only324 detail = exc.read().decode("utf-8", "replace")325 raise RuntimeError(f"graphql request failed: {exc.code} {detail}") from exc326 errors = payload.get("errors") or []327 if errors and not allow_partial:328 raise RuntimeError(json.dumps(errors))329 return payload["data"], errors330 331 332def _search_all(333 query: str, *, prs: bool, label: str | None = None334) -> tuple[int, list[dict[str, Any]], int]:335 nodes: list[dict[str, Any]] = []336 cursor: str | None = None337 issue_count = 0338 inaccessible = 0339 search_query = SEARCH_PRS_QUERY if prs else SEARCH_ISSUES_QUERY340 page = 0341 while True:342 page += 1343 data, errors = _post_graphql(344 search_query, {"query": query, "cursor": cursor}, allow_partial=True345 )346 search = data["search"]347 issue_count = search["issueCount"]348 inaccessible += len(errors)349 nodes.extend(node for node in search["nodes"] if isinstance(node, dict))350 if label and (351 page == 1352 or page % 5 == 0353 or not search["pageInfo"]["hasNextPage"]354 or len(nodes) >= 1000355 ):356 _user_activity_log(357 f"user enrichment {label}: page {page}, loaded {len(nodes)} visible rows, "358 f"search_count={issue_count}, inaccessible={inaccessible}"359 )360 if not search["pageInfo"]["hasNextPage"] or len(nodes) >= 1000:361 break362 cursor = search["pageInfo"]["endCursor"]363 return issue_count, nodes, inaccessible364 365 366def _public_orgs(login: str) -> list[dict[str, Any]]:367 payload = _get_json(f"https://api.github.com/users/{login}/orgs")368 if not isinstance(payload, list):369 return []370 return [371 {"login": row.get("login"), "name": row.get("name")}372 for row in payload373 if isinstance(row, dict) and row.get("login")374 ]375 376 377def _authenticated_memberships() -> list[dict[str, Any]]:378 payload = _get_json("https://api.github.com/user/memberships/orgs")379 if not isinstance(payload, list):380 return []381 rows = []382 for row in payload:383 if not isinstance(row, dict):384 continue385 org = row.get("organization") or {}386 if not org.get("login"):387 continue388 rows.append(389 {390 "login": org.get("login"),391 "name": org.get("name"),392 "state": row.get("state"),393 "role": row.get("role"),394 }395 )396 return rows397 398 399def _check_viewer_org_membership(logins: list[str]) -> list[dict[str, Any]]:400 if not logins:401 return []402 aliases = {f"org{i}": login for i, login in enumerate(logins)}403 fields = "\n".join(404 f'{alias}: organization(login: "{login}") {{ login name viewerIsAMember viewerCanAdminister }}'405 for alias, login in aliases.items()406 )407 query = f"query ViewerOrgChecks {{ rateLimit {{ cost remaining resetAt }}\n{fields}\n}}"408 data, _errors = _post_graphql(query, {})409 rows = []410 for alias in aliases:411 org = data.get(alias)412 if isinstance(org, dict) and org.get("login"):413 rows.append(414 {415 "login": org["login"],416 "name": org.get("name"),417 "viewer_is_a_member": bool(org.get("viewerIsAMember")),418 "viewer_can_administer": bool(org.get("viewerCanAdminister")),419 }420 )421 return rows422 423 424def _get_json(url: str) -> Any:425 token = resolve_github_token()426 if not token:427 raise RuntimeError("missing GITHUB_TOKEN/GRAPHQL_TOKEN/GH_TOKEN")428 request = urllib.request.Request(429 url,430 headers={431 "Authorization": f"bearer {token}",432 "User-Agent": "slop-farmer",433 "Accept": "application/vnd.github+json",434 "X-GitHub-Api-Version": "2022-11-28",435 },436 )437 try:438 with urlopen_with_retry(439 request,440 timeout=120,441 log=_user_activity_log,442 label=url,443 ) as response:444 return json.load(response)445 except urllib.error.HTTPError as exc: # pragma: no cover - live network only446 detail = exc.read().decode("utf-8", "replace")447 raise RuntimeError(f"rest request failed: {exc.code} {url} {detail}") from exc448 449 450def _contribution_calendar_summary(weeks: list[dict[str, Any]]) -> dict[str, Any]:451 daily = [day for week in weeks for day in week["contributionDays"]]452 active_days = sum(1 for day in daily if day["contributionCount"] > 0)453 weekly = [sum(day["contributionCount"] for day in week["contributionDays"]) for week in weeks]454 return {"active_days": active_days, "weekly_totals": weekly}455 456 457def _star_distribution(values: list[int]) -> dict[str, Any]:458 if not values:459 return {460 "count": 0,461 "min": None,462 "median": None,463 "max": None,464 "buckets": {"lt_100": 0, "100_to_999": 0, "1k_to_9k": 0, "10k_plus": 0},465 }466 ordered = sorted(values)467 mid = len(ordered) // 2468 median = ordered[mid] if len(ordered) % 2 else round((ordered[mid - 1] + ordered[mid]) / 2, 1)469 return {470 "count": len(values),471 "min": min(values),472 "median": median,473 "max": max(values),474 "buckets": {475 "lt_100": sum(1 for value in values if value < 100),476 "100_to_999": sum(1 for value in values if 100 <= value < 1000),477 "1k_to_9k": sum(1 for value in values if 1000 <= value < 10000),478 "10k_plus": sum(1 for value in values if value >= 10000),479 },480 }481 482 483def _rate(numerator: int, denominator: int) -> float | None:484 if denominator == 0:485 return None486 return round(numerator / denominator, 4)487 488 489def _user_activity_log(message: str) -> None:490 stamp = datetime.now(tz=UTC).strftime("%H:%M:%SZ")491 print(f"[{stamp}] {message}", flush=True)492