diff --git a/user_scanner/core/result.py b/user_scanner/core/result.py index b1d5ece..a990165 100644 --- a/user_scanner/core/result.py +++ b/user_scanner/core/result.py @@ -2,7 +2,10 @@ import io import json from enum import Enum +from typing import Any + from colorama import Fore, Style + from user_scanner.core.helpers import ScanConfig # Added {url} to the debug message @@ -89,19 +92,42 @@ def update(self, **kwargs): setattr(self, field, kwargs[field]) if "extra" in kwargs and isinstance(kwargs["extra"], dict): - for key, value in kwargs["extra"].items(): - if value is None or (isinstance(value, str) and not value.strip()): - continue + self.add_extras(kwargs["extra"]) - clean_key = key.strip().rstrip(":").strip().replace(" ", "_").lower() - if not clean_key: - continue + if "extras" in kwargs and isinstance(kwargs["extras"], dict): + self.add_extras(kwargs["extras"]) + + return self - if not isinstance(value, (bool, int)): - value = str(value) + def add_extra_unchecked(self, key: str, value: Any): + if value is None or (isinstance(value, str) and not value.strip()): + return self + + clean_key = key.strip().rstrip(":").strip().replace(" ", "_").lower() + if not clean_key: + return self - self.extra[clean_key] = value + if not isinstance(value, (bool, int)): + value = str(value) + + self.extra[clean_key] = value + return self + + def add_extras_unchecked(self, extras: dict[str, Any]): + for key, value in extras.items(): + self.add_extra_unchecked(key, value) + return self + + def add_extra(self, key: str, value: Any): + # Intentionally ignores 0, False, and "" in addition to None + if value: + return self.add_extra_unchecked(key, value) + return self + def add_extras(self, extras: dict[str, Any]): + # Intentionally ignores 0, False, and "" in addition to None + for key, value in extras.items(): + self.add_extra(key, value) return self @classmethod diff --git a/user_scanner/user_scan/community/ghost_forum.py b/user_scanner/user_scan/community/ghost_forum.py index 0a0844b..95222eb 100644 --- a/user_scanner/user_scan/community/ghost_forum.py +++ b/user_scanner/user_scan/community/ghost_forum.py @@ -1,4 +1,5 @@ -from user_scanner.core.orchestrator import generic_validate, Result +from user_scanner.core.orchestrator import Result, generic_validate + def validate_ghost_forum(user): url = f"https://forum.ghost.org/u/{user}.json" @@ -11,15 +12,16 @@ def process(response): data = response.json() u = data.get("user", {}) if u: - extra = {} - if u.get("id"): extra["id"] = u.get("id") - if u.get("name"): extra["name"] = u.get("name") - if u.get("username"): extra["username"] = u.get("username") - if u.get("title"): extra["title"] = u.get("title") - if u.get("last_posted_at"): extra["last_posted"] = u.get("last_posted_at") - if u.get("last_seen_at"): extra["last_seen"] = u.get("last_seen_at") - if u.get("created_at"): extra["registered"] = u.get("created_at") - + extra = { + "id": u.get("id"), + "name": u.get("name"), + "username": u.get("username"), + "title": u.get("title"), + "last_posted": u.get("last_posted_at"), + "last_seen": u.get("last_seen_at"), + "registered": u.get("created_at"), + } + # Resolve avatar avatar = u.get("avatar_template") if avatar: @@ -28,9 +30,9 @@ def process(response): if avatar.startswith("/"): avatar = "https://forum.ghost.org" + avatar extra["avatar"] = avatar - + return Result.taken(extra=extra) - + return Result.error(f"Unexpected response status: {response.status_code}") headers = {"Accept": "application/json", "User-Agent": "Mozilla/5.0"} diff --git a/user_scanner/user_scan/community/jupyter_forum.py b/user_scanner/user_scan/community/jupyter_forum.py index c560226..1224fe9 100644 --- a/user_scanner/user_scan/community/jupyter_forum.py +++ b/user_scanner/user_scan/community/jupyter_forum.py @@ -1,5 +1,6 @@ from user_scanner.core.orchestrator import Result, make_request + def validate_jupyter_forum(user): url = f"https://discourse.jupyter.org/u/{user}.json" show_url = f"https://discourse.jupyter.org/u/{user}" @@ -11,15 +12,16 @@ def validate_jupyter_forum(user): data = response.json() u = data.get("user", {}) if u: - extra = {} - if u.get("id"): extra["id"] = u.get("id") - if u.get("name"): extra["name"] = u.get("name") - if u.get("username"): extra["username"] = u.get("username") - if u.get("title"): extra["title"] = u.get("title") - if u.get("last_posted_at"): extra["last_posted"] = u.get("last_posted_at") - if u.get("last_seen_at"): extra["last_seen"] = u.get("last_seen_at") - if u.get("created_at"): extra["registered"] = u.get("created_at") - + extra = { + "id": u.get("id"), + "name": u.get("name"), + "username": u.get("username"), + "title": u.get("title"), + "last_posted": u.get("last_posted_at"), + "last_seen": u.get("last_seen_at"), + "registered": u.get("created_at"), + } + # Resolve avatar avatar = u.get("avatar_template") if avatar: @@ -28,12 +30,14 @@ def validate_jupyter_forum(user): if avatar.startswith("/"): avatar = "https://discourse.jupyter.org" + avatar extra["avatar"] = avatar - + return Result.taken(extra=extra, url=show_url) return Result.available(url=show_url) elif response.status_code == 404: return Result.available(url=show_url) else: - return Result.error(f"Unexpected status: {response.status_code}", url=show_url) + return Result.error( + f"Unexpected status: {response.status_code}", url=show_url + ) except Exception as e: return Result.error(e, url=show_url) diff --git a/user_scanner/user_scan/community/lemmy.py b/user_scanner/user_scan/community/lemmy.py index ab4cbbc..755aaec 100644 --- a/user_scanner/user_scan/community/lemmy.py +++ b/user_scanner/user_scan/community/lemmy.py @@ -28,24 +28,16 @@ def process(response): person = person_view.get("person", {}) counts = person_view.get("counts", {}) - if "id" in person: - extra["id"] = person["id"] - if "name" in person: - extra["name"] = person["name"] - if person.get("display_name"): - extra["display_name"] = person["display_name"] - if person.get("avatar"): - extra["avatar"] = person["avatar"] - if person.get("published"): - extra["joined"] = person["published"] - if "bot_account" in person: - extra["bot"] = person["bot_account"] - if "is_admin" in person_view: - extra["admin"] = person_view["is_admin"] - if "post_count" in counts: - extra["posts"] = counts["post_count"] - if "comment_count" in counts: - extra["comments"] = counts["comment_count"] + extra["id"] = person.get("id") + extra["name"] = person.get("name") + extra["display_name"] = person.get("display_name") + extra["avatar"] = person.get("avatar") + extra["joined"] = person.get("published") + extra["bot"] = person.get("bot_account") + extra["admin"] = person_view.get("is_admin") + extra["posts"] = counts.get("post_count") + extra["comments"] = counts.get("comment_count") + except Exception: pass return Result.taken(extra=extra) diff --git a/user_scanner/user_scan/community/wikipedia.py b/user_scanner/user_scan/community/wikipedia.py index 6f8da4b..31b7ca0 100644 --- a/user_scanner/user_scan/community/wikipedia.py +++ b/user_scanner/user_scan/community/wikipedia.py @@ -1,5 +1,6 @@ from user_scanner.core.orchestrator import Result, make_request + def validate_wikipedia(user): # Using formatversion=2 for a cleaner JSON response api_url = f"https://en.wikipedia.org/w/api.php?action=query&format=json&list=users&ususers={user}&usprop=editcount|registration|gender&formatversion=2" @@ -10,30 +11,31 @@ def validate_wikipedia(user): if response.status_code == 200: data = response.json() users = data.get("query", {}).get("users", []) - + if not users: return Result.error("Invalid API response format", url=show_url) - + user_data = users[0] - + # Wikipedia API returns a "missing" key if the user does not exist if "missing" in user_data: return Result.available(url=show_url) - - extra = {} + + extra = { + "registration": user_data.get("registration"), + "gender": user_data.get("gender"), + } if userid := user_data.get("userid"): extra["userid"] = str(userid) if editcount := user_data.get("editcount"): extra["editcount"] = str(editcount) - if registration := user_data.get("registration"): - extra["registration"] = registration - if gender := user_data.get("gender"): - extra["gender"] = gender - + return Result.taken(extra=extra, url=show_url) - + else: - return Result.error(f"Unexpected status: {response.status_code}", url=show_url) - + return Result.error( + f"Unexpected status: {response.status_code}", url=show_url + ) + except Exception as e: return Result.error(e, url=show_url) diff --git a/user_scanner/user_scan/dev/codecademy.py b/user_scanner/user_scan/dev/codecademy.py index 049cd9f..b67e459 100644 --- a/user_scanner/user_scan/dev/codecademy.py +++ b/user_scanner/user_scan/dev/codecademy.py @@ -1,7 +1,9 @@ -from user_scanner.core.orchestrator import generic_validate, Result import json import re +from user_scanner.core.orchestrator import Result, generic_validate + + def validate_codecademy(user): url = f"https://www.codecademy.com/profiles/{user}" @@ -11,22 +13,23 @@ def process(response): elif response.status_code == 200: try: # Codecademy embeds profile data in a Next.js JSON blob - match = re.search(r'', response.text) + match = re.search( + r'', + response.text, + ) if match: data = json.loads(match.group(1)) - user_data = data.get("props", {}).get("pageProps", {}).get("profile", {}) - if user_data: - extra = {} - if user_data.get("name"): - extra["name"] = user_data.get("name") - if user_data.get("bio"): - extra["bio"] = user_data.get("bio") - if user_data.get("location"): - extra["location"] = user_data.get("location") - if user_data.get("createdAt"): - extra["joined"] = user_data.get("createdAt") - return Result.taken(extra=extra) - + user_data = ( + data.get("props", {}).get("pageProps", {}).get("profile", {}) + ) + extra = { + "name": user_data.get("name"), + "bio": user_data.get("bio"), + "location": user_data.get("location"), + "joined": user_data.get("createdAt"), + } + return Result.taken(extra=extra) + # If we couldn't parse it but it's 200, assume taken but no extra data return Result.taken() except Exception: diff --git a/user_scanner/user_scan/dev/codeforces.py b/user_scanner/user_scan/dev/codeforces.py index cb37f81..c0eff0a 100644 --- a/user_scanner/user_scan/dev/codeforces.py +++ b/user_scanner/user_scan/dev/codeforces.py @@ -1,4 +1,5 @@ -from user_scanner.core.orchestrator import generic_validate, Result +from user_scanner.core.orchestrator import Result, generic_validate + def validate_codeforces(user): url = f"https://codeforces.com/api/user.info?handles={user}" @@ -8,37 +9,32 @@ def process(response): if response.status_code == 200: try: data = response.json() - if data.get('status') == 'OK' and data.get('result'): - res = data['result'][0] - extra = {} - if res.get('firstName'): - extra['firstName'] = res.get('firstName') - if res.get('lastName'): - extra['lastName'] = res.get('lastName') - if res.get('country'): - extra['country'] = res.get('country') - if res.get('city'): - extra['city'] = res.get('city') - if res.get('organization'): - extra['organization'] = res.get('organization') - if res.get('rating') is not None: - extra['rating'] = res.get('rating') - if res.get('maxRating') is not None: - extra['maxRating'] = res.get('maxRating') - if res.get('rank'): - extra['rank'] = res.get('rank') - if res.get('maxRank'): - extra['maxRank'] = res.get('maxRank') - if res.get('friendOfCount') is not None: - extra['friendOfCount'] = res.get('friendOfCount') + if data.get("status") == "OK" and data.get("result"): + res = data["result"][0] + + extra = { + "firstName": res.get("firstName"), + "lastName": res.get("lastName"), + "country": res.get("country"), + "city": res.get("city"), + "organization": res.get("organization"), + "rating": res.get("rating"), + "maxRating": res.get("maxRating"), + "rank": res.get("rank"), + "maxRank": res.get("maxRank"), + "friendOfCount": res.get("friendOfCount"), + } return Result.taken(extra=extra) except Exception: pass elif response.status_code == 400 or response.status_code == 404: return Result.available() - + return Result.error("Unexpected response body, report it via GitHub issues.") - headers = {"Accept": "application/json", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"} + headers = { + "Accept": "application/json", + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", + } return generic_validate(url, process, show_url=show_url, headers=headers) diff --git a/user_scanner/user_scan/dev/codewars.py b/user_scanner/user_scan/dev/codewars.py index 4434c4d..9051469 100644 --- a/user_scanner/user_scan/dev/codewars.py +++ b/user_scanner/user_scan/dev/codewars.py @@ -1,4 +1,5 @@ -from user_scanner.core.orchestrator import generic_validate, Result +from user_scanner.core.orchestrator import Result, generic_validate + def validate_codewars(user): url = f"https://www.codewars.com/api/v1/users/{user}" @@ -8,22 +9,25 @@ def process(response): if response.status_code == 200: try: data = response.json() - if data and data.get('id'): - extra = {} - if data.get('id'): extra['id'] = data.get('id') - if data.get('name'): extra['name'] = data.get('name') - if data.get('honor') is not None: extra['honor'] = data.get('honor') - if data.get('clan'): extra['clan'] = data.get('clan') - if data.get('leaderboardPosition') is not None: extra['leaderboard_position'] = data.get('leaderboardPosition') - ranks = data.get('ranks', {}).get('overall', {}) - if ranks.get('name'): extra['rank_name'] = ranks.get('name') - if ranks.get('score') is not None: extra['score'] = ranks.get('score') + if data and data.get("id"): + ranks = data.get("ranks", {}).get("overall", {}) + + extra = { + "id": data.get("id"), + "name": data.get("name"), + "honor": data.get("honor"), + "clan": data.get("clan"), + "leaderboard_position": data.get("leaderboardPosition"), + "rank_name": ranks.get("name"), + "score": ranks.get("score"), + } + return Result.taken(extra=extra) except Exception: pass elif response.status_code == 404: return Result.available() - + return Result.error("Unexpected response body, report it via GitHub issues.") headers = {"Accept": "application/json", "User-Agent": "Mozilla/5.0"} diff --git a/user_scanner/user_scan/dev/f_droid.py b/user_scanner/user_scan/dev/f_droid.py index 4ab7b9c..a8145c1 100644 --- a/user_scanner/user_scan/dev/f_droid.py +++ b/user_scanner/user_scan/dev/f_droid.py @@ -1,4 +1,5 @@ -from user_scanner.core.orchestrator import generic_validate, Result +from user_scanner.core.orchestrator import Result, generic_validate + def validate_f_droid(user): url = f"https://forum.f-droid.org/u/{user}.json" @@ -11,15 +12,16 @@ def process(response): data = response.json() u = data.get("user", {}) if u: - extra = {} - if u.get("id"): extra["id"] = u.get("id") - if u.get("name"): extra["name"] = u.get("name") - if u.get("username"): extra["username"] = u.get("username") - if u.get("title"): extra["title"] = u.get("title") - if u.get("last_posted_at"): extra["last_posted"] = u.get("last_posted_at") - if u.get("last_seen_at"): extra["last_seen"] = u.get("last_seen_at") - if u.get("created_at"): extra["registered"] = u.get("created_at") - + extra = { + "id": u.get("id"), + "name": u.get("name"), + "username": u.get("username"), + "title": u.get("title"), + "last_posted": u.get("last_posted_at"), + "last_seen": u.get("last_seen_at"), + "registered": u.get("created_at"), + } + # Resolve avatar avatar = u.get("avatar_template") if avatar: @@ -28,9 +30,9 @@ def process(response): if avatar.startswith("/"): avatar = "https://forum.f-droid.org" + avatar extra["avatar"] = avatar - + return Result.taken(extra=extra) - + return Result.error(f"Unexpected response status: {response.status_code}") headers = {"Accept": "application/json", "User-Agent": "Mozilla/5.0"} diff --git a/user_scanner/user_scan/other/trello.py b/user_scanner/user_scan/other/trello.py index 14a8621..a4708e8 100644 --- a/user_scanner/user_scan/other/trello.py +++ b/user_scanner/user_scan/other/trello.py @@ -1,4 +1,5 @@ -from user_scanner.core.orchestrator import generic_validate, Result +from user_scanner.core.orchestrator import Result, generic_validate + def validate_trello(user): url = f"https://trello.com/1/Members/{user}" @@ -8,26 +9,25 @@ def process(response): if response.status_code == 200: try: data = response.json() - if data and data.get('id'): - extra = {} - if data.get('id'): - extra['id'] = data.get('id') - if data.get('fullName'): - extra['fullName'] = data.get('fullName') - if data.get('bio'): - extra['bio'] = data.get('bio') - if data.get('initials'): - extra['initials'] = data.get('initials') - if data.get('username'): - extra['username'] = data.get('username') + if data and data.get("id"): + extras = { + "id": data.get("id"), + "fullName": data.get("fullName"), + "bio": data.get("bio"), + "initials": data.get("initials"), + "username": data.get("username"), + } - return Result.taken(extra=extra) + return Result.taken(extras=extras) except Exception: pass elif response.status_code == 404 or response.status_code == 401: return Result.available() - + return Result.error("Unexpected response body, report it via GitHub issues.") - headers = {"Accept": "application/json", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"} + headers = { + "Accept": "application/json", + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", + } return generic_validate(url, process, show_url=show_url, headers=headers)