diff --git a/CHANGELOG.md b/CHANGELOG.md index eb0c8126e..21c0717c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,20 @@ +## [2.2.4-beta.1](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v2.2.3...v2.2.4-beta.1) (2026-09-07) + + +### Bug Fixes + +* 🐛 read SCRAPEGRAPHAI_TELEMETRY_ENABLED from the environment, not the config file ([8769c3b](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/8769c3bddd7c865963cc7e245eefb496f55dc519)) +* **models:** add Gemini 2.5 token limits so they are not truncated to 8192 ([c21af20](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/c21af206862c13be1848eac75b4c04250718c8d9)) +* **fetch:** surface HTTP errors and missing content instead of answering NA ([f91478e](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/f91478eacf86485f6b9efcf843fc0c815dde1ec5)), closes [#1102](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/1102) [#1102](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/1102) + + +### CI + +* **release:** 2.2.0-beta.10 [skip ci] ([0bb8bc9](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/0bb8bc935028b4f0a91444db2866ec0142f97199)) +* **release:** 2.2.0-beta.7 [skip ci] ([decfc6b](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/decfc6bb6eb10a29ed6aaabb07244b8915042604)) +* **release:** 2.2.0-beta.8 [skip ci] ([d59c3df](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/d59c3dfceecdacbba4e17f237b017117cf7f1cee)), closes [#1102](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/1102) [#1102](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/1102) +* **release:** 2.2.0-beta.9 [skip ci] ([3047ef8](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/3047ef8eda694d19c6fe4654777ea6343744acba)) + ## [2.2.3](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v2.2.2...v2.2.3) (2026-09-07) diff --git a/pyproject.toml b/pyproject.toml index ccf6b7b5f..1348b0788 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "scrapegraphai" -version = "2.2.3" +version = "2.2.4b1" description = "A web scraping library based on LangChain which uses LLM and direct graph logic to create scraping pipelines." authors = [ diff --git a/scrapegraphai/helpers/models_tokens.py b/scrapegraphai/helpers/models_tokens.py index b5aef2f9b..e64b1225b 100644 --- a/scrapegraphai/helpers/models_tokens.py +++ b/scrapegraphai/helpers/models_tokens.py @@ -140,6 +140,11 @@ "gemini-2.0-flash-latest": 1000000, "gemini-2.0-flash-exp": 1000000, "gemini-2.0-pro-exp": 2000000, + "gemini-2.5-flash": 1000000, + "gemini-2.5-flash-latest": 1000000, + "gemini-2.5-flash-lite": 1000000, + "gemini-2.5-pro": 1000000, + "gemini-flash-latest": 1000000, "models/embedding-001": 2048, }, "google_vertexai": { @@ -150,6 +155,10 @@ "gemini-2.0-flash-exp": 1048576, "gemini-2.0-pro": 2000000, "gemini-2.0-pro-exp": 2000000, + "gemini-2.5-flash": 1048576, + "gemini-2.5-flash-lite": 1048576, + "gemini-2.5-pro": 1048576, + "gemini-flash-latest": 1048576, }, "ollama": { "command-r": 12800, diff --git a/scrapegraphai/telemetry/telemetry.py b/scrapegraphai/telemetry/telemetry.py index 07fa6e080..fba586b33 100644 --- a/scrapegraphai/telemetry/telemetry.py +++ b/scrapegraphai/telemetry/telemetry.py @@ -36,6 +36,19 @@ def _load_config(config_location: str) -> configparser.ConfigParser: return config +def _parse_bool(value: str) -> bool: + """Parse a boolean from a string using configparser's accepted spellings. + + Accepts the same values as the config file does, so + ``SCRAPEGRAPHAI_TELEMETRY_ENABLED=false`` and ``telemetry_enabled = false`` + behave identically. Raises ValueError on anything unrecognised. + """ + try: + return configparser.ConfigParser.BOOLEAN_STATES[value.strip().lower()] + except KeyError: + raise ValueError(f"invalid boolean value: {value!r}") + + def _check_config_and_environ_for_telemetry_flag(default_value: bool, config_obj): telemetry_enabled = default_value if "telemetry_enabled" in config_obj["DEFAULT"]: @@ -44,13 +57,18 @@ def _check_config_and_environ_for_telemetry_flag(default_value: bool, config_obj except Exception: pass - if os.environ.get("SCRAPEGRAPHAI_TELEMETRY_ENABLED") is not None: + env_value = os.environ.get("SCRAPEGRAPHAI_TELEMETRY_ENABLED") + if env_value is not None: try: - telemetry_enabled = config_obj.getboolean( - "DEFAULT", "telemetry_enabled" + telemetry_enabled = _parse_bool(env_value) + except ValueError: + logger.warning( + "SCRAPEGRAPHAI_TELEMETRY_ENABLED is set to %r, which is not a " + "recognised boolean. Telemetry is left at %s. Use one of: " + "true/false, yes/no, on/off, 1/0.", + env_value, + telemetry_enabled, ) - except Exception: - pass return telemetry_enabled diff --git a/tests/test_models_tokens.py b/tests/test_models_tokens.py index bfde8df9c..9b40e5fcd 100644 --- a/tests/test_models_tokens.py +++ b/tests/test_models_tokens.py @@ -7,9 +7,9 @@ class TestModelsTokens: def test_openai_tokens(self): """Test that the 'openai' provider exists and its tokens are valid positive integers.""" openai_models = models_tokens.get("openai") - assert openai_models is not None, ( - "'openai' key should be present in models_tokens" - ) + assert ( + openai_models is not None + ), "'openai' key should be present in models_tokens" for model, token in openai_models.items(): assert isinstance(model, str), "Model name should be a string" assert isinstance(token, int), "Token limit should be an integer" @@ -30,19 +30,50 @@ def test_google_providers(self): assert google_genai is not None, "'google_genai' key should be present" assert google_vertexai is not None, "'google_vertexai' key should be present" # Check a specific key from google_genai - assert "gemini-pro" in google_genai, ( - "'gemini-pro' should be in google_genai models" - ) + assert ( + "gemini-pro" in google_genai + ), "'gemini-pro' should be in google_genai models" # Validate token values types for provider in [google_genai, google_vertexai]: for token in provider.values(): assert isinstance(token, int), "Token limit must be an integer" + def test_gemini_2_5_models_are_registered(self): + """Gemini 2.5 / flash-latest must be in the table so they are not truncated to 8192. + + #1121: an unknown model silently falls back to an 8192-token window. + google_genai/gemini-2.5-flash is the reported case; gemini-flash-latest + is the current flash alias. Both have a 1M input context. + """ + google_genai = models_tokens["google_genai"] + google_vertexai = models_tokens["google_vertexai"] + + for model in ( + "gemini-2.5-flash", + "gemini-2.5-flash-latest", + "gemini-2.5-flash-lite", + "gemini-2.5-pro", + "gemini-flash-latest", + ): + assert ( + google_genai.get(model) == 1000000 + ), f"Expected 1M context for {model} in google_genai" + + for model in ( + "gemini-2.5-flash", + "gemini-2.5-flash-lite", + "gemini-2.5-pro", + "gemini-flash-latest", + ): + assert ( + google_vertexai.get(model) == 1048576 + ), f"Expected 1M context for {model} in google_vertexai" + def test_non_existent_provider(self): """Test that a non-existent provider returns None.""" - assert models_tokens.get("non_existent") is None, ( - "Non-existent provider should return None" - ) + assert ( + models_tokens.get("non_existent") is None + ), "Non-existent provider should return None" def test_total_model_keys(self): """Test that the total number of models across all providers is above an expected count.""" @@ -59,120 +90,120 @@ def test_non_empty_model_keys(self): """Ensure that model token names are non-empty strings.""" for provider, model_dict in models_tokens.items(): for model in model_dict.keys(): - assert model != "", ( - f"Model name in provider '{provider}' should not be empty." - ) + assert ( + model != "" + ), f"Model name in provider '{provider}' should not be empty." def test_token_limits_range(self): """Test that token limits for all models fall within a plausible range (e.g., 1 to 300000).""" for provider, model_dict in models_tokens.items(): for model, token in model_dict.items(): - assert 1 <= token <= 1100000, ( - f"Token limit for {model} in provider {provider} is out of plausible range." - ) + assert ( + 1 <= token <= 1100000 + ), f"Token limit for {model} in provider {provider} is out of plausible range." def test_provider_structure(self): """Test that every provider in models_tokens has a dictionary as its value.""" for provider, models in models_tokens.items(): - assert isinstance(models, dict), ( - f"Provider {provider} should map to a dictionary, got {type(models).__name__}" - ) + assert isinstance( + models, dict + ), f"Provider {provider} should map to a dictionary, got {type(models).__name__}" def test_non_empty_provider(self): """Test that each provider dictionary is not empty.""" for provider, models in models_tokens.items(): - assert len(models) > 0, ( - f"Provider {provider} should contain at least one model." - ) + assert ( + len(models) > 0 + ), f"Provider {provider} should contain at least one model." def test_specific_model_token_values(self): """Test specific expected token values for selected models from various providers.""" # Verify a token for a selected model from the 'openai' provider openai = models_tokens.get("openai") - assert openai.get("gpt-3.5-turbo-0125") == 16385, ( - "Expected token limit for gpt-3.5-turbo-0125 in openai to be 16385" - ) + assert ( + openai.get("gpt-3.5-turbo-0125") == 16385 + ), "Expected token limit for gpt-3.5-turbo-0125 in openai to be 16385" # Verify a token for a selected model from the 'azure_openai' provider azure = models_tokens.get("azure_openai") - assert azure.get("gpt-3.5") == 4096, ( - "Expected token limit for gpt-3.5 in azure_openai to be 4096" - ) + assert ( + azure.get("gpt-3.5") == 4096 + ), "Expected token limit for gpt-3.5 in azure_openai to be 4096" # Verify a token for a selected model from the 'anthropic' provider anthropic = models_tokens.get("anthropic") - assert anthropic.get("claude_instant") == 100000, ( - "Expected token limit for claude_instant in anthropic to be 100000" - ) + assert ( + anthropic.get("claude_instant") == 100000 + ), "Expected token limit for claude_instant in anthropic to be 100000" def test_providers_count(self): """Test that the total number of providers is as expected (at least 15).""" - assert len(models_tokens) >= 15, ( - "Expected at least 15 providers in models_tokens" - ) + assert ( + len(models_tokens) >= 15 + ), "Expected at least 15 providers in models_tokens" def test_non_existent_model(self): """Test that a non-existent model within a valid provider returns None.""" openai = models_tokens.get("openai") - assert openai.get("non_existent_model") is None, ( - "Non-existent model should return None from a valid provider." - ) + assert ( + openai.get("non_existent_model") is None + ), "Non-existent model should return None from a valid provider." def test_no_whitespace_in_model_names(self): """Test that model names do not contain leading or trailing whitespace.""" for provider, model_dict in models_tokens.items(): for model in model_dict.keys(): # Assert that stripping whitespace does not change the model name - assert model == model.strip(), ( - f"Model name '{model}' in provider '{provider}' contains leading or trailing whitespace." - ) + assert ( + model == model.strip() + ), f"Model name '{model}' in provider '{provider}' contains leading or trailing whitespace." def test_specific_models_additional(self): """Test specific token values for additional models across various providers.""" # Check some models in the 'ollama' provider ollama = models_tokens.get("ollama") - assert ollama.get("llama2") == 4096, ( - "Expected token limit for 'llama2' in ollama to be 4096" - ) - assert ollama.get("llama2:70b") == 4096, ( - "Expected token limit for 'llama2:70b' in ollama to be 4096" - ) + assert ( + ollama.get("llama2") == 4096 + ), "Expected token limit for 'llama2' in ollama to be 4096" + assert ( + ollama.get("llama2:70b") == 4096 + ), "Expected token limit for 'llama2:70b' in ollama to be 4096" # Check a specific model from the 'mistralai' provider mistralai = models_tokens.get("mistralai") - assert mistralai.get("open-codestral-mamba") == 256000, ( - "Expected token limit for 'open-codestral-mamba' in mistralai to be 256000" - ) + assert ( + mistralai.get("open-codestral-mamba") == 256000 + ), "Expected token limit for 'open-codestral-mamba' in mistralai to be 256000" # Check a specific model from the 'deepseek' provider deepseek = models_tokens.get("deepseek") - assert deepseek.get("deepseek-chat") == 28672, ( - "Expected token limit for 'deepseek-chat' in deepseek to be 28672" - ) + assert ( + deepseek.get("deepseek-chat") == 28672 + ), "Expected token limit for 'deepseek-chat' in deepseek to be 28672" # Check a model from the 'ernie' provider ernie = models_tokens.get("ernie") - assert ernie.get("ernie-bot") == 4096, ( - "Expected token limit for 'ernie-bot' in ernie to be 4096" - ) + assert ( + ernie.get("ernie-bot") == 4096 + ), "Expected token limit for 'ernie-bot' in ernie to be 4096" def test_nvidia_specific(self): """Test specific token value for 'meta/codellama-70b' in the nvidia provider.""" nvidia = models_tokens.get("nvidia") assert nvidia is not None, "'nvidia' provider should exist" # Verify token for 'meta/codellama-70b' equals 16384 as defined in the nvidia dictionary - assert nvidia.get("meta/codellama-70b") == 16384, ( - "Expected token limit for 'meta/codellama-70b' in nvidia to be 16384" - ) + assert ( + nvidia.get("meta/codellama-70b") == 16384 + ), "Expected token limit for 'meta/codellama-70b' in nvidia to be 16384" def test_groq_specific(self): """Test specific token value for 'claude-3-haiku-20240307\'' in the groq provider.""" groq = models_tokens.get("groq") assert groq is not None, "'groq' provider should exist" # Note: The model name has an embedded apostrophe at the end in its name. - assert groq.get("claude-3-haiku-20240307'") == 8192, ( - "Expected token limit for 'claude-3-haiku-20240307\\'' in groq to be 8192" - ) + assert ( + groq.get("claude-3-haiku-20240307'") == 8192 + ), "Expected token limit for 'claude-3-haiku-20240307\\'' in groq to be 8192" def test_togetherai_specific(self): """Test specific token value for 'meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo' in the toghetherai provider.""" @@ -180,15 +211,15 @@ def test_togetherai_specific(self): assert togetherai is not None, "'toghetherai' provider should exist" expected = 128000 model_name = "meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo" - assert togetherai.get(model_name) == expected, ( - f"Expected token limit for '{model_name}' in toghetherai to be {expected}" - ) + assert ( + togetherai.get(model_name) == expected + ), f"Expected token limit for '{model_name}' in toghetherai to be {expected}" def test_ernie_all_values(self): """Test that all models in the 'ernie' provider have token values exactly 4096.""" ernie = models_tokens.get("ernie") assert ernie is not None, "'ernie' provider should exist" for model, token in ernie.items(): - assert token == 4096, ( - f"Expected token limit for '{model}' in ernie to be 4096, got {token}" - ) + assert ( + token == 4096 + ), f"Expected token limit for '{model}' in ernie to be 4096, got {token}" diff --git a/tests/test_telemetry_flag.py b/tests/test_telemetry_flag.py new file mode 100644 index 000000000..c3743c1f4 --- /dev/null +++ b/tests/test_telemetry_flag.py @@ -0,0 +1,70 @@ +"""Tests for the telemetry opt-out flag. + +These cover the environment variable path, which previously read its value from +the config file instead of from the variable, so `SCRAPEGRAPHAI_TELEMETRY_ENABLED=false` +left telemetry enabled. +""" + +import configparser + +import pytest + +from scrapegraphai.telemetry.telemetry import ( + _check_config_and_environ_for_telemetry_flag, + _parse_bool, +) + + +def _config(**defaults): + cfg = configparser.ConfigParser() + cfg["DEFAULT"] = {k: str(v) for k, v in defaults.items()} + return cfg + + +class TestParseBool: + @pytest.mark.parametrize("value", ["false", "False", "FALSE", "no", "off", "0", " false "]) + def test_falsey_spellings(self, value): + assert _parse_bool(value) is False + + @pytest.mark.parametrize("value", ["true", "True", "yes", "on", "1"]) + def test_truthy_spellings(self, value): + assert _parse_bool(value) is True + + def test_rejects_nonsense(self): + with pytest.raises(ValueError): + _parse_bool("maybe") + + +class TestTelemetryFlag: + def test_defaults_to_the_given_default(self): + assert _check_config_and_environ_for_telemetry_flag(True, _config()) is True + + def test_config_file_can_disable(self): + cfg = _config(telemetry_enabled="False") + assert _check_config_and_environ_for_telemetry_flag(True, cfg) is False + + def test_env_var_disables_with_no_config_key(self, monkeypatch): + """The regression. Previously returned True, because the value was read + out of the config file rather than out of the environment variable.""" + monkeypatch.setenv("SCRAPEGRAPHAI_TELEMETRY_ENABLED", "false") + assert _check_config_and_environ_for_telemetry_flag(True, _config()) is False + + def test_env_var_overrides_the_config_file(self, monkeypatch): + monkeypatch.setenv("SCRAPEGRAPHAI_TELEMETRY_ENABLED", "false") + cfg = _config(telemetry_enabled="True") + assert _check_config_and_environ_for_telemetry_flag(True, cfg) is False + + def test_env_var_can_also_enable(self, monkeypatch): + monkeypatch.setenv("SCRAPEGRAPHAI_TELEMETRY_ENABLED", "true") + cfg = _config(telemetry_enabled="False") + assert _check_config_and_environ_for_telemetry_flag(True, cfg) is True + + def test_unparseable_env_var_leaves_the_flag_alone(self, monkeypatch): + monkeypatch.setenv("SCRAPEGRAPHAI_TELEMETRY_ENABLED", "banana") + cfg = _config(telemetry_enabled="False") + assert _check_config_and_environ_for_telemetry_flag(True, cfg) is False + + def test_unset_env_var_leaves_the_config_in_charge(self, monkeypatch): + monkeypatch.delenv("SCRAPEGRAPHAI_TELEMETRY_ENABLED", raising=False) + cfg = _config(telemetry_enabled="False") + assert _check_config_and_environ_for_telemetry_flag(True, cfg) is False