GSoC Module A : week 4 : feat(harvester): implement repository file filtering (stacked on top of #985)

application/tests/harvester_test/__init__.py

@@ -0,0 +1,3 @@
+"""
+tests for Module A configuration layer (empty for now)
+"""

application/tests/harvester_test/change_detector_test.py

@@ -0,0 +1,52 @@
+import unittest
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+from application.utils.harvester.change_detector import (
+    ChangeDetector,
+)
+
+
+class ChangeDetectorTests(unittest.TestCase):
+    @patch("application.utils.harvester.change_detector.subprocess.run")
+    def test_get_modified_files_since(self, mock_run):
+        mock_run.return_value = MagicMock(
+            stdout="a.md\nb.md\na.md\n",
+        )
+
+        client = MagicMock()
+        detector = ChangeDetector(client)
+
+        files = detector.get_modified_files_since("abc123")
+
+        self.assertEqual(
+            files,
+            [
+                "a.md",
+                "b.md",
+            ],
+        )
+
+    @patch("application.utils.harvester.change_detector.subprocess.run")
+    def test_get_commits_since(self, mock_run):
+        mock_run.return_value = MagicMock(
+            stdout="111\n222\n333\n",
+        )
+
+        client = MagicMock()
+        detector = ChangeDetector(client)
+
+        commits = detector.get_commits_since("abc123")
+
+        self.assertEqual(
+            commits,
+            [
+                "111",
+                "222",
+                "333",
+            ],
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

application/tests/harvester_test/checkpoint_store_test.py

@@ -0,0 +1,92 @@
+import unittest
+from datetime import datetime
+from pathlib import Path
+
+from application.utils.harvester.checkpoint_store import (
+    CheckpointStore,
+)
+from application.utils.harvester.models import (
+    RepositoryCheckpoint,
+)
+
+
+class CheckpointStoreTests(unittest.TestCase):
+    def test_save_and_load_checkpoint(self):
+        tmp_dir = Path(self._testMethodName)
+
+        try:
+            store = CheckpointStore(
+                tmp_dir / "checkpoints.json",
+            )
+
+            checkpoint = RepositoryCheckpoint(
+                repository_id="owasp-asvs",
+                last_processed_commit="abc123",
+                updated_at=datetime.now(),
+            )
+
+            store.save(checkpoint)
+
+            loaded = store.load("owasp-asvs")
+
+            if loaded is None:
+                self.fail("Checkpoint should have been loaded")
+
+            self.assertEqual(
+                loaded.last_processed_commit,
+                "abc123",
+            )
+
+        finally:
+            if tmp_dir.exists():
+                import shutil
+
+                shutil.rmtree(tmp_dir)
+
+    def test_load_missing_file(self):
+        tmp_dir = Path(self._testMethodName)
+
+        try:
+            store = CheckpointStore(
+                tmp_dir / "missing.json",
+            )
+
+            self.assertIsNone(
+                store.load("repo"),
+            )
+
+        finally:
+            if tmp_dir.exists():
+                import shutil
+
+                shutil.rmtree(tmp_dir)
+
+    def test_load_missing_repository(self):
+        tmp_dir = Path(self._testMethodName)
+
+        try:
+            store = CheckpointStore(
+                tmp_dir / "checkpoint.json",
+            )
+
+            store.save(
+                RepositoryCheckpoint(
+                    repository_id="repo-a",
+                    last_processed_commit="abc123",
+                    updated_at=datetime.now(),
+                )
+            )
+
+            self.assertIsNone(
+                store.load("repo-b"),
+            )
+
+        finally:
+            if tmp_dir.exists():
+                import shutil
+
+                shutil.rmtree(tmp_dir)
+
+
+if __name__ == "__main__":
+    unittest.main()

application/tests/harvester_test/config_loader_test.py

@@ -0,0 +1,63 @@
+from pathlib import Path
+import unittest
+
+from application.utils.harvester.config_loader import (
+    ConfigLoaderError,
+    ConfigFileNotFoundError,
+    load_repo_config,
+)
+
+FIXTURES_DIR = Path(__file__).parent / "fixtures"
+
+
+class ConfigLoaderTests(unittest.TestCase):
+    def test_load_valid_config(self):
+        config_path = FIXTURES_DIR / "valid_repos.yaml"
+
+        config = load_repo_config(config_path)
+
+        self.assertEqual(len(config.repositories), 1)
+
+        repo = config.repositories[0]
+
+        self.assertEqual(repo.id, "owasp-asvs")
+        self.assertEqual(repo.owner, "OWASP")
+        self.assertEqual(repo.repo, "ASVS")
+
+    def test_missing_repository_id(self):
+        config_path = FIXTURES_DIR / "invalid_missing_id.yaml"
+
+        with self.assertRaises(ConfigLoaderError):
+            load_repo_config(config_path)
+
+    def test_invalid_chunk_size(self):
+        config_path = FIXTURES_DIR / "invalid_chunk_size.yaml"
+
+        with self.assertRaisesRegex(ConfigLoaderError, "max_tokens"):
+            load_repo_config(config_path)
+
+    def test_invalid_yaml_syntax(self):
+        config_path = FIXTURES_DIR / "invalid_yaml.yaml"
+
+        with self.assertRaises(ConfigLoaderError):
+            load_repo_config(config_path)
+
+    def test_missing_config_file(self):
+        with self.assertRaises(ConfigFileNotFoundError):
+            load_repo_config("does_not_exist.yaml")
+
+    def test_invalid_polling_interval(self):
+        config_path = FIXTURES_DIR / "invalid_polling_interval.yaml"
+
+        with self.assertRaises(ConfigLoaderError):
+            load_repo_config(config_path)
+
+    def test_empty_include_paths(self):
+        config_path = FIXTURES_DIR / "empty_include_paths.yaml"
+
+        with self.assertRaises(ConfigLoaderError):
+            load_repo_config(config_path)
+
+
+if __name__ == "__main__":
+    unittest.main()

application/tests/harvester_test/file_filter_test.py

@@ -0,0 +1,62 @@
+import unittest
+
+from application.utils.harvester.file_filter import (
+    FileFilter,
+)
+
+
+class FileFilterTests(unittest.TestCase):
+    def test_extension_filtering(self):
+        file_filter = FileFilter()
+
+        result = file_filter.filter_files(
+            [
+                "README.md",
+                "image.png",
+                "script.js",
+            ]
+        )
+
+        self.assertEqual(
+            result,
+            ["README.md"],
+        )
+
+    def test_regex_filtering(self):
+        file_filter = FileFilter()
+
+        result = file_filter.filter_files(
+            [
+                ".github/workflows/test.yml",
+                "docs/setup.md",
+            ]
+        )
+
+        self.assertEqual(
+            result,
+            ["docs/setup.md"],
+        )
+
+    def test_combined_filtering(self):
+        file_filter = FileFilter()
+
+        result = file_filter.filter_files(
+            [
+                "README.md",
+                ".github/workflows/test.yml",
+                "node_modules/react/index.js",
+                "docs/setup.md",
+            ]
+        )
+
+        self.assertEqual(
+            result,
+            [
+                "README.md",
+                "docs/setup.md",
+            ],
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

application/tests/harvester_test/filtering_benchmark_test.py

@@ -0,0 +1,30 @@
+import unittest
+
+from application.utils.harvester.file_filter import FileFilter
+from application.utils.harvester.filtering_benchmark import FilteringBenchmark
+
+
+class FilteringBenchmarkTests(unittest.TestCase):
+    def test_filtering_benchmark(self):
+        files = [
+            "README.md",
+            ".github/workflows/ci.yml",
+            "docs/guide.md",
+            "image.png",
+            "notes.txt",
+            "package-lock.json",
+        ]
+
+        benchmark = FilteringBenchmark(file_filter=FileFilter())
+
+        result = benchmark.run(files)
+
+        self.assertEqual(result.total_files, 6)
+        self.assertEqual(result.retained_files, 3)
+        self.assertEqual(result.filtered_files, 3)
+        self.assertEqual(result.retention_rate, 0.5)
+        self.assertEqual(result.filtering_rate, 0.5)
+
+
+if __name__ == "__main__":
+    unittest.main()

application/tests/harvester_test/filtering_metrics_test.py

@@ -0,0 +1,35 @@
+import unittest
+
+from application.utils.harvester.filtering_metrics import (
+    FilteringMetricsCollector,
+)
+
+
+class FilteringMetricsCollectorTests(unittest.TestCase):
+    def test_filtering_metrics_collection(self):
+        collector = FilteringMetricsCollector()
+
+        collector.record_retained()
+        collector.record_retained()
+        collector.record_filtered()
+
+        metrics = collector.build()
+
+        self.assertEqual(
+            metrics.total_files,
+            3,
+        )
+
+        self.assertEqual(
+            metrics.retained_files,
+            2,
+        )
+
+        self.assertEqual(
+            metrics.filtered_files,
+            1,
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

application/tests/harvester_test/fixtures/duplicate_include_paths.yaml

@@ -0,0 +1,22 @@
+repositories:
+  - id: duplicate-includes
+    type: github
+    enabled: true
+
+    owner: OWASP
+    repo: ASVS
+    branch: master
+
+    paths:
+      include:
+        - "docs/**/*.md"
+        - "docs/**/*.md"
+
+    chunking:
+      strategy: markdown_heading
+      max_tokens: 1200
+      overlap_tokens: 100
+
+    polling:
+      mode: incremental
+      interval_minutes: 60