diff --git a/nimrod/__main__.py b/nimrod/__main__.py index d5604f12..9763c797 100644 --- a/nimrod/__main__.py +++ b/nimrod/__main__.py @@ -25,7 +25,23 @@ from nimrod.input_parsing.input_parser import CsvInputParser, JsonInputParser -def get_test_suite_generators(config: Dict[str, Any]) -> List[TestSuiteGenerator]: +def get_llm_test_suite_generators(config: Dict[str, Any]) -> List[TestSuiteGenerator]: + """ + Creates test suite generators for all available LLM models configured in api_params. + Each model gets its own generator instance to enable parallel processing and comparison. + """ + generators: List[TestSuiteGenerator] = list() + api_params = config.get('api_params', {}) + + for model_key, model_config in api_params.items(): + # Create a generator for each configured model + generator = OllamaTestSuiteGenerator(Java(), model_key, model_config) + generators.append(generator) + + return generators + + +def get_test_suite_generators(config: Dict[str, str]) -> List[TestSuiteGenerator]: config_generators = config.get( 'test_suite_generators', ['randoop', 'randoop-modified', 'evosuite', 'evosuite-differential', 'ollama', 'project']) generators: List[TestSuiteGenerator] = list() @@ -40,20 +56,15 @@ def get_test_suite_generators(config: Dict[str, Any]) -> List[TestSuiteGenerator if 'evosuite-differential' in config_generators: generators.append(EvosuiteDifferentialTestSuiteGenerator(Java())) if 'ollama' in config_generators: - # Create one generator instance for each configured model - api_params = config.get('api_params', {}) - if api_params: - for model_key, model_config in api_params.items(): - generators.append(OllamaTestSuiteGenerator(Java(), model_key, model_config)) - else: - generators.append(OllamaTestSuiteGenerator(Java())) + # Create one generator for each configured model + generators.extend(get_llm_test_suite_generators(config)) if 'project' in config_generators: generators.append(ProjectTestSuiteGenerator(Java())) return generators -def get_output_generators(config: Dict[str, Any]) -> List[OutputGenerator]: +def get_output_generators(config: Dict[str, str]) -> List[OutputGenerator]: config_generators = config.get( 'output_generators', ['behavior_changes', 'semantic_conflicts', 'test_suites']) generators: List[OutputGenerator] = list() @@ -69,7 +80,7 @@ def get_output_generators(config: Dict[str, Any]) -> List[OutputGenerator]: return generators -def parse_scenarios_from_input(config: Dict[str, Any]) -> List[MergeScenarioUnderAnalysis]: +def parse_scenarios_from_input(config: Dict[str, str]) -> List[MergeScenarioUnderAnalysis]: json_input = config.get('input_path', "") csv_input_path = config.get('path_hash_csv', "") diff --git a/nimrod/test_suite_generation/generators/llm_output_processor.py b/nimrod/test_suite_generation/generators/llm_output_processor.py index 9209ec2e..f7254c2e 100644 --- a/nimrod/test_suite_generation/generators/llm_output_processor.py +++ b/nimrod/test_suite_generation/generators/llm_output_processor.py @@ -5,7 +5,7 @@ class OutputSanitizationRule(ABC): """Abstract base class for output sanitization rules.""" - + @abstractmethod def apply(self, output: str) -> str: """Apply the sanitization rule to the output.""" @@ -14,14 +14,14 @@ def apply(self, output: str) -> str: class RemoveThinkTagsRule(OutputSanitizationRule): """Removes content between tags.""" - + def apply(self, output: str) -> str: return re.sub(r'.*?', '', output, flags=re.DOTALL) class ExtractCodeBlocksRule(OutputSanitizationRule): """Extracts content from code blocks (``` markers).""" - + def apply(self, output: str) -> str: matches = re.findall(r'```(?:\w+)?\n?(.*?)```', output, flags=re.DOTALL) return '\n'.join(matches).strip() @@ -29,17 +29,17 @@ def apply(self, output: str) -> str: class RemoveNumberedLinesRule(OutputSanitizationRule): """Removes lines starting with 'number. ' pattern.""" - + def apply(self, output: str) -> str: return re.sub(r"^\d+\.\s.*$", "", output, flags=re.MULTILINE) class ExtractFromAnnotationsRule(OutputSanitizationRule): """Keeps only content starting from the first annotation marker.""" - + def __init__(self, markers: Optional[List[str]] = None): self.markers = markers or ["@Before", "@BeforeClass", "@Test"] - + def apply(self, output: str) -> str: index = min( (output.find(marker) for marker in self.markers if marker in output), @@ -55,11 +55,11 @@ class LLMOutputProcessor: This class provides a flexible framework for cleaning LLM outputs by applying a series of sanitization rules in sequence. """ - + def __init__(self) -> None: self._rules: List[OutputSanitizationRule] = [] self._load_default_rules() - + def _load_default_rules(self) -> None: """Load the default set of sanitization rules.""" self._rules = [ @@ -68,19 +68,19 @@ def _load_default_rules(self) -> None: RemoveNumberedLinesRule(), ExtractFromAnnotationsRule() ] - + def add_rule(self, rule: OutputSanitizationRule) -> None: """Add a custom sanitization rule.""" self._rules.append(rule) - + def remove_rule(self, rule_type: type) -> None: """Remove all rules of the specified type.""" self._rules = [rule for rule in self._rules if not isinstance(rule, rule_type)] - + def clear_rules(self) -> None: """Remove all sanitization rules.""" self._rules.clear() - + def process(self, output: str) -> str: """ Process the LLM output by applying all sanitization rules in sequence. @@ -92,7 +92,7 @@ def process(self, output: str) -> str: The processed and sanitized output """ processed_output = output - + for rule in self._rules: try: processed_output = rule.apply(processed_output) @@ -100,9 +100,9 @@ def process(self, output: str) -> str: # Log the error but continue processing with other rules import logging logging.warning(f"Error applying rule {rule.__class__.__name__}: {e}") - + return processed_output - + def get_active_rules(self) -> List[str]: """Get the names of currently active rules.""" - return [rule.__class__.__name__ for rule in self._rules] + return [rule.__class__.__name__ for rule in self._rules] \ No newline at end of file diff --git a/nimrod/test_suite_generation/generators/ollama_test_suite_generator.py b/nimrod/test_suite_generation/generators/ollama_test_suite_generator.py index 43009207..b6021bd4 100644 --- a/nimrod/test_suite_generation/generators/ollama_test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/ollama_test_suite_generator.py @@ -1,7 +1,7 @@ import json import logging import os -import requests # type: ignore +import requests # type: ignore from typing import List, Dict, Union, Any, Optional import re @@ -79,7 +79,7 @@ def generate_output(self, messages: List[Dict[str, str]]) -> Dict[str, Any]: except Exception as e: logging.error(f"Error generating output: {e}") return {"error": "Output generation error", "total_duration": self.timeout_seconds * 1_000_000_000} - + class OllamaTestSuiteGenerator(TestSuiteGenerator): @@ -113,7 +113,7 @@ def generate_messages_list(self, method_info: Dict[str, str], full_class_name: s api.set_branch(branch) # Set the branch class_name = full_class_name.split('.')[-1] method_name = method_info.get("method_name", "") - + logging.debug(f"Generating messages for {class_name}.{method_name} using template: {self.prompt_template}") # Generates messages using the PromptManager @@ -123,7 +123,7 @@ def generate_messages_list(self, method_info: Dict[str, str], full_class_name: s branch=branch, template_name=self.prompt_template ) - + # Saves generated messages self.prompt_manager.save_generated_messages( messages_dict=messages_dict, @@ -131,7 +131,7 @@ def generate_messages_list(self, method_info: Dict[str, str], full_class_name: s class_name=class_name, method_name=method_name ) - + logging.info(f"Generated {len(messages_dict)} prompt variations for {class_name}.{method_name}") return messages_dict @@ -162,7 +162,7 @@ def _ensure_api_initialized(self) -> Api: model=model_params.get("model", "codellama:70b") ) return self.api - + def _get_api_instance(self) -> Api: """Returns a valid API instance, initializing it if necessary.""" return self._ensure_api_initialized() @@ -202,10 +202,10 @@ def save_output(self, test_template: str, output: str, dir: str, output_file_nam """Saves the output generated by the model to a file, replacing #TEST_METHODS# in the template.""" # Process and sanitize the LLM output using the configured processor processed_output = self.output_processor.process(output) - + # Replace the placeholder in the template with the processed content filled_template = test_template.replace("#TEST_METHODS#", processed_output) - + # Save to file llm_outputs_dir = os.path.join(dir, "llm_outputs") output_file_path = os.path.join(llm_outputs_dir, f"{output_file_name}.txt") @@ -247,12 +247,22 @@ def extract_class_info(self, source_code_path: str, full_method_name: str, full_ name: (identifier) @constructor_name) @constructor_declaration (method_declaration name: (identifier) @method_name) @method_def + (class_declaration) @nested_class (#eq? @method_name "{method_name}") (#eq? @constructor_name "{class_name}") ] ) (#eq? @class_name "{class_name}") ) + + ; Searches for methods in any class (including nested ones) + (method_declaration + name: (identifier) @any_method_name + (#eq? @any_method_name "{method_name}") + ) @any_method_def + + ; Searches for fields in any class (including nested ones) + (field_declaration) @any_field_declaration """ query = JAVA_LANGUAGE.query(query_text) cursor = QueryCursor(query) @@ -279,6 +289,34 @@ def extract_class_info(self, source_code_path: str, full_method_name: str, full_ class_constructors.append(captured_text) elif capture_name == "method_def": class_method = captured_text + elif capture_name == "any_method_def" and not class_method: + class_method = captured_text + elif capture_name == "any_field_declaration" and method_name in captured_text and not class_method: + class_method = captured_text + + if not class_method: + logging.debug(f"Method/field '{method_name}' not found with main query, trying broader field search") + + # More specific query for fields with exact name (including in nested classes) + field_query_text = f""" + (field_declaration + declarator: (variable_declarator + name: (identifier) @field_name + ) + (#eq? @field_name "{method_name}") + ) @target_field_declaration + """ + + field_query = JAVA_LANGUAGE.query(field_query_text) + field_cursor = QueryCursor(field_query) + field_captures_dict = field_cursor.captures(tree.root_node) + + for capture_name, nodes in field_captures_dict.items(): + for node in nodes: + if capture_name == "target_field_declaration": + class_method = self.extract_snippet(source_code, node.start_byte, node.end_byte) + logging.debug(f"Found field '{method_name}' as target: {class_method[:100]}...") + break return class_fields, class_constructors, class_method diff --git a/nimrod/tests/env-config.json b/nimrod/tests/env-config.json index 9934a146..3bf38229 100644 --- a/nimrod/tests/env-config.json +++ b/nimrod/tests/env-config.json @@ -20,4 +20,4 @@ "timeout_seconds": 300 } } -} +} \ No newline at end of file diff --git a/nimrod/tests/utils.py b/nimrod/tests/utils.py index d3726a3b..6b6dccdf 100644 --- a/nimrod/tests/utils.py +++ b/nimrod/tests/utils.py @@ -1,7 +1,10 @@ import logging +import logging.handlers import os import json import shutil +from datetime import datetime +from pathlib import Path from typing import Dict, Any PATH = os.path.dirname(os.path.abspath(__file__)) @@ -71,9 +74,13 @@ def calculator_sum_aor_1(): def setup_logging(): - config = get_config() - config_level = config.get('logger_level', 'INFO').upper() - level = logging._nameToLevel.get(config_level, logging.INFO) + try: + config = get_config() + config_level = config.get('logger_level', 'INFO').upper() + except (FileNotFoundError, json.JSONDecodeError, KeyError): + config_level = 'INFO' + + level = getattr(logging, config_level, logging.INFO) logger = logging.getLogger() logger.setLevel(level) @@ -81,19 +88,43 @@ def setup_logging(): if logger.hasHandlers(): logger.handlers.clear() - formatter = logging.Formatter( - '[%(asctime)s] %(levelname)s %(filename)s:%(lineno)d: %(message)s', + log_dir = Path("logs") + log_dir.mkdir(exist_ok=True) + + detailed_formatter = logging.Formatter( + '[%(asctime)s] %(levelname)-8s [%(name)s:%(lineno)d] %(funcName)s() - %(message)s', datefmt='%Y-%m-%d %H:%M:%S' ) + + console_formatter = logging.Formatter( + '[%(asctime)s] %(levelname)-8s - %(message)s', + datefmt='%H:%M:%S' + ) - file_handler = logging.FileHandler('logfile.log', mode='a') - file_handler.setFormatter(formatter) - - stream_handler = logging.StreamHandler() - stream_handler.setFormatter(formatter) - - logger.addHandler(file_handler) - logger.addHandler(stream_handler) + try: + main_log_file = log_dir / "smat_nimrod.log" + main_handler = logging.handlers.RotatingFileHandler( + main_log_file, + maxBytes=50 * 1024 * 1024, # 50MB + backupCount=5, + encoding='utf-8' + ) + main_handler.setLevel(logging.DEBUG) + main_handler.setFormatter(detailed_formatter) + logger.addHandler(main_handler) + except (OSError, PermissionError): + fallback_log = log_dir / f"smat_nimrod_{datetime.now().strftime('%Y%m%d')}.log" + fallback_handler = logging.FileHandler(fallback_log, mode='a', encoding='utf-8') + fallback_handler.setLevel(logging.DEBUG) + fallback_handler.setFormatter(detailed_formatter) + logger.addHandler(fallback_handler) + + console_handler = logging.StreamHandler() + console_handler.setLevel(level) + console_handler.setFormatter(console_formatter) + logger.addHandler(console_handler) + + logger.info("SMAT Logging initialized - Level: %s", config_level) def get_base_output_path() -> str: current_dir = os.getcwd() diff --git a/setup.py b/setup.py index c0854b14..99d02cbe 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ def readme(): 'argparse==1.4.0', 'beautifulsoup4==4.6.0', 'pygithub==2.5.0', - 'gitpython==3.1.41' + 'gitpython==3.1.50' ], test_suite='nose.collector', tests_require=[