Skip to content

Split LiteCoder terminal system prompts from tasks #293

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
neubig merged 12 commits into from
Jul 5, 2026
Merged

Split LiteCoder terminal system prompts from tasks #293

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
0bbfde8
Split LiteCoder terminal prompts
neubig Jul 5, 2026
6625315
Structure LiteCoder terminal completion JSON
neubig Jul 5, 2026
7e0956b
Populate LiteCoder finish messages
neubig Jul 5, 2026
7c9a5f5
Move LiteCoder cleanup to standardization
neubig Jul 5, 2026
e088040
Regenerate LiteCoder SFT samples
neubig Jul 5, 2026
b1bcbef
Guard empty LiteCoder finish messages
neubig Jul 5, 2026
dad2c6f
Reuse shared terminal standardization helpers
neubig Jul 5, 2026
4db4c48
Use ATIF standardization helpers in LiteCoder
neubig Jul 5, 2026
7dc9ac5
Share terminal empty-turn filtering
neubig Jul 5, 2026
657e14c
Share terminal trajectory normalization
neubig Jul 5, 2026
5b82401
Order terminal normalizer after dependencies
neubig Jul 5, 2026
47f1d97
Delegate terminal ATIF standardization entrypoints
neubig Jul 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion datasets/litecoder-terminal-sft/atif_to_std.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# ruff: noqa: E402, I001

from scripts.atif_to_std_common import main
from scripts.atif_to_std_common import terminal_main as main


if __name__ == "__main__":
Expand Down
1,308 changes: 1,041 additions & 267 deletions datasets/litecoder-terminal-sft/sample_atif.json
View file
Open in desktop

Large diffs are not rendered by default.

1,494 changes: 1,296 additions & 198 deletions datasets/litecoder-terminal-sft/sample_sft/openhands_sdk.json
View file
Open in desktop

Large diffs are not rendered by default.

332 changes: 244 additions & 88 deletions datasets/litecoder-terminal-sft/sample_sft/openhands_v0.json
View file
Open in desktop

Large diffs are not rendered by default.

1,349 changes: 1,082 additions & 267 deletions datasets/litecoder-terminal-sft/sample_std.json
View file
Open in desktop

Large diffs are not rendered by default.

38 changes: 1 addition & 37 deletions datasets/nemotron_terminal_corpus/atif_to_std.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,42 +1,6 @@
# ruff: noqa: E402, I001

import json
import sys

from schema.atif import ATIFTrajectory, Step, normalize_atif_trajectory
from scripts.atif_to_std_common import (
split_terminal_task_description_prompt,
standardize_tools,
structure_terminal_completion_step,
)
from scripts.raw_to_atif_common import renumber_steps, text_from_content


def is_empty_turn(step: Step) -> bool:
return (
step.observation is None
and not step.tool_calls
and not text_from_content(step.message).strip()
)


def normalize_nemotron(trajectory: ATIFTrajectory) -> ATIFTrajectory:
normalized = standardize_tools(normalize_atif_trajectory(trajectory))
split_terminal_task_description_prompt(normalized)
for step in normalized.steps:
structure_terminal_completion_step(step)
normalized.steps = renumber_steps(
[step for step in normalized.steps if not is_empty_turn(step)]
)
return normalized


def main(script_file: str | None = None) -> None: # noqa: ARG001
for line in sys.stdin:
if not line.strip():
continue
trajectory = ATIFTrajectory(**json.loads(line))
print(normalize_nemotron(trajectory).model_dump_json(exclude_none=True))
from scripts.atif_to_std_common import terminal_main as main


if __name__ == "__main__":
Expand Down
29 changes: 29 additions & 0 deletions scripts/atif_to_std_common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,14 @@ def structure_terminal_completion_step(step: Step) -> bool:
return True


def is_empty_message_only_step(step: Step) -> bool:
return (
step.observation is None
and not step.tool_calls
and not content_to_text(step.message).strip()
)


def standardize_tools(trajectory: ATIFTrajectory) -> ATIFTrajectory:
normalized = trajectory.model_copy(deep=True)
normalize_prompt_boilerplate(normalized)
Expand All @@ -185,6 +193,17 @@ def standardize_tools(trajectory: ATIFTrajectory) -> ATIFTrajectory:
return normalized


def normalize_terminal_trajectory(trajectory: ATIFTrajectory) -> ATIFTrajectory:
Comment thread
neubig marked this conversation as resolved.
normalized = standardize_tools(normalize_atif_trajectory(trajectory))
split_terminal_task_description_prompt(normalized)
for step in normalized.steps:
structure_terminal_completion_step(step)
normalized.steps = renumber_steps(
[step for step in normalized.steps if not is_empty_message_only_step(step)]
)
return normalized


def main(script_file: str | None = None) -> None: # noqa: ARG001
for line in sys.stdin:
line = line.strip()
Expand All @@ -195,5 +214,15 @@ def main(script_file: str | None = None) -> None: # noqa: ARG001
print(normalized.model_dump_json(exclude_none=True))


def terminal_main(script_file: str | None = None) -> None: # noqa: ARG001
for line in sys.stdin:
line = line.strip()
if not line:
continue
trajectory = ATIFTrajectory(**json.loads(line))
normalized = normalize_terminal_trajectory(trajectory)
print(normalized.model_dump_json(exclude_none=True))


if __name__ == "__main__":
main(__file__)
Loading