Added persist to the DataSession model

.dockerignore

@@ -0,0 +1,45 @@
+# Skaffold derives its rebuild-watch list from this Dockerfile's COPY instructions,
+# and the final `COPY ./ ./` is the whole directory minus whatever is listed here.
+# Anything not excluded below will trigger an image rebuild when it changes.
+
+# Version control: .git/index, FETCH_HEAD, refs, etc. are rewritten by nearly
+# every git command (even `git status`), which caused spurious rebuilds.
+.git
+.gitignore
+.github
+
+# Python bytecode / test caches: regenerated on every test or manage.py run
+**/__pycache__
+**/*.py[cod]
+**/*$py.class
+*.egg-info
+.pytest_cache
+.mypy_cache
+.coverage
+.coverage.*
+htmlcov
+
+# Deploy config: changing a manifest should redeploy, not rebuild the image
+k8s
+skaffold.yaml
+local-cluster.yaml
+local-registry.yaml
+local-kubeconfig
+
+# Local dev environment, not used inside the image
+flake.nix
+flake.lock
+.devenv
+.venv
+venv
+.env
+.vscode
+.idea
+.pre-commit-config.yaml
+
+# Scratch / stray output
+tmp
+MagicMock
+dump.rdb
+*.log
+db.sqlite3

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.11-slim as base
+FROM python:3.11-slim AS base
 LABEL maintainer="jnation@lco.global"
 
 # use bash
@@ -34,15 +34,15 @@ ENV PYTHONUNBUFFERED=1 PYTHONFAULTHANDLER=1
 # usefull for running tests in docker container
 # this won't be included in the final image
 # e.g. docker build --target dev .
-FROM base as dev
+FROM base AS dev
 
 RUN pip install -r <(poetry export --dev)
 
 ENTRYPOINT ["bash"]
 
 
 # final image
-FROM base as prod
+FROM base AS prod
 
 # add a non-root user to run the app
 RUN useradd appuser

datalab/datalab_session/management/commands/delete_old_datasessions.py

@@ -0,0 +1,43 @@
+import logging
+from datetime import timedelta
+
+from django.core.management.base import BaseCommand
+from django.utils import timezone
+
+from datalab.datalab_session.models import DataOperation, DataSession
+
+log = logging.getLogger()
+log.setLevel(logging.INFO)
+
+DEFAULT_CUTOFF_DAYS = 30  # DataSessions with persist=False are deleted after this long
+DELETE_BATCH_SIZE = 500  # Number of DataSessions to delete per query, to bound memory and transaction size
+
+
+class Command(BaseCommand):
+    help = 'Deletes DataSessions with persist=False older than the cutoff. Meant to run daily from a CronJob'
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--cutoff-days', type=int, default=DEFAULT_CUTOFF_DAYS,
+            help=f'Delete non-persisted DataSessions created more than this many days ago (default: {DEFAULT_CUTOFF_DAYS})'
+        )
+
+    def handle(self, *args, **options):
+        ''' The DataOperations belonging to each session cascade away with the session. '''
+        cutoff = timezone.now() - timedelta(days=options['cutoff_days'])
+        expired = DataSession.objects.filter(persist=False, created__lt=cutoff)
+
+        sessions_deleted = 0
+        operations_deleted = 0
+        while True:
+            batch_pks = list(expired.values_list('pk', flat=True)[:DELETE_BATCH_SIZE])
+            if not batch_pks:
+                break
+            _, deleted_by_model = DataSession.objects.filter(pk__in=batch_pks).delete()
+            sessions_deleted += deleted_by_model.get(DataSession._meta.label, 0)
+            operations_deleted += deleted_by_model.get(DataOperation._meta.label, 0)
+
+        log.info(
+            f"delete_old_datasessions: deleted {sessions_deleted} DataSessions and "
+            f"{operations_deleted} DataOperations created before {cutoff.isoformat()}"
+        )

datalab/datalab_session/migrations/0004_datasession_persist.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.30 on 2026-09-17 16:32
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('datalab_session', '0003_alter_dataoperation_options_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='datasession',
+            name='persist',
+            field=models.BooleanField(default=False, verbose_name='Persist this datasession beyond 30 days'),
+        ),
+    ]

datalab/datalab_session/models.py

@@ -21,6 +21,10 @@ class Meta:
         help_text='List of input Data objects for this session in serialized format'
     )
 
+    persist = models.BooleanField(default=False,
+        verbose_name='Persist this datasession beyond 30 days'
+    )
+
     created = models.DateTimeField(
         auto_now_add=True,
         help_text='Time when this DataSession was created'

datalab/datalab_session/serializers.py

@@ -28,15 +28,31 @@ class Meta:
         model = DataSession
         fields = '__all__'
 
+    def __init__(self, *args, **kwargs):
+        ''' Accepts an optional `response_fields` list limiting the response to those fields.
+            Without it every field is returned, as normal.
+        '''
+        response_fields = kwargs.pop('response_fields', None)
+        self._response_fields = set(response_fields) if response_fields is not None else None
+        super().__init__(*args, **kwargs)
+
+    @property
+    def _readable_fields(self):
+        ''' Only response_fields are serialized if it's present.
+        '''
+        for field in super()._readable_fields:
+            if self._response_fields is None or field.field_name in self._response_fields:
+                yield field
+
     def update(self, instance, validated_data):
-      input_data = validated_data.get('input_data', [])
-
-			# Check for duplicates
-      existing_input_data = instance.input_data
-      new_input_data = []
-      for item in input_data:
-              if item not in existing_input_data:
-                      new_input_data.append(item)
-
-      validated_data['input_data'] = existing_input_data + new_input_data
-      return super().update(instance, validated_data)
+        if 'input_data' in validated_data:
+            input_data = validated_data.get('input_data', [])
+            # Check for duplicates
+            existing_input_data = instance.input_data
+            new_input_data = []
+            for item in input_data:
+                if item not in existing_input_data:
+                    new_input_data.append(item)
+
+            validated_data['input_data'] = existing_input_data + new_input_data
+        return super().update(instance, validated_data)

datalab/datalab_session/tests/test_api.py

@@ -1,6 +1,7 @@
 from rest_framework.test import APITestCase
 from mixer.backend.django import mixer
 from django.contrib.auth.models import User
+from django.core.cache import caches
 from django.urls import reverse
 from unittest import mock
 from types import SimpleNamespace
@@ -10,6 +11,142 @@
 from datalab.datalab_session.models import DataOperation, DataSession
 
 
+class TestDataSessionApi(APITestCase):
+    def setUp(self) -> None:
+        super().setUp()
+        self.user = mixer.blend(User)
+        self.client.force_login(self.user)
+        self.input_data = [
+            {'basename': 'fits_1', 'source': 'archive'},
+            {'basename': 'fits_2', 'source': 'archive'},
+        ]
+        self.session = mixer.blend(DataSession, user=self.user, persist=False, input_data=self.input_data)
+
+    def test_patch_sets_persist(self):
+        response = self.client.patch(reverse('api:datasessions-detail', args=(self.session.id,)), data={'persist': True})
+
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue(response.json()['persist'])
+        # Ensure input_data doesn't change
+        self.assertEqual(response.json()['input_data'], self.input_data)
+
+        self.session.refresh_from_db()
+        self.assertTrue(self.session.persist)
+
+    def test_patch_unsets_persist(self):
+        self.session.persist = True
+        self.session.save()
+
+        response = self.client.patch(reverse('api:datasessions-detail', args=(self.session.id,)), data={'persist': False})
+
+        self.assertEqual(response.status_code, 200)
+        self.assertFalse(response.json()['persist'])
+        # Ensure input_data doesn't change
+        self.assertEqual(response.json()['input_data'], self.input_data)
+
+        self.session.refresh_from_db()
+        self.assertFalse(self.session.persist)
+
+    def test_patching_persist_alone_leaves_operations_untouched(self):
+        operation = mixer.blend(DataOperation, session=self.session)
+
+        response = self.client.patch(reverse('api:datasessions-detail', args=(self.session.id,)), data={'persist': True})
+
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual([op['id'] for op in response.json()['operations']], [operation.id])
+        self.assertEqual(DataOperation.objects.filter(session=self.session).count(), 1)
+
+    def test_patching_input_data_appends_without_duplicating(self):
+        new_item = {'basename': 'fits_3', 'source': 'archive'}
+        data = {'input_data': [self.input_data[0], new_item]}
+
+        response = self.client.patch(reverse('api:datasessions-detail', args=(self.session.id,)), data=data, format='json')
+
+        self.assertEqual(response.status_code, 200)
+        self.session.refresh_from_db()
+        self.assertEqual(self.session.input_data, self.input_data + [new_item])
+
+    def test_cannot_patch_persist_on_another_users_session(self):
+        other_session = mixer.blend(DataSession, persist=False)
+
+        response = self.client.patch(reverse('api:datasessions-detail', args=(other_session.id,)), data={'persist': True})
+
+        self.assertEqual(response.status_code, 404)
+        other_session.refresh_from_db()
+        self.assertFalse(other_session.persist)
+
+    def test_patch_without_response_fields_returns_every_field(self):
+        response = self.client.patch(reverse('api:datasessions-detail', args=(self.session.id,)), data={'persist': True})
+
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(
+            set(response.json().keys()),
+            {'id', 'user', 'name', 'input_data', 'persist', 'created', 'accessed', 'modified', 'operations'}
+        )
+
+    def test_patch_with_response_fields_returns_only_those_fields(self):
+        url = reverse('api:datasessions-detail', args=(self.session.id,)) + '?response_fields=id,persist,modified'
+
+        response = self.client.patch(url, data={'persist': True})
+
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(set(response.json().keys()), {'id', 'persist', 'modified'})
+        self.assertTrue(response.json()['persist'])
+        self.session.refresh_from_db()
+        self.assertTrue(self.session.persist)
+
+    def test_response_fields_does_not_affect_what_is_written(self):
+        # persist is written even though the response only carries the id
+        url = reverse('api:datasessions-detail', args=(self.session.id,)) + '?response_fields=id'
+
+        response = self.client.patch(url, data={'persist': True})
+
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(set(response.json().keys()), {'id'})
+        self.session.refresh_from_db()
+        self.assertTrue(self.session.persist)
+
+    def test_response_fields_skips_serializing_operations(self):
+        mixer.blend(DataOperation, session=self.session, cache_key='abc')
+        url = reverse('api:datasessions-detail', args=(self.session.id,)) + '?response_fields=id,persist'
+
+        # Every operation reads status, progress, output and message from the cache, so a cache
+        # read here means the operations were serialized despite not being asked for
+        with mock.patch.object(caches['default'], 'get', side_effect=AssertionError('operations were serialized')):
+            response = self.client.patch(url, data={'persist': True})
+
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(set(response.json().keys()), {'id', 'persist'})
+
+    def test_response_fields_still_merges_input_data(self):
+        new_item = {'basename': 'fits_3', 'source': 'archive'}
+        url = reverse('api:datasessions-detail', args=(self.session.id,)) + '?response_fields=id,input_data'
+
+        response = self.client.patch(url, data={'input_data': [self.input_data[0], new_item]}, format='json')
+
+        self.assertEqual(response.status_code, 200)
+        # The response echoes the stored merge, not the two items that were sent
+        self.assertEqual(response.json()['input_data'], self.input_data + [new_item])
+        self.session.refresh_from_db()
+        self.assertEqual(self.session.input_data, self.input_data + [new_item])
+
+    def test_response_fields_ignores_unknown_field_names(self):
+        url = reverse('api:datasessions-detail', args=(self.session.id,)) + '?response_fields=id,bogus'
+
+        response = self.client.patch(url, data={'persist': True})
+
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(set(response.json().keys()), {'id'})
+
+    def test_response_fields_applies_to_get(self):
+        url = reverse('api:datasessions-detail', args=(self.session.id,)) + '?response_fields=id,name'
+
+        response = self.client.get(url)
+
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(set(response.json().keys()), {'id', 'name'})
+
+
 class TestOperationsApi(APITestCase):
     def setUp(self) -> None:
         super().setUp()