build: update dependencies

pyproject.toml

@@ -241,8 +241,20 @@ extend-exclude = ["config.py"]
 line-length = 120
 
 [tool.ruff.lint]
-extend-select = ["A", "C4", "INT", "ISC", "T20", "PIE", "Q", "RET", "SIM", "I", "N", "PERF", "W", "D", "F", "UP", "RUF"]
-ignore = ["D100", "D101", "D103", "D200", "D202", "D205", "D400", "D401", "F403", "F405", "F841", "N815", "PERF203", "RUF012"]
+extend-select = ["A", "C4", "D", "F", "I", "INT", "ISC", "N", "PERF", "PIE", "Q", "RET", "RUF", "SIM", "T20", "UP", "W"]
+ignore = [
+    "BLE001",  # blind-except: catching `Exception` is deliberate around external services and harvesters
+    "D401",    # non-imperative-mood: docstring wording is left to the author
+    "DTZ",     # flake8-datetimez: naive dates are intentional (embargo dates, sitemap, display formatting)
+    "RUF012",  # mutable-class-default: Invenio base classes rely on mutable class attributes
+    "TRY002",  # raise-vanilla-class: TODO replace the bare `Exception` raises with a SonarError hierarchy
+]
+
+[tool.ruff.lint.per-file-ignores]
+# Marshmallow and serializer schemas mirror the camelCase fields of the JSON schemas.
+"**/marshmallow/*.py" = ["N815"]
+"**/schema.py" = ["N815"]
+"**/schemas/*.py" = ["N815"]
 
 [tool.ruff.lint.pydocstyle]
 convention = "pep257"

sonar/heg/cli/harvest.py

@@ -71,7 +71,7 @@ def import_records(file, remove_file):
             file_path = path.join(target_directory, single_file)
 
             with open(file_path) as json_file:
-                for data in json_file.readlines():
+                for data in json_file:
                     data = json.loads(data)
                     try:
                         heg_record = HEGRecord(data)

sonar/heg/serializers/schemas/medline.py

@@ -42,7 +42,6 @@ def get_abstracts(self, obj):
 
     def get_subjects(self, obj):
         """Get subjects."""
-
         subjects = [{"label": {"language": obj["language"], "value": [item]}} for item in obj.get("keywords", [])]
 
         for item in obj.get("mesh_terms", []):

sonar/modules/babel_extractors.py

@@ -8,12 +8,14 @@
 KEY_VAL_REGEX = re.compile(r'"(.*?)"\s*:\s*"(.*?)"')
 
 
-def extract(fileobj, keys=["title"]):
+def extract(fileobj, keys=None):
     """Extract translation from a json file.
 
     :param fileobj: File object to extract translations from.
     :param keys: Properties to take into account.
     """
+    if keys is None:
+        keys = ["title"]
     translations = []
     line = 1
     for v in fileobj:

sonar/modules/cli/fixtures.py

@@ -169,11 +169,11 @@ def import_data(file, doc_type, random_files, with_file_support):
                     # Register record to DB
                     db_record = record_class.create(data=record, with_bucket=with_file_support)
                     # Add files
-                    for file in files:
-                        file_path = os.path.join(directory, file["path"])
+                    for record_file in files:
+                        file_path = os.path.join(directory, record_file["path"])
                         if os.path.isfile(file_path):
                             with open(file_path, "rb") as f:
-                                db_record.add_file(f.read(), file["key"])
+                                db_record.add_file(f.read(), record_file["key"])
                     if random_files:
                         data = extract_metadata({"metadata": db_record})
                         for i in range(1, randint(2, 5)):

sonar/modules/documents/cli/oai.py

@@ -20,6 +20,7 @@ def oai():
 @click.argument("pattern")
 @with_appcontext
 def create_set(code, name, pattern):
+    """Create an OAI set."""
     oaiset = OAISet(
         spec=code,
         name=name,

sonar/modules/documents/dnb.py

@@ -94,7 +94,6 @@ def get(cls, urn_code):
         """
         # Documentation: https://wiki.dnb.de/display/URNSERVDOK/URN-Service+API
         # https://wiki.dnb.de/display/URNSERVDOK/Beispiele%3A+URN-Verwaltung
-        answer = False
         try:
             response = requests.get(f"{cls.base_url()}/urn/{urn_code}", headers=cls.headers())
             if response.status_code != 200:

sonar/modules/documents/dojson/sru/model.py

@@ -297,8 +297,6 @@ def marc21_to_language_and_provision_activity_from_008(self, key, value):
 
     self["provisionActivity"] = provision_activity
 
-    return
-
 
 @overdo.over("title", "^245..")
 @utils.for_each_value
@@ -430,8 +428,6 @@ def marc21_to_extent_from_300(self, key, value):
     if value.get("e"):
         self["additionalMaterials"] = value["e"]
 
-    return
-
 
 @overdo.over("dissertation", "^502..")
 @utils.ignore_value
@@ -582,8 +578,6 @@ def marc21_to_provision_activity_from_264_1(self, key, value):
 
     self["provisionActivity"] = provision_activity
 
-    return
-
 
 @overdo.over("provisionActivity", "^264.(1|3)")
 @utils.ignore_value
@@ -605,8 +599,6 @@ def marc21_to_provision_activity_from_264_3(self, key, value):
     provision_activity.append(manufacture)
     self["provisionActivity"] = provision_activity
 
-    return
-
 
 @overdo.over("notes", "^(500|504|508|510|511|530|545|555)..")
 @utils.for_each_value

sonar/modules/documents/permissions.py

@@ -26,7 +26,6 @@ def list(cls, user, record=None):
         :param record: Record to check.
         :returns: True is action can be done.
         """
-
         # Documents are accessible in public view, but eventually filtered
         # later by organisation
         if request.args.get("view"):
@@ -114,12 +113,13 @@ def has_urn(cls, record):
         :param record: Record to check.
         :returns: True if action can be done.
         """
-        # Delete only documents with no URN or no registred URN
+        # A URN is never released, so documents holding one cannot be deleted,
+        # whether the URN is only reserved or already registered on the DNB.
         document = DocumentRecord.get_record_by_pid(record["pid"])
         if document:
             # check if document has urn
             try:
-                urn_identifier = PersistentIdentifier.get_by_object("urn", "rec", document.id)
+                PersistentIdentifier.get_by_object("urn", "rec", document.id)
             except PIDDoesNotExistError:
                 return False
 

sonar/modules/documents/serializers/json.py

@@ -5,8 +5,6 @@
 
 from datetime import datetime
 
-from flask import request
-
 from sonar.modules.collections.api import Record as CollectionRecord
 from sonar.modules.organisations.api import OrganisationRecord
 from sonar.modules.serializers import JSONSerializer as BasedJSONSerializer
@@ -18,8 +16,6 @@ class JSONSerializer(BasedJSONSerializer):
 
     def post_process_serialize_search(self, results, pid_fetcher):
         """Post process the search results."""
-        view = request.args.get("view")
-
         if results["aggregations"].get("year"):
             results["aggregations"]["year"]["type"] = "range"
             results["aggregations"]["year"]["config"] = {

sonar/modules/documents/urn.py

@@ -79,12 +79,12 @@ def _calculate_check_digit(cls, urn):
         # chars
         digit_sequence = ""
         for idx, element in enumerate(urn, 0):
-            digits = conversion_table[urn[idx].upper()]
+            digits = conversion_table[element.upper()]
             digit_sequence = digit_sequence + digits
         # calculate product sum
         product_sum = 0
         for idx, element in enumerate(digit_sequence, 0):
-            product_sum = product_sum + ((idx + 1) * int(digit_sequence[idx]))
+            product_sum = product_sum + ((idx + 1) * int(element))
         # read the last number of the digit sequence and calculate the quotient
         last_number = int(digit_sequence[-1])
         quotient = int(product_sum / last_number)

sonar/modules/organisations/api.py

@@ -113,7 +113,6 @@ def create(cls, data, id_=None, dbcommit=False, with_bucket=True, **kwargs):
         :param with_bucket: If True create a bucket for the organisation.
         :returns: The created record.
         """
-
         return super().create(data, id_=id_, dbcommit=dbcommit, with_bucket=with_bucket, **kwargs)
 
     @classmethod

sonar/modules/permissions.py

@@ -300,7 +300,7 @@ class FilesPermission(RecordPermission):
     ]
     delete_actions = ["object-delete", "object-delete-version", "multipart-delete"]
 
-    def __init__(self, record, func, user=None, pid=None, parent_record={}):
+    def __init__(self, record, func, user=None, pid=None, parent_record=None):
         """Initialize a file permission object.
 
         :param record: Record to check.
@@ -310,6 +310,8 @@ def __init__(self, record, func, user=None, pid=None, parent_record={}):
         instance.
         :param parent_record: the record related to the bucket.
         """
+        if parent_record is None:
+            parent_record = {}
         self.pid = pid
         self.parent_record = parent_record
         super().__init__(record, func, user)

sonar/modules/swisscovery/rest.py

@@ -55,7 +55,7 @@ def get_record():
 
     # Regular expression to remove the << and >> around a value in the title
     # Ex: <<La>> vie est belle => La vie est belle
-    pattern = re.compile("<<(.+)>>", re.S)
+    pattern = re.compile("<<(.+)>>", re.DOTALL)
     for title in record.get("title", []):
         for main_title in title.get("mainTitle", []):
             main_title["value"] = re.sub(pattern, r"\1", main_title["value"])

sonar/modules/users/api.py

@@ -334,5 +334,4 @@ def delete(self, record):
 
         :param record: Record instance.
         """
-
         return super().delete(record)

sonar/modules/users/extensions/remove_roles.py

@@ -11,5 +11,4 @@ class DeleteRolesExtension(RecordExtension):
 
     def post_delete(self, record, force=False):
         """Called after a record is deleted."""
-
         record.remove_roles()

sonar/modules/utils.py

@@ -11,7 +11,7 @@
 from flask import abort, current_app, g, request
 from invenio_i18n.selectors import get_locale
 from invenio_mail.api import TemplatedMessage
-from netaddr import IPAddress, IPGlob, IPNetwork, IPSet
+from netaddr import AddrFormatError, IPAddress, IPGlob, IPNetwork, IPSet
 from requests.adapters import HTTPAdapter
 from requests.packages.urllib3.util.retry import Retry
 from wand.color import Color
@@ -175,7 +175,7 @@ def is_ip_in_list(ip_address, addresses_list):
     :returns: True if given IP is in list.
     """
     if not isinstance(addresses_list, list):
-        raise Exception("Given parameter is not a list.")
+        raise TypeError("Given parameter is not a list.")
 
     ip_set = IPSet()
 
@@ -190,8 +190,8 @@ def is_ip_in_list(ip_address, addresses_list):
             # Simple IP
             else:
                 ip_set.add(IPAddress(ip_range))
-        except Exception:
-            pass
+        except AddrFormatError, ValueError:
+            continue
 
     try:
         return ip_address in ip_set
@@ -285,8 +285,8 @@ def get_ips_list(ranges):
             # Simple IP
             else:
                 ip_set.add(IPAddress(ip_range))
-        except Exception:
-            pass
+        except AddrFormatError, ValueError:
+            continue
 
     return [str(ip.cidr) for ip in ip_set.iter_cidrs()]
 

sonar/resources/projects/service.py

@@ -106,8 +106,10 @@ class ProjectsRecordServiceConfig(RecordServiceConfig):
 
 
 class ProjectServiceSchemaWrapper(ServiceSchemaWrapper):
+    """Schema wrapper injecting the organisation into project data."""
+
     def _get_organisation_pid(self, data):
-        """Get organisation PID from data"""
+        """Get organisation PID from data."""
         if org := data.get("metadata", {}).get("organisation"):
             return get_pid_from_ref_or_data(org)
         return None

sonar/suggestions/rest.py

@@ -41,7 +41,9 @@ def completion():
     try:
         service = sonar.service(resource)
         search = service.config.search.search_cls(index=resource)
-    except Exception as err:
+    except AttributeError:
+        # `sonar.service()` returns None for resources without a service,
+        # fall back on the REST endpoints.
         endpoints = current_app.config.get("RECORDS_REST_ENDPOINTS")
         for config in endpoints.values():
             if config.get("search_index") == resource:

sonar/theme/views.py

@@ -108,7 +108,7 @@ def profile(pid=None):
 @blueprint.route("/error")
 def error():
     """Error to generate exception for test purposes."""
-    raise Exception("this is an error for test purposes")
+    raise RuntimeError("this is an error for test purposes")
 
 
 @blueprint.route("/manage/")

tests/api/collections/test_collections_documents_facets.py

@@ -8,6 +8,7 @@
 
 
 def test_list(app, db, client, document, collection, superuser):
+    """Test collection facet on documents list."""
     document["collections"] = [{"$ref": f"https://sonar.ch/api/collections/{collection['pid']}"}]
     document.commit()
     db.session.commit()

tests/api/collections/test_collections_files_permissions.py

@@ -36,7 +36,6 @@ def test_update_delete(client, superuser, admin, moderator, submitter, user, col
 
 def test_read_metadata(client, superuser, admin, moderator, submitter, user, collection_with_file):
     """Test read files permissions."""
-
     users = [superuser, admin, moderator, submitter, user, None]
     url_files = url_for(
         "invenio_records_files.coll_bucket_api",
@@ -53,7 +52,6 @@ def test_read_metadata(client, superuser, admin, moderator, submitter, user, col
 
 def test_read_content(client, superuser, admin, moderator, submitter, user, collection_with_file):
     """Test read collections permissions."""
-
     file_name = "test1.pdf"
     users = [superuser, admin, moderator, submitter, user, None]
     url_file_content = url_for(

tests/api/deposits/test_deposits_files_permissions.py

@@ -36,7 +36,6 @@ def test_update_delete(client, superuser, admin, moderator, submitter, user, dep
 
 def test_read_metadata(client, superuser, admin, moderator, submitter, user, deposit):
     """Test read files permissions."""
-
     users = [superuser, admin, moderator, submitter, user, None]
     url_files = url_for("invenio_records_files.depo_bucket_api", pid_value=deposit.get("pid"))
     for u, status in zip(users, [200, 200, 200, 404, 404, 404]):
@@ -50,7 +49,6 @@ def test_read_metadata(client, superuser, admin, moderator, submitter, user, dep
 
 def test_read_content(client, superuser, admin, moderator, submitter, user, deposit):
     """Test read deposits permissions."""
-
     file_name = "main.pdf"
     users = [superuser, admin, moderator, submitter, user, None]
     url_file_content = url_for(

tests/api/documents/test_documents_ark.py

@@ -8,7 +8,6 @@
 
 def test_ark_query(db, client, organisation, document, search_clear):
     """Test ark search query."""
-
     # an empty query: the document should be in the results
     res = client.get(url_for("invenio_records_rest.doc_list", view="global"))
     assert res.status_code == 200

tests/api/documents/test_documents_files_permissions.py

@@ -38,7 +38,6 @@ def test_update_delete(client, superuser, admin, moderator, submitter, user, doc
 
 def test_read_metadata(client, superuser, admin, moderator, submitter, user, document_with_file):
     """Test read files permissions."""
-
     users = [superuser, admin, moderator, submitter, user, None]
     url_files = url_for("invenio_records_files.doc_bucket_api", pid_value=document_with_file.get("pid"))
     for u, status in zip(users, [200, 200, 200, 200, 200, 200]):
@@ -72,7 +71,6 @@ def test_read_content(
     document_with_file,
 ):
     """Test read documents permissions."""
-
     file_name = "test1.pdf"
     users = [superuser, admin, moderator, submitter, user, user_without_org, None]
     url_file_content = url_for(

tests/api/documents/test_documents_permissions.py

@@ -151,7 +151,6 @@ def test_create(client, document_json, superuser, admin, moderator, submitter, u
 
 def test_read(client, document, make_user, superuser, admin, moderator, submitter, user):
     """Test read documents permissions."""
-
     # Not logged
     res = client.get(url_for("invenio_records_rest.doc_item", pid_value=document["pid"]))
     assert res.status_code == 200
@@ -230,7 +229,6 @@ def test_update(
     make_subdivision,
 ):
     """Test update documents permissions."""
-
     headers = {"Content-Type": "application/json", "Accept": "application/json"}
 
     # Not logged
@@ -358,7 +356,6 @@ def test_delete(
     user,
 ):
     """Test delete documents permissions."""
-
     # Not logged
     res = client.delete(url_for("invenio_records_rest.doc_item", pid_value=document["pid"]))
     assert res.status_code == 401