From 781024e887515ed0f1f58a5aadc61bc59d49f6c9 Mon Sep 17 00:00:00 2001 From: Ernesto Ruge Date: Sun, 1 Nov 2026 07:42:59 +0100 Subject: [PATCH 1/2] new datex2 data sources --- README.md | 122 +++++++---- pyproject.toml | 8 + .../ocpi/connector_tariff_api_test.py | 20 +- .../public_api/ocpi/v2_2/tariff_api_test.py | 8 +- .../ocpi/v3_0/tariff_association_api_test.py | 4 +- .../repositories/location_repository_test.py | 4 +- .../bnetza/bnetza_import_api_test.py | 8 +- .../datex2/datex2_import_api_test.py | 16 +- .../server_rest_api/giroe/giroe_api_test.py | 4 +- .../server_rest_api/openapi_docs_test.py | 4 +- .../base_import_service_test.py | 20 +- .../data/datex2_giroe_realtime_reduced.json | 201 ++++++++++++++++++ .../datex2/enbw_datex2_import_service_test.py | 35 +++ .../ocpi/eaaze_pbw/eaaze_pbw_service_test.py | 8 +- tests/unit/common/unset_parameter_test.py | 12 +- .../multi_dataclass_validator_test.py | 4 +- .../validation/multi_dict_validator_test.py | 4 +- tests/unit/models/capability_bits_test.py | 20 +- .../models/parking_restriction_bits_test.py | 20 +- .../datex2/datex2_v3_5_input_test.py | 81 +++++++ .../datex2/datex2_v3_7_realtime_input_test.py | 81 +++++++ .../datex2/tariff_grouping_mixin_test.py | 36 +++- .../eliso/eliso_validators_test.py | 111 ++++++++++ webapp/public_api/ocpi/ocpi_redirect.py | 6 +- .../ocpi/v2_2/locations/rest_api.py | 4 +- .../import_services/datex2/__init__.py | 17 ++ .../import_services/datex2/v3_5/__init__.py | 17 ++ .../ah_beteiligung_datex2_import_service.py | 31 +++ .../datex2/v3_5/audi_datex2_import_service.py | 32 +++ .../v3_5/base_datex2_v3_5_import_service.py | 40 +++- .../v3_5/eaaze_datex2_import_service.py | 32 +++ .../v3_5/enerando_datex2_import_service.py | 31 +++ .../v3_5/eulektro_datex2_import_service.py | 31 +++ .../v3_5/ev_price_datex2_import_service.py | 32 +++ .../datex2/v3_5/ewe_datex2_import_service.py | 32 +++ .../v3_5/flavia_datex2_import_service.py | 31 +++ .../v3_5/gp_joule_datex2_import_service.py | 32 +++ .../v3_5/inselwerke_datex2_import_service.py | 31 +++ .../v3_5/kul_rudnick_datex2_import_service.py | 32 +++ .../v3_5/ladesonne_datex2_import_service.py | 31 +++ .../r_m_beteiligung_datex2_import_service.py | 31 +++ .../datex2/v3_5/road_datex2_import_service.py | 31 +++ ...enstrom_saarpfalz_datex2_import_service.py | 31 +++ .../datex2/v3_5/ulm_datex2_import_service.py | 32 +++ .../v3_5/virta_datex2_import_service.py | 32 +++ .../v3_5/volkswagen_datex2_import_service.py | 4 +- .../v3_7/base_datex2_v3_7_import_service.py | 79 ++++--- .../import_services/eliso/__init__.py | 19 ++ .../import_services/eliso/eliso_service.py | 158 ++++++++++++++ .../import_services/eliso/eliso_validators.py | 176 +++++++++++++++ .../import_services/import_services.py | 36 ++++ webapp/shared/datex2/models.py | 30 ++- 52 files changed, 1828 insertions(+), 124 deletions(-) create mode 100644 tests/integration/services/import_services/datex2/data/datex2_giroe_realtime_reduced.json create mode 100644 tests/unit/services/import_services/datex2/datex2_v3_5_input_test.py create mode 100644 tests/unit/services/import_services/datex2/datex2_v3_7_realtime_input_test.py create mode 100644 tests/unit/services/import_services/eliso/eliso_validators_test.py create mode 100644 webapp/services/import_services/datex2/v3_5/ah_beteiligung_datex2_import_service.py create mode 100644 webapp/services/import_services/datex2/v3_5/audi_datex2_import_service.py create mode 100644 webapp/services/import_services/datex2/v3_5/eaaze_datex2_import_service.py create mode 100644 webapp/services/import_services/datex2/v3_5/enerando_datex2_import_service.py create mode 100644 webapp/services/import_services/datex2/v3_5/eulektro_datex2_import_service.py create mode 100644 webapp/services/import_services/datex2/v3_5/ev_price_datex2_import_service.py create mode 100644 webapp/services/import_services/datex2/v3_5/ewe_datex2_import_service.py create mode 100644 webapp/services/import_services/datex2/v3_5/flavia_datex2_import_service.py create mode 100644 webapp/services/import_services/datex2/v3_5/gp_joule_datex2_import_service.py create mode 100644 webapp/services/import_services/datex2/v3_5/inselwerke_datex2_import_service.py create mode 100644 webapp/services/import_services/datex2/v3_5/kul_rudnick_datex2_import_service.py create mode 100644 webapp/services/import_services/datex2/v3_5/ladesonne_datex2_import_service.py create mode 100644 webapp/services/import_services/datex2/v3_5/r_m_beteiligung_datex2_import_service.py create mode 100644 webapp/services/import_services/datex2/v3_5/road_datex2_import_service.py create mode 100644 webapp/services/import_services/datex2/v3_5/sonnenstrom_saarpfalz_datex2_import_service.py create mode 100644 webapp/services/import_services/datex2/v3_5/ulm_datex2_import_service.py create mode 100644 webapp/services/import_services/datex2/v3_5/virta_datex2_import_service.py create mode 100644 webapp/services/import_services/eliso/__init__.py create mode 100644 webapp/services/import_services/eliso/eliso_service.py create mode 100644 webapp/services/import_services/eliso/eliso_validators.py diff --git a/README.md b/README.md index 70aab853..95e43522 100644 --- a/README.md +++ b/README.md @@ -5,45 +5,89 @@ OCPI-style JSON-API and a vector tile server. ## Data sources -| name | uid | realtime | credentials | comment | -|-------------------------------------|-------------------------|----------|-------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Bundesnetzagentur: API | bnetza_api | false | false | Additional config `ignore_operators:: list[str]` is supported, which will ignore given operators during import. Set to weekly download, as it does not change so often. | -| Bundesnetzagentur: Excel | bnetza_excel | false | false | | -| chargecloud: Stadtwerke Pforzheim | chargecloud_pforzheim | true | false | | -| chargecloud: Stadtwerke Stuttgart | chargecloud_stuttgart | true | false | | -| chargecloud: Stadtwerke Tübingen | chargecloud_tuebingen | true | false | | -| chargecloud: Stadtwerke Ludwigsburg | chargecloud_ludwigsburg | true | true | | -| Datex2: Ampeco | datex2_ampeco | true | true | Via Mobilithek, currently disfunctional | -| Datex2: Chargecloud | datex2_chargecloud | true | true | Via Mobilithek | -| Datex2: e-clearing.net | datex2_e_clearing_net | true | true | Via Mobilithek | -| Datex2: EcoMovement | datex2_ecomovement | true | true | Via Mobilithek | -| Datex2: Elu Mobility | datex2_elu_mobility | true | true | Via Mobilithek, currently disfunctional | -| Datex2: EnBW | datex2_enbw | true | true | Via Mobilithek | -| Datex2: enio | datex2_enio | true | true | Via Mobilithek | -| Datex2: eRound | datex2_eround | true | true | Via Mobilithek | -| Datex2: Grid&Co | datex2_gridandco | true | true | Via Mobilithek, currently disfunctional | -| Datex2: Inergia | datex2_inergia | true | true | Via Mobilithek | -| Datex2: ladebusiness | datex2_ladebusiness | true | true | Via Mobilithek, currently disfunctional | -| Datex2: Lichtblick | datex2_lichtblick | true | true | Via Mobilithek | -| Datex2: Midorion | datex2_midorion | true | true | Via Mobilithek | -| Datex2: Monta | datex2_monta | true | true | Via Mobilithek, currently disfunctional | -| Datex2: msu | datex2_msu | true | true | Via Mobilithek | -| Datex2: Pump | datex2_pump | false | true | Via Mobilithek, currently disfunctional | -| Datex2: Qwello | datex2_qwello | true | true | Via Mobilithek | -| Datex2: Smatrics | datex2_smatrics | true | true | Via Mobilithek, currently disfunctional | -| Datex2: Taubert Consulting | datex2_taubert | true | true | Via Mobilithek | -| Datex2: Tesla | datex2_tesla | true | true | Via Mobilithek | -| Datex2: Vaylens | datex2_vaylens | true | true | Via Mobilithek, currently disfunctional | -| Datex2: Volkswagen Charging Group | datex2_volkswagen | true | true | Via Mobilithek | -| Datex2: Wirelane | datex2_wirelane | true | true | Via Mobilithek | -| PBW | eaaze_pbw | true | true | | -| Giro-e | giroe | true | true | | -| Heilbronn Heckarbogen | heilbronn_neckarbogen | true | true | | -| Lichtblick | lichtblick | true | true | Currently dysfunctional | -| OCHP: Albwerk | ochp_albwerk | true | true | | -| OCHP: Ladenetz | ochp_ladenetz | true | true | | -| OCPI: Stadtnavi | ocpi_stadtnavi | true | false | | -| OpenData Swiss | opendata_swiss | true | false | | +Mobilithek sources were last validated on 2026-10-28 with `dev/mobilithek_helper/validate_datex2.py`. DATEX II data +has to be valid against the DATEX II schema, otherwise the import fails as a whole. See +[Mobilithek helper scripts](#mobilithek-helper-scripts) for details. + +| name | uid | realtime | credentials | comment | +|----------------------------------------|------------------------------|----------|-------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Bundesnetzagentur: API | bnetza_api | false | false | Additional config `ignore_operators:: list[str]` is supported, which will ignore given operators during import. Set to weekly download, as it does not change so often. | +| Bundesnetzagentur: Excel | bnetza_excel | false | false | | +| chargecloud: Stadtwerke Ludwigsburg | chargecloud_ludwigsburg | true | true | | +| chargecloud: Stadtwerke Pforzheim | chargecloud_pforzheim | true | false | | +| chargecloud: Stadtwerke Stuttgart | chargecloud_stuttgart | true | false | | +| chargecloud: Stadtwerke Tübingen | chargecloud_tuebingen | true | false | | +| Datex2: AH Beteiligungsgesellschaft | datex2_ah_beteiligung | true | true | | +| Datex2: Ampeco | datex2_ampeco | true | true | Static data is schema-invalid (organisation names and identifiers, operating hours) | +| Datex2: Audi charging hub | datex2_audi | true | true | Static data is schema-invalid (energy rates without `lastUpdated`), realtime data is empty | +| Datex2: Chargecloud | datex2_chargecloud | true | true | | +| Datex2: e-clearing.net | datex2_e_clearing_net | true | true | | +| Datex2: EAAZE | datex2_eaaze | true | true | | +| Datex2: EcoMovement | datex2_ecomovement | true | true | | +| Datex2: Elu Mobility | datex2_elu_mobility | true | true | Static data is schema-invalid (`pointLocation` instead of `locPointLocation`) | +| Datex2: EnBW | datex2_enbw | true | true | | +| Datex2: ENERANDO | datex2_enerando | true | true | | +| Datex2: enio | datex2_enio | true | true | | +| Datex2: eRound | datex2_eround | true | true | | +| Datex2: Eulektro | datex2_eulektro | true | true | | +| Datex2: EV Price | datex2_ev_price | true | true | | +| Datex2: EWE | datex2_ewe | true | true | | +| Datex2: Flavia | datex2_flavia | true | true | | +| Datex2: GLS Mobility | datex2_giroe | true | true | Realtime data is a delta pull and was empty on 2026-10-29, so EVSE matching is unverified | +| Datex2: GP JOULE Connect | datex2_gp_joule | true | true | | +| Datex2: Grid&Co | datex2_gridandco | true | true | Static data is schema-invalid (external identifier without identifier) | +| Datex2: Inergia | datex2_inergia | true | true | Via Mobilithek | +| Datex2: Inselwerke | datex2_inselwerke | true | true | | +| Datex2: K.u.L. Rudnick | datex2_kul_rudnick | false | true | No data published | +| Datex2: ladebusiness | datex2_ladebusiness | true | true | Data offer was withdrawn | +| Datex2: Ladesonne | datex2_ladesonne | true | true | | +| Datex2: Lichtblick | datex2_lichtblick | true | true | | +| Datex2: Midorion | datex2_midorion | true | true | | +| Datex2: Monta | datex2_monta | true | true | Static data is schema-invalid (times without seconds and offset) | +| Datex2: msu | datex2_msu | true | true | | +| Datex2: Pump | datex2_pump | false | true | Static data is schema-invalid (`versionG` at energy rates) | +| Datex2: Qwello | datex2_qwello | true | true | | +| Datex2: R & M Beteiligungsgesellschaft | datex2_r_m_beteiligung | true | true | | +| Datex2: Road | datex2_road | true | true | | +| Datex2: Smatrics | datex2_smatrics | true | true | | +| Datex2: Sonnenstrom Saarpfalz | datex2_sonnenstrom_saarpfalz | true | true | | +| Datex2: Taubert Consulting | datex2_taubert | true | true | | +| Datex2: Tesla | datex2_tesla | true | true | | +| Datex2: Ulm | datex2_ulm | true | true | | +| Datex2: Vaylens | datex2_vaylens | true | true | Static data is schema-invalid (times without seconds and offset) | +| Datex2: Virta | datex2_virta | true | true | | +| Datex2: Volkswagen Charging Group | datex2_volkswagen | true | true | Realtime data is outdated since 2026-10-02 | +| Datex2: Wirelane | datex2_wirelane | true | true | | +| eliso | eliso | true | true | Custom JSON format instead of DATEX II | +| Giro-e | giroe | true | true | | +| Heilbronn Heckarbogen | heilbronn_neckarbogen | true | true | | +| Lichtblick | lichtblick | true | true | Currently dysfunctional | +| OCHP: Albwerk | ochp_albwerk | true | true | | +| OCHP: Ladenetz | ochp_ladenetz | true | true | | +| OCPI: Stadtnavi | ocpi_stadtnavi | true | false | | +| OpenData Swiss | opendata_swiss | true | false | | +| PBW | eaaze_pbw | true | true | | + + +### Mobilithek helper scripts + +`dev/mobilithek_helper` contains scripts to manage the Mobilithek subscriptions of the charging data sources. Managing +subscriptions needs a user login, so the scripts read `MOBILITHEK_USER` and `MOBILITHEK_PASSWORD` from `.env`. + +```bash +python dev/mobilithek_helper/list_charging_offers.py # all charging data offers with subscription and import service +python dev/mobilithek_helper/list_subscriptions.py # all subscriptions of the account +python dev/mobilithek_helper/subscribe.py [ ...] +python dev/mobilithek_helper/unsubscribe.py [ ...] +``` + +`validate_datex2.py` validates the data of a subscription against the DATEX II v3.5 and v3.7 import services without +touching the database. It shows which base class fits, why an import fails and how many realtime EVSE statuses match +static EVSEs. It needs the webapp and the Mobilithek machine certificate in `files/`, so it runs in the flask container: + +```bash +docker compose run --rm flask python dev/mobilithek_helper/validate_datex2.py [] +``` ### Duplicate matching diff --git a/pyproject.toml b/pyproject.toml index 701747f4..379a2209 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -166,6 +166,14 @@ line-length = 120 # Don't require __init__.py files "INP", ] +"dev/*" = [ + # Don't require __init__.py files + "INP", + # Allow print in helper scripts + "T201", + # Token URLs are no passwords + "S105", +] [tool.ruff.lint.flake8-quotes] inline-quotes = "single" diff --git a/tests/integration/public_api/ocpi/connector_tariff_api_test.py b/tests/integration/public_api/ocpi/connector_tariff_api_test.py index f9790def..f0a8b29d 100644 --- a/tests/integration/public_api/ocpi/connector_tariff_api_test.py +++ b/tests/integration/public_api/ocpi/connector_tariff_api_test.py @@ -36,7 +36,9 @@ def test_connector_outputs_tariff_uids_of_its_evse( db: SQLAlchemy, public_test_client: OpenApiFlaskClient, ) -> None: - """Sources publish tariffs per EVSE, so the connectors of an EVSE output its tariff uids.""" + """ + Sources publish tariffs per EVSE, so the connectors of an EVSE output its tariff uids. + """ tariff = get_tariff_1() db.session.add(tariff) db.session.flush() @@ -56,7 +58,9 @@ def test_connector_outputs_all_tariff_uids_of_its_evse( db: SQLAlchemy, public_test_client: OpenApiFlaskClient, ) -> None: - """An EVSE offered under several tariffs passes all of their uids to its connectors.""" + """ + An EVSE offered under several tariffs passes all of their uids to its connectors. + """ first_tariff = get_tariff_1() second_tariff = get_tariff_2() db.session.add_all([first_tariff, second_tariff]) @@ -106,7 +110,9 @@ def test_connector_tariff_uids_take_precedence_over_the_evse_ones( db: SQLAlchemy, public_test_client: OpenApiFlaskClient, ) -> None: - """A connector carrying its own tariffs does not inherit the ones of its EVSE.""" + """ + A connector carrying its own tariffs does not inherit the ones of its EVSE. + """ evse_tariff = get_tariff_1() connector_tariff = get_tariff_2() db.session.add_all([evse_tariff, connector_tariff]) @@ -128,7 +134,9 @@ def test_connector_without_tariffs_omits_tariff_ids( db: SQLAlchemy, public_test_client: OpenApiFlaskClient, ) -> None: - """tariff_ids is optional in OCPI, so a connector without any tariff leaves it out.""" + """ + tariff_ids is optional in OCPI, so a connector without any tariff leaves it out. + """ db.session.add(get_location_1(evses=[get_full_evse_1()], operator=get_business_1())) db.session.commit() @@ -142,7 +150,9 @@ def test_evse_endpoint_outputs_connector_tariff_uids( db: SQLAlchemy, public_test_client: OpenApiFlaskClient, ) -> None: - """The connectors nested into an EVSE response carry the tariff uids too.""" + """ + The connectors nested into an EVSE response carry the tariff uids too. + """ tariff = get_tariff_1() db.session.add(tariff) db.session.flush() diff --git a/tests/integration/public_api/ocpi/v2_2/tariff_api_test.py b/tests/integration/public_api/ocpi/v2_2/tariff_api_test.py index 2a41d35f..571c3403 100644 --- a/tests/integration/public_api/ocpi/v2_2/tariff_api_test.py +++ b/tests/integration/public_api/ocpi/v2_2/tariff_api_test.py @@ -145,7 +145,9 @@ def test_get_ocpi_22_tariff_merges_tariff_data( db: SQLAlchemy, public_test_client: OpenApiFlaskClient, ) -> None: - """Two associations pointing to the same tariff produce two OCPI 2.2 tariffs with same pricing but different types.""" + """ + Two associations pointing to the same tariff produce two OCPI 2.2 tariffs with same pricing but different types. + """ tariff = get_tariff_1() tariff.elements = [ TariffElement( @@ -178,7 +180,9 @@ def test_get_ocpi_22_tariffs_does_not_load_evses_and_connectors( db: SQLAlchemy, public_test_client: OpenApiFlaskClient, ) -> None: - """OCPI 2.2 tariffs do not render the EVSEs and connectors of a tariff association, so they are not loaded.""" + """ + OCPI 2.2 tariffs do not render the EVSEs and connectors of a tariff association, so they are not loaded. + """ tariff = get_tariff_1() db.session.add(tariff) db.session.flush() diff --git a/tests/integration/public_api/ocpi/v3_0/tariff_association_api_test.py b/tests/integration/public_api/ocpi/v3_0/tariff_association_api_test.py index 5a0c4eef..2ae138d0 100644 --- a/tests/integration/public_api/ocpi/v3_0/tariff_association_api_test.py +++ b/tests/integration/public_api/ocpi/v3_0/tariff_association_api_test.py @@ -130,7 +130,9 @@ def test_get_ocpi_30_tariff_associations_loads_just_evse_ids( db: SQLAlchemy, public_test_client: OpenApiFlaskClient, ) -> None: - """Tariff associations render just the ids of their EVSEs, so the EVSEs are not loaded completely.""" + """ + Tariff associations render just the ids of their EVSEs, so the EVSEs are not loaded completely. + """ tariff = get_tariff_1() db.session.add(tariff) db.session.flush() diff --git a/tests/integration/repositories/location_repository_test.py b/tests/integration/repositories/location_repository_test.py index 8b3d6824..22d9127e 100644 --- a/tests/integration/repositories/location_repository_test.py +++ b/tests/integration/repositories/location_repository_test.py @@ -26,7 +26,9 @@ def test_fetch_location_by_uid_respects_source(db: SQLAlchemy) -> None: - """Different sources can use the same location uid, so the source has to be part of the lookup.""" + """ + Different sources can use the same location uid, so the source has to be part of the lookup. + """ db.session.add_all([ get_location_1(source=SOURCE_UID_1, name='Location of source 1'), get_location_1(source=SOURCE_UID_2, name='Location of source 2'), diff --git a/tests/integration/server_rest_api/bnetza/bnetza_import_api_test.py b/tests/integration/server_rest_api/bnetza/bnetza_import_api_test.py index 87b352c9..b4dc21ff 100644 --- a/tests/integration/server_rest_api/bnetza/bnetza_import_api_test.py +++ b/tests/integration/server_rest_api/bnetza/bnetza_import_api_test.py @@ -29,14 +29,18 @@ @pytest.fixture def stub_celery_delay(): - """Stub out CeleryHelper.delay so the bnetza import task is not actually enqueued.""" + """ + Stub out CeleryHelper.delay so the bnetza import task is not actually enqueued. + """ with patch.object(dependencies.get_celery_helper(), 'delay') as mock_delay: yield mock_delay @pytest.fixture def isolated_bnetza_dir(tmp_path: Path): - """Redirect BNETZA_IMPORT_DIR to a per-test temp directory to avoid polluting `data/`.""" + """ + Redirect BNETZA_IMPORT_DIR to a per-test temp directory to avoid polluting `data/`. + """ config = dependencies.get_config_helper().get_config() original = config.get('BNETZA_IMPORT_DIR') config['BNETZA_IMPORT_DIR'] = str(tmp_path) diff --git a/tests/integration/server_rest_api/datex2/datex2_import_api_test.py b/tests/integration/server_rest_api/datex2/datex2_import_api_test.py index f2b5ae84..da69507d 100644 --- a/tests/integration/server_rest_api/datex2/datex2_import_api_test.py +++ b/tests/integration/server_rest_api/datex2/datex2_import_api_test.py @@ -60,7 +60,9 @@ def _import_static_data(requests_mock: Mocker) -> None: def _fetch_realtime_data_updated_at(db: SQLAlchemy) -> datetime | None: - """Return the source's ``realtime_data_updated_at`` as an aware UTC datetime.""" + """ + Return the source's ``realtime_data_updated_at`` as an aware UTC datetime. + """ db.session.expire_all() source = db.session.query(Source).filter(Source.uid == SOURCE_UID).first() if source is None or source.realtime_data_updated_at is None: @@ -74,7 +76,9 @@ def _fetch_realtime_data_updated_at(db: SQLAlchemy) -> datetime | None: @pytest.fixture def isolated_datex2_dir(tmp_path: Path): - """Redirect DATEX2_IMPORT_DIR to a per-test temp directory.""" + """ + Redirect DATEX2_IMPORT_DIR to a per-test temp directory. + """ config = dependencies.get_config_helper().get_config() original = config.get('DATEX2_IMPORT_DIR') config['DATEX2_IMPORT_DIR'] = str(tmp_path) @@ -100,7 +104,9 @@ def run_task_synchronously(task, *args, **kwargs): @pytest.fixture def stubbed_celery_delay(): - """Replace ``CeleryHelper.delay`` with a no-op mock for tests that don't need the task to run.""" + """ + Replace ``CeleryHelper.delay`` with a no-op mock for tests that don't need the task to run. + """ with patch.object(dependencies.get_celery_helper(), 'delay') as mock_delay: yield mock_delay @@ -430,7 +436,9 @@ def test_push_realtime_v35_delta_push_updates_realtime_data_updated_at( isolated_datex2_dir: Path, stubbed_celery_delay, ) -> None: - """A synchronously applied deltaPush must update ``realtime_data_updated_at`` as well.""" + """ + A synchronously applied deltaPush must update ``realtime_data_updated_at`` as well. + """ _import_static_data(requests_mock) realtime_data = _load_test_data('datex2_enbw_realtime_reduced.json') diff --git a/tests/integration/server_rest_api/giroe/giroe_api_test.py b/tests/integration/server_rest_api/giroe/giroe_api_test.py index 79783d32..4cb723ba 100644 --- a/tests/integration/server_rest_api/giroe/giroe_api_test.py +++ b/tests/integration/server_rest_api/giroe/giroe_api_test.py @@ -60,7 +60,9 @@ def _giroe_connector_uid(connector_id: int) -> str: def _valid_location_payload(location_id: int, *, public: bool = True) -> dict: - """Build a minimal-but-complete giroe Location payload accepted by LocationInput.""" + """ + Build a minimal-but-complete giroe Location payload accepted by LocationInput. + """ now = datetime.now(timezone.utc).isoformat() return { 'id': location_id, diff --git a/tests/integration/server_rest_api/openapi_docs_test.py b/tests/integration/server_rest_api/openapi_docs_test.py index fe246fe5..91d45cfd 100644 --- a/tests/integration/server_rest_api/openapi_docs_test.py +++ b/tests/integration/server_rest_api/openapi_docs_test.py @@ -56,7 +56,9 @@ def test_server_openapi_json_renders(test_client: OpenApiFlaskClient) -> None: def test_server_openapi_html_renders(test_client: OpenApiFlaskClient) -> None: - """The Stoplight-Elements HTML wrapper page must load without errors.""" + """ + The Stoplight-Elements HTML wrapper page must load without errors. + """ response = test_client.get('/documentation/server.html') assert response.status_code == HTTPStatus.OK diff --git a/tests/integration/services/import_services/base_import_service_test.py b/tests/integration/services/import_services/base_import_service_test.py index 5d3a2701..c1b959c8 100644 --- a/tests/integration/services/import_services/base_import_service_test.py +++ b/tests/integration/services/import_services/base_import_service_test.py @@ -498,7 +498,9 @@ def test_save_location_updates_replaces_per_evse_associations_with_a_shared_one( def _create_location_update_with_two_evses(*, tariff_association_uids: list[str]) -> LocationUpdate: - """One location with two EVSEs, each pointing at the tariff association of the given uid.""" + """ + One location with two EVSEs, each pointing at the tariff association of the given uid. + """ tariff_update = TariffUpdate( uid='tariff1', source='test_source_uid', @@ -631,7 +633,9 @@ def _create_location_update_with_tariff( def test_save_location_updates_deletes_tariffs_with_new_uid(db, testing_import_service: BaseImportService): - """A tariff which the source does not send anymore is deleted instead of staying behind unreferenced.""" + """ + A tariff which the source does not send anymore is deleted instead of staying behind unreferenced. + """ testing_import_service.save_location_updates([_create_location_update_with_tariff(tariff_uid='tariff1')]) testing_import_service.save_location_updates([_create_location_update_with_tariff(tariff_uid='tariff2')]) @@ -647,7 +651,9 @@ def test_save_location_updates_deletes_tariff_associations_of_kept_tariff( db, testing_import_service: BaseImportService, ): - """The tariff survives its renamed association, but the old association does not.""" + """ + The tariff survives its renamed association, but the old association does not. + """ testing_import_service.save_location_updates([_create_location_update_with_tariff(tariff_association_uid='ta1')]) testing_import_service.save_location_updates([_create_location_update_with_tariff(tariff_association_uid='ta2')]) @@ -656,7 +662,9 @@ def test_save_location_updates_deletes_tariff_associations_of_kept_tariff( def test_save_location_updates_deletes_tariffs_of_deleted_locations(db, testing_import_service: BaseImportService): - """Tariffs do not belong to a location, so a vanished location has to take its tariffs with it.""" + """ + Tariffs do not belong to a location, so a vanished location has to take its tariffs with it. + """ testing_import_service.save_location_updates([ _create_location_update_with_tariff(location_uid='loc1', tariff_association_uid='ta1', tariff_uid='tariff1'), _create_location_update_with_tariff(location_uid='loc2', tariff_association_uid='ta2', tariff_uid='tariff2'), @@ -674,7 +682,9 @@ def test_save_location_updates_deletes_tariffs_of_deleted_locations(db, testing_ def test_save_location_updates_keeps_tariffs_of_other_sources(db, testing_import_service: BaseImportService): - """The cleanup is limited to the source which is imported.""" + """ + The cleanup is limited to the source which is imported. + """ other_source_tariff = Tariff( uid='tariff1', source='other_source_uid', diff --git a/tests/integration/services/import_services/datex2/data/datex2_giroe_realtime_reduced.json b/tests/integration/services/import_services/datex2/data/datex2_giroe_realtime_reduced.json new file mode 100644 index 00000000..77e9a429 --- /dev/null +++ b/tests/integration/services/import_services/datex2/data/datex2_giroe_realtime_reduced.json @@ -0,0 +1,201 @@ +{ + "messageContainer": { + "exchangeInformation": { + "dynamicInformation": { + "exchangeStatus": { + "value": "online" + }, + "messageGenerationTimestamp": "2026-10-28T20:00:00+00:00" + }, + "exchangeContext": { + "codedExchangeProtocol": { + "value": "snapshotPush" + }, + "exchangeSpecificationVersion": "3.7", + "supplierOrCisRequester": { + "name": "GLS Mobility GmbH" + } + } + }, + "payload": [ + { + "aegiEnergyInfrastructureStatusPublication": { + "energyInfrastructureSiteStatus": [ + { + "energyInfrastructureStationStatus": [ + { + "lastUpdated": "2026-10-28T20:00:00+00:00", + "reference": { + "idG": "1", + "targetClass": "FacilityObject", + "versionG": "1" + }, + "refillPointStatus": [ + { + "aegiElectricChargingPointStatus": { + "lastUpdated": "2026-10-28T20:00:00+00:00", + "reference": { + "idG": "1", + "targetClass": "FacilityObject", + "versionG": "1" + }, + "status": { + "value": "available" + } + } + }, + { + "aegiElectricChargingPointStatus": { + "lastUpdated": "2026-10-28T20:00:00+00:00", + "reference": { + "idG": "2", + "targetClass": "FacilityObject", + "versionG": "1" + }, + "status": { + "value": "charging" + } + } + } + ] + }, + { + "lastUpdated": "2026-10-28T20:00:00+00:00", + "reference": { + "idG": "2", + "targetClass": "FacilityObject", + "versionG": "1" + }, + "refillPointStatus": [ + { + "aegiElectricChargingPointStatus": { + "lastUpdated": "2026-10-28T20:00:00+00:00", + "reference": { + "idG": "3", + "targetClass": "FacilityObject", + "versionG": "1" + }, + "status": { + "value": "charging" + } + } + }, + { + "aegiElectricChargingPointStatus": { + "lastUpdated": "2026-10-28T20:00:00+00:00", + "reference": { + "idG": "4", + "targetClass": "FacilityObject", + "versionG": "1" + }, + "status": { + "value": "available" + } + } + } + ] + } + ], + "lastUpdated": "2026-10-28T20:00:00+00:00", + "reference": { + "idG": "1", + "targetClass": "FacilityObject", + "versionG": "1" + } + }, + { + "energyInfrastructureStationStatus": [ + { + "lastUpdated": "2026-10-28T20:00:00+00:00", + "reference": { + "idG": "3", + "targetClass": "FacilityObject", + "versionG": "1" + }, + "refillPointStatus": [ + { + "aegiElectricChargingPointStatus": { + "lastUpdated": "2026-10-28T20:00:00+00:00", + "reference": { + "idG": "5", + "targetClass": "FacilityObject", + "versionG": "1" + }, + "status": { + "value": "available" + } + } + }, + { + "aegiElectricChargingPointStatus": { + "lastUpdated": "2026-10-28T20:00:00+00:00", + "reference": { + "idG": "6", + "targetClass": "FacilityObject", + "versionG": "1" + }, + "status": { + "value": "charging" + } + } + } + ] + }, + { + "lastUpdated": "2026-10-28T20:00:00+00:00", + "reference": { + "idG": "4", + "targetClass": "FacilityObject", + "versionG": "1" + }, + "refillPointStatus": [ + { + "aegiElectricChargingPointStatus": { + "lastUpdated": "2026-10-28T20:00:00+00:00", + "reference": { + "idG": "7", + "targetClass": "FacilityObject", + "versionG": "1" + }, + "status": { + "value": "charging" + } + } + }, + { + "aegiElectricChargingPointStatus": { + "lastUpdated": "2026-10-28T20:00:00+00:00", + "reference": { + "idG": "8", + "targetClass": "FacilityObject", + "versionG": "1" + }, + "status": { + "value": "available" + } + } + } + ] + } + ], + "lastUpdated": "2026-10-28T20:00:00+00:00", + "reference": { + "idG": "2", + "targetClass": "FacilityObject", + "versionG": "1" + } + } + ], + "lang": "de", + "publicationCreator": { + "country": "DE", + "nationalIdentifier": "DE-GLS" + }, + "publicationTime": "2026-10-28T20:00:00+00:00" + }, + "modelBaseVersionG": "3", + "versionG": "1" + } + ] + } +} diff --git a/tests/integration/services/import_services/datex2/enbw_datex2_import_service_test.py b/tests/integration/services/import_services/datex2/enbw_datex2_import_service_test.py index 69b53655..ab3bf7da 100644 --- a/tests/integration/services/import_services/datex2/enbw_datex2_import_service_test.py +++ b/tests/integration/services/import_services/datex2/enbw_datex2_import_service_test.py @@ -362,3 +362,38 @@ def test_enbw_datex2_realtime_loops_until_last_modified_stabilizes( assert source.realtime_data_updated_at.replace(tzinfo=timezone.utc) == datetime( 2026, 4, 19, 12, 0, 0, tzinfo=timezone.utc ) + + +def test_enbw_datex2_realtime_not_modified_ends_pull( + db: SQLAlchemy, + requests_mock: Mocker, +) -> None: + """ + Mobilithek answers with 304 Not Modified once there is no newer data. This ends the pull loop: the statuses + received so far have to be saved and the source stays ACTIVE. + """ + _mock_static(requests_mock) + requests_mock.get( + REALTIME_SUBSCRIPTION_URL, + [ + { + 'status_code': 200, + 'text': _load_test_data('datex2_enbw_realtime_reduced.json'), + 'headers': {'Content-Type': 'application/json', 'Last-Modified': 'Sat, 19 Apr 2026 10:00:00 GMT'}, + }, + {'status_code': 304, 'text': ''}, + ], + ) + + enbw_service: EnBWDatex2ImportService = dependencies.get_import_services().importer_by_uid['datex2_enbw'] + enbw_service.fetch_static_data() + enbw_service.fetch_realtime_data() + + db.session.expire_all() + + source = enbw_service.get_source() + assert source.realtime_status == SourceStatus.ACTIVE + assert source.realtime_data_updated_at.replace(tzinfo=timezone.utc) == parsedate_to_datetime( + 'Sat, 19 Apr 2026 10:00:00 GMT', + ) + assert db.session.query(Evse).filter(Evse.uid == 'DE*EBW*E914082*2').first().status == EvseStatus.AVAILABLE diff --git a/tests/integration/services/import_services/ocpi/eaaze_pbw/eaaze_pbw_service_test.py b/tests/integration/services/import_services/ocpi/eaaze_pbw/eaaze_pbw_service_test.py index da86ec4c..ebddd714 100644 --- a/tests/integration/services/import_services/ocpi/eaaze_pbw/eaaze_pbw_service_test.py +++ b/tests/integration/services/import_services/ocpi/eaaze_pbw/eaaze_pbw_service_test.py @@ -34,7 +34,9 @@ def test_eaaze_pbw_sends_x_request_id(db: SQLAlchemy, requests_mock: Mocker) -> None: - """Every request to eaaze_pbw must carry an X-Request-ID header containing a random uuid4.""" + """ + Every request to eaaze_pbw must carry an X-Request-ID header containing a random uuid4. + """ eaaze_pbw_service: EaazePbwImportService = dependencies.get_import_services().importer_by_uid['eaaze_pbw'] requests_mock.get(SOURCE_URL, status_code=200, json=EMPTY_OCPI_RESPONSE) @@ -50,7 +52,9 @@ def test_eaaze_pbw_sends_x_request_id(db: SQLAlchemy, requests_mock: Mocker) -> def test_eaaze_pbw_x_request_id_is_unique_per_request(db: SQLAlchemy, requests_mock: Mocker) -> None: - """The X-Request-ID must be regenerated for each request, not reused.""" + """ + The X-Request-ID must be regenerated for each request, not reused. + """ eaaze_pbw_service: EaazePbwImportService = dependencies.get_import_services().importer_by_uid['eaaze_pbw'] requests_mock.get(SOURCE_URL, status_code=200, json=EMPTY_OCPI_RESPONSE) diff --git a/tests/unit/common/unset_parameter_test.py b/tests/unit/common/unset_parameter_test.py index 35349aa5..d249171c 100644 --- a/tests/unit/common/unset_parameter_test.py +++ b/tests/unit/common/unset_parameter_test.py @@ -22,11 +22,15 @@ class UnsetParameterTest: - """Tests for the UnsetParameter sentinel object and its type UnsetParameterType.""" + """ + Tests for the UnsetParameter sentinel object and its type UnsetParameterType. + """ @staticmethod def test_unset_parameter(): - """Test UnsetParameter and its magic methods.""" + """ + Test UnsetParameter and its magic methods. + """ assert type(UnsetParameter) is UnsetParameterType assert repr(UnsetParameter) == 'UnsetParameter' assert str(UnsetParameter) == '' @@ -34,7 +38,9 @@ def test_unset_parameter(): @staticmethod def test_unset_parameter_unique(): - """Test that UnsetParameter is a unique sentinel object, i.e. all UnsetParameter values are the same.""" + """ + Test that UnsetParameter is a unique sentinel object, i.e. all UnsetParameter values are the same. + """ unset_parameter1 = UnsetParameter unset_parameter2 = copy(unset_parameter1) unset_parameter3 = UnsetParameterType() diff --git a/tests/unit/common/validation/multi_dataclass_validator_test.py b/tests/unit/common/validation/multi_dataclass_validator_test.py index dcc1094f..2d7211fa 100644 --- a/tests/unit/common/validation/multi_dataclass_validator_test.py +++ b/tests/unit/common/validation/multi_dataclass_validator_test.py @@ -45,7 +45,9 @@ class _MixedShape(ValidataclassMixin): class MultiValidataclassValidatorTest: - """Tests for MultiValidataclassValidator and MultiValidataclassValidationError.""" + """ + Tests for MultiValidataclassValidator and MultiValidataclassValidationError. + """ @staticmethod def test_requires_at_least_one_dataclass(): diff --git a/tests/unit/common/validation/multi_dict_validator_test.py b/tests/unit/common/validation/multi_dict_validator_test.py index 32759071..0dcd3c35 100644 --- a/tests/unit/common/validation/multi_dict_validator_test.py +++ b/tests/unit/common/validation/multi_dict_validator_test.py @@ -41,7 +41,9 @@ def _mixed_shape_validator() -> DictValidator: class MultiDictValidatorTest: - """Tests for MultiDictValidator and MultiDictValidationError.""" + """ + Tests for MultiDictValidator and MultiDictValidationError. + """ @staticmethod def test_requires_at_least_one_validator(): diff --git a/tests/unit/models/capability_bits_test.py b/tests/unit/models/capability_bits_test.py index fed89a35..8eba1d8b 100644 --- a/tests/unit/models/capability_bits_test.py +++ b/tests/unit/models/capability_bits_test.py @@ -49,17 +49,23 @@ class CapabilityBitsTest: @staticmethod def test_bit_values_are_unchanged(): - """Every capability must keep the bit value it was stored with.""" + """ + Every capability must keep the bit value it was stored with. + """ assert {item.name: bit for bit, item in CAPABILITY_BITS} == EXPECTED_BITS @staticmethod def test_every_capability_has_a_bit(): - """A capability missing from the table would silently never be stored or returned.""" + """ + A capability missing from the table would silently never be stored or returned. + """ assert {item for _bit, item in CAPABILITY_BITS} == set(Capability) @staticmethod def test_bits_are_unique_and_single(): - """Each entry must be a distinct power of two, otherwise capabilities alias each other.""" + """ + Each entry must be a distinct power of two, otherwise capabilities alias each other. + """ bits = [bit for bit, _item in CAPABILITY_BITS] assert len(set(bits)) == len(bits) @@ -69,12 +75,16 @@ def test_bits_are_unique_and_single(): @staticmethod def test_bit_by_member_matches_the_table(): - """The lookup used by the setter must agree with the table used by the getter.""" + """ + The lookup used by the setter must agree with the table used by the getter. + """ assert CAPABILITY_BIT_BY_MEMBER == {item: bit for bit, item in CAPABILITY_BITS} @staticmethod def test_table_is_ordered_by_bit_value(): - """The getter returns capabilities in table order, which callers expect to be ascending by bit.""" + """ + The getter returns capabilities in table order, which callers expect to be ascending by bit. + """ bits = [bit for bit, _item in CAPABILITY_BITS] assert bits == sorted(bits) diff --git a/tests/unit/models/parking_restriction_bits_test.py b/tests/unit/models/parking_restriction_bits_test.py index 5127ebc2..ecf17c04 100644 --- a/tests/unit/models/parking_restriction_bits_test.py +++ b/tests/unit/models/parking_restriction_bits_test.py @@ -43,17 +43,23 @@ class ParkingRestrictionBitsTest: @staticmethod def test_bit_values_are_unchanged(): - """Every parking restriction must keep the bit value it was stored with.""" + """ + Every parking restriction must keep the bit value it was stored with. + """ assert {item.name: bit for bit, item in PARKING_RESTRICTION_BITS} == EXPECTED_BITS @staticmethod def test_every_parking_restriction_has_a_bit(): - """A restriction missing from the table would silently never be stored or returned.""" + """ + A restriction missing from the table would silently never be stored or returned. + """ assert {item for _bit, item in PARKING_RESTRICTION_BITS} == set(ParkingRestriction) @staticmethod def test_bits_are_unique_and_single(): - """Each entry must be a distinct power of two, otherwise restrictions alias each other.""" + """ + Each entry must be a distinct power of two, otherwise restrictions alias each other. + """ bits = [bit for bit, _item in PARKING_RESTRICTION_BITS] assert len(set(bits)) == len(bits) @@ -63,12 +69,16 @@ def test_bits_are_unique_and_single(): @staticmethod def test_bit_by_member_matches_the_table(): - """The lookup used by the setter must agree with the table used by the getter.""" + """ + The lookup used by the setter must agree with the table used by the getter. + """ assert PARKING_RESTRICTION_BIT_BY_MEMBER == {item: bit for bit, item in PARKING_RESTRICTION_BITS} @staticmethod def test_table_is_ordered_by_bit_value(): - """The getter returns restrictions in table order, which callers expect to be ascending by bit.""" + """ + The getter returns restrictions in table order, which callers expect to be ascending by bit. + """ bits = [bit for bit, _item in PARKING_RESTRICTION_BITS] assert bits == sorted(bits) diff --git a/tests/unit/services/import_services/datex2/datex2_v3_5_input_test.py b/tests/unit/services/import_services/datex2/datex2_v3_5_input_test.py new file mode 100644 index 00000000..48982287 --- /dev/null +++ b/tests/unit/services/import_services/datex2/datex2_v3_5_input_test.py @@ -0,0 +1,81 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +import json +from pathlib import Path + +from validataclass.validators import DataclassValidator + +from webapp.services.import_services.datex2.v3_5.base_datex2_v3_5_import_service import BaseDatex2V35ImportService +from webapp.services.import_services.datex2.v3_5.datex2_v3_5_json_static_mapper import Datex2V35JSONStaticMapper +from webapp.shared.datex2.models import MessageContainerWrapperInput, ProtocolTypeEnum +from webapp.shared.datex2.v3_5_json_static.models.location_reference_g_input import LocationReferenceGInput + +REALTIME_DATA_PATH = Path( + Path(__file__).parents[4], + 'integration', + 'services', + 'import_services', + 'datex2', + 'data', + 'datex2_enbw_realtime_reduced.json', +) + + +def _load_realtime_data() -> dict: + with REALTIME_DATA_PATH.open() as realtime_file: + return json.load(realtime_file) + + +def test_location_reference_without_facility_location() -> None: + location_reference = DataclassValidator(LocationReferenceGInput).validate({ + 'locPointLocation': {'pointByCoordinates': {'pointCoordinates': {'latitude': 47.67, 'longitude': 9.14}}}, + }) + + assert Datex2V35JSONStaticMapper._get_facility_location_by_location_reference(location_reference) is None + + +def test_message_container_with_empty_extensions() -> None: + data = _load_realtime_data() + data['messageContainer']['exchangeInformation']['dynamicInformation']['exDynamicInformationExtensionG'] = {} + + message_container = DataclassValidator(MessageContainerWrapperInput).validate(data) + + assert message_container.messageContainer.exchangeInformation.dynamicInformation is not None + + +def test_wrap_realtime_payload_wraps_d2_payload() -> None: + payload = _load_realtime_data()['messageContainer']['payload'][0] + + data = BaseDatex2V35ImportService.wrap_realtime_payload({'payload': payload}) + message_container = DataclassValidator(MessageContainerWrapperInput).validate(data) + + assert len(message_container.messageContainer.payload) == 1 + exchange_context = message_container.messageContainer.exchangeInformation.exchangeContext + assert exchange_context.codedExchangeProtocol.value is ProtocolTypeEnum.SNAPSHOT_PULL + + +def test_wrap_realtime_payload_keeps_other_documents() -> None: + message_container_data = _load_realtime_data() + payload = message_container_data['messageContainer']['payload'] + # A payload list beside exchange information is neither a D2Payload nor a MessageContainer document + invalid_data = {'payload': payload, 'exchangeInformation': message_container_data['messageContainer']} + + assert BaseDatex2V35ImportService.wrap_realtime_payload(message_container_data) is message_container_data + assert BaseDatex2V35ImportService.wrap_realtime_payload(invalid_data) is invalid_data + assert BaseDatex2V35ImportService.wrap_realtime_payload({'payload': payload}) == {'payload': payload} diff --git a/tests/unit/services/import_services/datex2/datex2_v3_7_realtime_input_test.py b/tests/unit/services/import_services/datex2/datex2_v3_7_realtime_input_test.py new file mode 100644 index 00000000..c8ae2009 --- /dev/null +++ b/tests/unit/services/import_services/datex2/datex2_v3_7_realtime_input_test.py @@ -0,0 +1,81 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +import json +from pathlib import Path + +import pytest +from validataclass.exceptions import ValidationError + +from webapp.services.import_services.datex2.v3_7.base_datex2_v3_7_import_service import BaseDatex2V37ImportService + +REALTIME_DATA_PATH = Path( + Path(__file__).parents[4], + 'integration', + 'services', + 'import_services', + 'datex2', + 'data', + 'datex2_giroe_realtime_reduced.json', +) + + +def _load_realtime_data() -> dict: + with REALTIME_DATA_PATH.open() as realtime_file: + return json.load(realtime_file) + + +def _count_refill_point_statuses(payloads: list) -> int: + return sum( + len(station_status.refillPointStatus) + for payload in payloads + for site_status in payload.aegiEnergyInfrastructureStatusPublication.energyInfrastructureSiteStatus + for station_status in site_status.energyInfrastructureStationStatus + ) + + +def test_get_realtime_payloads_from_message_container() -> None: + payloads = BaseDatex2V37ImportService.get_realtime_payloads(_load_realtime_data()) + + assert len(payloads) == 1 + assert _count_refill_point_statuses(payloads) == 8 + + +def test_get_realtime_payloads_from_d2_payload() -> None: + payload = _load_realtime_data()['messageContainer']['payload'][0] + + payloads = BaseDatex2V37ImportService.get_realtime_payloads({'payload': payload}) + + assert len(payloads) == 1 + assert _count_refill_point_statuses(payloads) == 8 + + +def test_get_realtime_payloads_rejects_message_container_without_wrapper() -> None: + # Payloads and exchange information outside a messageContainer field are no valid DATEX II document + data = _load_realtime_data()['messageContainer'] + + with pytest.raises(ValidationError): + BaseDatex2V37ImportService.get_realtime_payloads(data) + + +def test_get_realtime_payloads_rejects_unknown_message_container_fields() -> None: + data = _load_realtime_data() + data['messageContainer']['unknownField'] = True + + with pytest.raises(ValidationError): + BaseDatex2V37ImportService.get_realtime_payloads(data) diff --git a/tests/unit/services/import_services/datex2/tariff_grouping_mixin_test.py b/tests/unit/services/import_services/datex2/tariff_grouping_mixin_test.py index d8678ec4..df0b4400 100644 --- a/tests/unit/services/import_services/datex2/tariff_grouping_mixin_test.py +++ b/tests/unit/services/import_services/datex2/tariff_grouping_mixin_test.py @@ -40,7 +40,9 @@ class TariffGroupingService(TariffGroupingMixin): - """Minimal host for the mixin: it only needs a source_info to name the source in its log line.""" + """ + Minimal host for the mixin: it only needs a source_info to name the source in its log line. + """ source_info = SourceInfo( uid=SOURCE_UID, @@ -118,7 +120,9 @@ def _build_location_update(*tariff_updates: TariffUpdate) -> LocationUpdate: def _build_location_update_from_associations( *tariff_association_updates_per_evse: list[TariffAssociationUpdate], ) -> LocationUpdate: - """Wrap each list of tariff associations into its own EVSE.""" + """ + Wrap each list of tariff associations into its own EVSE. + """ evse_updates: list[EvseUpdate] = [] for index, tariff_association_updates in enumerate(tariff_association_updates_per_evse): evse_updates.append( @@ -147,7 +151,9 @@ def _build_location_update_from_associations( def _grouped_tariff_association_updates(*location_updates: LocationUpdate) -> list[TariffAssociationUpdate]: - """The tariff associations which survived the grouping, in the order the EVSEs hold them.""" + """ + The tariff associations which survived the grouping, in the order the EVSEs hold them. + """ return [ tariff_association_update for location_update in location_updates @@ -158,7 +164,9 @@ def _grouped_tariff_association_updates(*location_updates: LocationUpdate) -> li def _grouped_tariff_updates(*location_updates: LocationUpdate) -> list[TariffUpdate]: - """The tariffs which survived the grouping, in the order their associations appear.""" + """ + The tariffs which survived the grouping, in the order their associations appear. + """ return [ tariff_association_update.tariff for tariff_association_update in _grouped_tariff_association_updates(*location_updates) @@ -166,7 +174,9 @@ def _grouped_tariff_updates(*location_updates: LocationUpdate) -> list[TariffUpd def test_group_identical_tariffs_collapses_identical_fees() -> None: - """Tariffs with identical fees collapse into one tariff, no matter which uid they started with.""" + """ + Tariffs with identical fees collapse into one tariff, no matter which uid they started with. + """ tariff_updates = [_build_tariff_update(uid='first'), _build_tariff_update(uid='second')] location_update = _build_location_update(*tariff_updates) @@ -220,7 +230,9 @@ def test_group_identical_tariffs_keeps_differing_audiences_apart() -> None: def test_group_identical_tariffs_uses_newest_last_updated() -> None: - """A group is stamped with the newest last_updated of its members.""" + """ + A group is stamped with the newest last_updated of its members. + """ oldest = datetime(2026, 1, 1, 10, 0, tzinfo=timezone.utc) newest = datetime(2026, 5, 17, 8, 30, tzinfo=timezone.utc) middle = datetime(2026, 3, 2, 12, 0, tzinfo=timezone.utc) @@ -246,7 +258,9 @@ def test_group_identical_tariffs_uses_newest_last_updated() -> None: def test_group_identical_tariffs_uses_oldest_start_date_time() -> None: - """The grouped association starts when the first EVSE of its group started to charge these fees.""" + """ + The grouped association starts when the first EVSE of its group started to charge these fees. + """ oldest = datetime(2026, 1, 1, 10, 0, tzinfo=timezone.utc) newest = datetime(2026, 5, 17, 8, 30, tzinfo=timezone.utc) location_update = _build_location_update_from_associations( @@ -279,7 +293,9 @@ def test_group_identical_tariffs_keeps_timestamps_none_without_timestamps() -> N def test_group_identical_tariffs_keeps_differing_fees_apart() -> None: - """Any difference in the fees keeps the tariffs, and with them their associations, separate.""" + """ + Any difference in the fees keeps the tariffs, and with them their associations, separate. + """ reference = _build_tariff_update(uid='reference') tariff_updates = [ reference, @@ -302,7 +318,9 @@ def test_group_identical_tariffs_keeps_differing_fees_apart() -> None: def test_group_identical_tariffs_groups_across_locations() -> None: - """Grouping spans the whole import, not just a single location.""" + """ + Grouping spans the whole import, not just a single location. + """ first_location_update = _build_location_update(_build_tariff_update(uid='first')) second_location_update = _build_location_update(_build_tariff_update(uid='second')) diff --git a/tests/unit/services/import_services/eliso/eliso_validators_test.py b/tests/unit/services/import_services/eliso/eliso_validators_test.py new file mode 100644 index 00000000..8ac25c8e --- /dev/null +++ b/tests/unit/services/import_services/eliso/eliso_validators_test.py @@ -0,0 +1,111 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from datetime import datetime, timezone +from decimal import Decimal + +import pytest +from validataclass.exceptions import ValidationError +from validataclass.validators import DataclassValidator + +from webapp.models.connector import ConnectorFormat, ConnectorType, PowerType +from webapp.models.evse import EvseStatus +from webapp.services.import_services.eliso.eliso_validators import ElisoEvseStatusInput, ElisoLocationInput + + +def _get_location_dict(**kwargs) -> dict: + return { + 'city': 'Haßfurt', + 'address': 'Am Wasserwerk 2', + 'postalCode': '97437', + 'country_iso_3166_alpha_2': 'DE', + 'coordinates': {'latitude': 50.030589789349, 'longitude': 10.520250965406}, + 'operator_name': 'eliso GmbH', + 'hotline_number': '+49 711 40042100', + 'opening_time': '24/7', + 'mobilithek_last_updated_dts': '2026-10-28T04:04:51.23388+00:00', + 'evses': [ + { + 'evseId': 'DE*ELI*E5207046', + 'connectors': [{'maxPower': 135, 'powerType': 'DC', 'type_of_connector': 'Combo2/CCS (DC)'}], + }, + { + 'evseId': 'DE*ELI*E5207045', + 'connectors': [{'maxPower': 22, 'powerType': 'AC_3_PHASE', 'type_of_connector': 'Type 2 (AC)'}], + }, + ], + **kwargs, + } + + +def test_location_to_location_update() -> None: + location_input = DataclassValidator(ElisoLocationInput).validate(_get_location_dict()) + + location_update = location_input.to_location_update('eliso') + + assert location_update.uid == 'DE*ELI*E5207045' + assert location_update.country == 'DEU' + assert location_update.lat == Decimal('50.0305898') + assert location_update.twentyfourseven is True + assert location_update.operator.name == 'eliso GmbH' + evses = location_update.charging_pool[0].evses + assert [evse.evse_id for evse in evses] == ['DE*ELI*E5207046', 'DE*ELI*E5207045'] + assert evses[0].connectors[0].standard == ConnectorType.IEC_62196_T2_COMBO + assert evses[0].connectors[0].format == ConnectorFormat.CABLE + assert evses[0].connectors[0].power_type == PowerType.DC + assert evses[0].connectors[0].max_electric_power == 135000 + assert evses[1].connectors[0].standard == ConnectorType.IEC_62196_T2 + assert evses[1].connectors[0].format == ConnectorFormat.SOCKET + + +def test_location_with_unknown_connector_type() -> None: + location_dict = _get_location_dict() + location_dict['evses'][0]['connectors'][0]['type_of_connector'] = 'CHAdeMO' + + with pytest.raises(ValidationError): + DataclassValidator(ElisoLocationInput).validate(location_dict) + + +@pytest.mark.parametrize( + ('operational_status', 'availability_status', 'evse_status'), + [ + ('Operational', 'Not in use', EvseStatus.AVAILABLE), + ('Operational', 'In use', EvseStatus.CHARGING), + ('Operational', None, EvseStatus.UNKNOWN), + ('Non-operational', None, EvseStatus.INOPERATIVE), + ], +) +def test_evse_status_to_evse_realtime_update( + operational_status: str, + availability_status: str | None, + evse_status: EvseStatus, +) -> None: + evse_status_dict = { + 'evseId': 'DE*ELI*E3584570', + 'adhoc_price': 0.49, + 'operational_status': operational_status, + 'mobilithek_last_updated_dts': '2026-10-28T19:53:17.994069+00:00', + } + if availability_status is not None: + evse_status_dict['availability_status'] = availability_status + + evse_update = DataclassValidator(ElisoEvseStatusInput).validate(evse_status_dict).to_evse_realtime_update() + + assert evse_update.evse_id == 'DE*ELI*E3584570' + assert evse_update.status == evse_status + assert evse_update.status_last_updated == datetime(2026, 10, 28, 19, 53, 17, 994069, tzinfo=timezone.utc) diff --git a/webapp/public_api/ocpi/ocpi_redirect.py b/webapp/public_api/ocpi/ocpi_redirect.py index d029f7b7..307291d4 100644 --- a/webapp/public_api/ocpi/ocpi_redirect.py +++ b/webapp/public_api/ocpi/ocpi_redirect.py @@ -20,8 +20,10 @@ class OcpiPathRewriteMiddleware: - """WSGI middleware that rewrites /api/ocpi/ paths to /api/public/ocpi/ so the existing - OCPI endpoints are transparently available under both prefixes.""" + """ + WSGI middleware that rewrites /api/ocpi/ paths to /api/public/ocpi/ so the existing + OCPI endpoints are transparently available under both prefixes. + """ def __init__(self, app: Callable): self.app = app diff --git a/webapp/public_api/ocpi/v2_2/locations/rest_api.py b/webapp/public_api/ocpi/v2_2/locations/rest_api.py index ccecd6c5..c434dcbd 100644 --- a/webapp/public_api/ocpi/v2_2/locations/rest_api.py +++ b/webapp/public_api/ocpi/v2_2/locations/rest_api.py @@ -69,7 +69,9 @@ def __init__(self): class OcpiLegacyLocationBlueprint(BaseBlueprint): - """Backward compatibility: /api/ocpi/2.2/location""" + """ + Backward compatibility: /api/ocpi/2.2/location + """ location_handler: LocationHandler diff --git a/webapp/services/import_services/datex2/__init__.py b/webapp/services/import_services/datex2/__init__.py index 5c232005..c061c72b 100644 --- a/webapp/services/import_services/datex2/__init__.py +++ b/webapp/services/import_services/datex2/__init__.py @@ -17,27 +17,44 @@ """ from .v3_5 import ( + AhBeteiligungDatex2ImportService, AmpecoDatex2ImportService, + AudiDatex2ImportService, ChargecloudDatex2ImportService, + EaazeDatex2ImportService, EClearingNetDatex2ImportService, EcoMovementDatex2ImportService, EluMobilityDatex2ImportService, EnBWDatex2ImportService, + EnerandoDatex2ImportService, EnioDatex2ImportService, ERoundDatex2ImportService, + EulektroDatex2ImportService, + EvPriceDatex2ImportService, + EweDatex2ImportService, + FlaviaDatex2ImportService, + GpJouleDatex2ImportService, GridAndCoDatex2ImportService, InergiaDatex2ImportService, + InselwerkeDatex2ImportService, + KulRudnickDatex2ImportService, LadebusinessDatex2ImportService, + LadesonneDatex2ImportService, LichtblickDatex2ImportService, MidorionDatex2ImportService, MontaDatex2ImportService, MsuDatex2ImportService, PumpDatex2ImportService, QwelloDatex2ImportService, + RMBeteiligungDatex2ImportService, + RoadDatex2ImportService, SmatricsDatex2ImportService, + SonnenstromSaarpfalzDatex2ImportService, TaubertDatex2ImportService, TeslaDatex2ImportService, + UlmDatex2ImportService, VaylensDatex2ImportService, + VirtaDatex2ImportService, VolkswagenDatex2ImportService, WirelaneDatex2ImportService, ) diff --git a/webapp/services/import_services/datex2/v3_5/__init__.py b/webapp/services/import_services/datex2/v3_5/__init__.py index 22595673..89d51909 100644 --- a/webapp/services/import_services/datex2/v3_5/__init__.py +++ b/webapp/services/import_services/datex2/v3_5/__init__.py @@ -16,28 +16,45 @@ along with this program. If not, see . """ +from .ah_beteiligung_datex2_import_service import AhBeteiligungDatex2ImportService from .ampeco_datex2_import_service import AmpecoDatex2ImportService +from .audi_datex2_import_service import AudiDatex2ImportService from .base_datex2_v3_5_import_service import BaseDatex2V35ImportService, RealtimeResult from .chargecloud_datex2_import_service import ChargecloudDatex2ImportService from .datex2_v3_5_json_static_mapper import Datex2V35JSONStaticMapper from .e_clearing_net_datex2_import_service import EClearingNetDatex2ImportService +from .eaaze_datex2_import_service import EaazeDatex2ImportService from .ecomovement_datex2_import_service import EcoMovementDatex2ImportService from .elu_mobility_datex2_import_service import EluMobilityDatex2ImportService from .enbw_datex2_import_service import EnBWDatex2ImportService +from .enerando_datex2_import_service import EnerandoDatex2ImportService from .enio_datex2_import_service import EnioDatex2ImportService from .eround_datex2_import_service import ERoundDatex2ImportService +from .eulektro_datex2_import_service import EulektroDatex2ImportService +from .ev_price_datex2_import_service import EvPriceDatex2ImportService +from .ewe_datex2_import_service import EweDatex2ImportService +from .flavia_datex2_import_service import FlaviaDatex2ImportService +from .gp_joule_datex2_import_service import GpJouleDatex2ImportService from .gridandco_datex2_import_service import GridAndCoDatex2ImportService from .inergia_datex2_import_service import InergiaDatex2ImportService +from .inselwerke_datex2_import_service import InselwerkeDatex2ImportService +from .kul_rudnick_datex2_import_service import KulRudnickDatex2ImportService from .ladebusiness_datex2_import_service import LadebusinessDatex2ImportService +from .ladesonne_datex2_import_service import LadesonneDatex2ImportService from .lichtblick_datex2_import_service import LichtblickDatex2ImportService from .midorion_datex2_import_service import MidorionDatex2ImportService from .monta_datex2_import_service import MontaDatex2ImportService from .msu_datex2_import_service import MsuDatex2ImportService from .pump_datex2_import_service import PumpDatex2ImportService from .qwello_datex2_import_service import QwelloDatex2ImportService +from .r_m_beteiligung_datex2_import_service import RMBeteiligungDatex2ImportService +from .road_datex2_import_service import RoadDatex2ImportService from .smatrics_datex2_import_service import SmatricsDatex2ImportService +from .sonnenstrom_saarpfalz_datex2_import_service import SonnenstromSaarpfalzDatex2ImportService from .taubert_datex2_import_service import TaubertDatex2ImportService from .tesla_datex2_import_service import TeslaDatex2ImportService +from .ulm_datex2_import_service import UlmDatex2ImportService from .vaylens_datex2_import_service import VaylensDatex2ImportService +from .virta_datex2_import_service import VirtaDatex2ImportService from .volkswagen_datex2_import_service import VolkswagenDatex2ImportService from .wirelane_datex2_import_service import WirelaneDatex2ImportService diff --git a/webapp/services/import_services/datex2/v3_5/ah_beteiligung_datex2_import_service.py b/webapp/services/import_services/datex2/v3_5/ah_beteiligung_datex2_import_service.py new file mode 100644 index 00000000..56b1c976 --- /dev/null +++ b/webapp/services/import_services/datex2/v3_5/ah_beteiligung_datex2_import_service.py @@ -0,0 +1,31 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from webapp.services.import_services.datex2.v3_5.base_datex2_v3_5_import_service import BaseDatex2V35ImportService +from webapp.services.import_services.models import SourceInfo + + +class AhBeteiligungDatex2ImportService(BaseDatex2V35ImportService): + source_info = SourceInfo( + uid='datex2_ah_beteiligung', + name='AH Beteiligungsgesellschaft Datex II', + public_url='https://mobilithek.info/offers/993495185148837888', + source_url='https://mobilithek.info/offers/993495185148837888', + attribution_contributor='AH Beteiligungsgesellschaft mbH', + has_realtime_data=True, + ) diff --git a/webapp/services/import_services/datex2/v3_5/audi_datex2_import_service.py b/webapp/services/import_services/datex2/v3_5/audi_datex2_import_service.py new file mode 100644 index 00000000..69fccaf3 --- /dev/null +++ b/webapp/services/import_services/datex2/v3_5/audi_datex2_import_service.py @@ -0,0 +1,32 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from webapp.services.import_services.datex2.v3_5.base_datex2_v3_5_import_service import BaseDatex2V35ImportService +from webapp.services.import_services.models import SourceInfo + + +class AudiDatex2ImportService(BaseDatex2V35ImportService): + source_info = SourceInfo( + uid='datex2_audi', + name='Audi charging hub Datex II', + public_url='https://mobilithek.info/offers/998571342932365312', + source_url='https://mobilithek.info/offers/998571342932365312', + attribution_contributor='Audi AG', + attribution_license='CC-0', + has_realtime_data=True, + ) diff --git a/webapp/services/import_services/datex2/v3_5/base_datex2_v3_5_import_service.py b/webapp/services/import_services/datex2/v3_5/base_datex2_v3_5_import_service.py index a97bd760..3f764971 100644 --- a/webapp/services/import_services/datex2/v3_5/base_datex2_v3_5_import_service.py +++ b/webapp/services/import_services/datex2/v3_5/base_datex2_v3_5_import_service.py @@ -156,6 +156,10 @@ def fetch_realtime_data(self): try: data, last_modified = self.request_data(subscription_id, if_modified_since) except RemoteException as e: + # Mobilithek answers with 304 Not Modified if there is no newer data + if e.http_status == 304: + last_modified = if_modified_since + break logger.error( f'Datex2 realtime request failed: {e.to_dict()}. Text: {e.data}', extra={'attributes': {'type': LogMessageType.IMPORT_SOURCE}}, @@ -172,7 +176,7 @@ def fetch_realtime_data(self): return try: - message_container = self.validate_realtime_data(data) + message_container = self.validate_realtime_data(self.wrap_realtime_payload(data)) self.add_realtime_data(message_container, result) except ImportException as e: logger.error( @@ -261,7 +265,9 @@ def mark_realtime_push_received( @staticmethod def get_publication_time(message_container: MessageContainerWrapperInput) -> datetime | None: - """Return the payload's ``publicationTime``, or ``None`` if the payload does not carry one.""" + """ + Return the payload's ``publicationTime``, or ``None`` if the payload does not carry one. + """ if not message_container.messageContainer.payload: return None @@ -303,6 +309,32 @@ def store_realtime_data(self, message_container: MessageContainerWrapperInput) - extra={'attributes': {'type': LogMessageType.IMPORT_LOCATION}}, ) + @staticmethod + def wrap_realtime_payload(data: dict) -> dict: + """ + Realtime data can also be a plain D2Payload document instead of a MessageContainer. As the import works with + MessageContainers, the payload gets wrapped in one with a generic snapshotPull exchange information. + """ + if set(data.keys()) != {'payload'} or not isinstance(data['payload'], dict): + return data + + return { + 'messageContainer': { + 'payload': [data['payload']], + 'exchangeInformation': { + 'exchangeContext': { + 'codedExchangeProtocol': {'value': ProtocolTypeEnum.SNAPSHOT_PULL.value}, + 'exchangeSpecificationVersion': '3.5', + 'supplierOrCisRequester': {}, + }, + 'dynamicInformation': { + 'exchangeStatus': {'value': 'online'}, + 'messageGenerationTimestamp': datetime.now(tz=timezone.utc).isoformat(), + }, + }, + }, + } + def validate_realtime_data(self, data: dict) -> MessageContainerWrapperInput: """ Validate the raw DATEX II v3.5 realtime envelope and return the parsed message container. @@ -339,7 +371,9 @@ def validate_realtime_data(self, data: dict) -> MessageContainerWrapperInput: @staticmethod def count_station_statuses(message_container: MessageContainerWrapperInput) -> int: - """Return the number of ``energyInfrastructureStationStatus`` entries carried by the payload.""" + """ + Return the number of ``energyInfrastructureStationStatus`` entries carried by the payload. + """ payload = message_container.messageContainer.payload[0] publication = payload.aegiEnergyInfrastructureStatusPublication if publication is UnsetValue or publication.energyInfrastructureSiteStatus is UnsetValue: diff --git a/webapp/services/import_services/datex2/v3_5/eaaze_datex2_import_service.py b/webapp/services/import_services/datex2/v3_5/eaaze_datex2_import_service.py new file mode 100644 index 00000000..49201810 --- /dev/null +++ b/webapp/services/import_services/datex2/v3_5/eaaze_datex2_import_service.py @@ -0,0 +1,32 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from webapp.services.import_services.datex2.v3_5.base_datex2_v3_5_import_service import BaseDatex2V35ImportService +from webapp.services.import_services.models import SourceInfo + + +class EaazeDatex2ImportService(BaseDatex2V35ImportService): + source_info = SourceInfo( + uid='datex2_eaaze', + name='EAAZE Datex II', + public_url='https://mobilithek.info/offers/980426161955500032', + source_url='https://mobilithek.info/offers/980426161955500032', + attribution_contributor='EAAZE GmbH', + attribution_license='CC-0', + has_realtime_data=True, + ) diff --git a/webapp/services/import_services/datex2/v3_5/enerando_datex2_import_service.py b/webapp/services/import_services/datex2/v3_5/enerando_datex2_import_service.py new file mode 100644 index 00000000..88076fc8 --- /dev/null +++ b/webapp/services/import_services/datex2/v3_5/enerando_datex2_import_service.py @@ -0,0 +1,31 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from webapp.services.import_services.datex2.v3_5.base_datex2_v3_5_import_service import BaseDatex2V35ImportService +from webapp.services.import_services.models import SourceInfo + + +class EnerandoDatex2ImportService(BaseDatex2V35ImportService): + source_info = SourceInfo( + uid='datex2_enerando', + name='ENERANDO Datex II', + public_url='https://mobilithek.info/offers/1037036344672452608', + source_url='https://mobilithek.info/offers/1037036344672452608', + attribution_contributor='ENERANDO Technologies GmbH', + has_realtime_data=True, + ) diff --git a/webapp/services/import_services/datex2/v3_5/eulektro_datex2_import_service.py b/webapp/services/import_services/datex2/v3_5/eulektro_datex2_import_service.py new file mode 100644 index 00000000..fbd9a8ff --- /dev/null +++ b/webapp/services/import_services/datex2/v3_5/eulektro_datex2_import_service.py @@ -0,0 +1,31 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from webapp.services.import_services.datex2.v3_5.base_datex2_v3_5_import_service import BaseDatex2V35ImportService +from webapp.services.import_services.models import SourceInfo + + +class EulektroDatex2ImportService(BaseDatex2V35ImportService): + source_info = SourceInfo( + uid='datex2_eulektro', + name='Eulektro Datex II', + public_url='https://mobilithek.info/offers/973938793916375040', + source_url='https://mobilithek.info/offers/973938793916375040', + attribution_contributor='Eulektro GmbH', + has_realtime_data=True, + ) diff --git a/webapp/services/import_services/datex2/v3_5/ev_price_datex2_import_service.py b/webapp/services/import_services/datex2/v3_5/ev_price_datex2_import_service.py new file mode 100644 index 00000000..1f12be34 --- /dev/null +++ b/webapp/services/import_services/datex2/v3_5/ev_price_datex2_import_service.py @@ -0,0 +1,32 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from webapp.services.import_services.datex2.v3_5.base_datex2_v3_5_import_service import BaseDatex2V35ImportService +from webapp.services.import_services.models import SourceInfo + + +class EvPriceDatex2ImportService(BaseDatex2V35ImportService): + source_info = SourceInfo( + uid='datex2_ev_price', + name='EV Price Datex II', + public_url='https://mobilithek.info/offers/989311807176560640', + source_url='https://mobilithek.info/offers/989311807176560640', + attribution_contributor='EW Pricing GmbH', + attribution_license='CC-0', + has_realtime_data=True, + ) diff --git a/webapp/services/import_services/datex2/v3_5/ewe_datex2_import_service.py b/webapp/services/import_services/datex2/v3_5/ewe_datex2_import_service.py new file mode 100644 index 00000000..39d18d2f --- /dev/null +++ b/webapp/services/import_services/datex2/v3_5/ewe_datex2_import_service.py @@ -0,0 +1,32 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from webapp.services.import_services.datex2.v3_5.base_datex2_v3_5_import_service import BaseDatex2V35ImportService +from webapp.services.import_services.models import SourceInfo + + +class EweDatex2ImportService(BaseDatex2V35ImportService): + source_info = SourceInfo( + uid='datex2_ewe', + name='EWE Datex II', + public_url='https://mobilithek.info/offers/1024264572575293440', + source_url='https://mobilithek.info/offers/1024264572575293440', + attribution_contributor='EWE', + attribution_license='CC-0', + has_realtime_data=True, + ) diff --git a/webapp/services/import_services/datex2/v3_5/flavia_datex2_import_service.py b/webapp/services/import_services/datex2/v3_5/flavia_datex2_import_service.py new file mode 100644 index 00000000..267a5d3b --- /dev/null +++ b/webapp/services/import_services/datex2/v3_5/flavia_datex2_import_service.py @@ -0,0 +1,31 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from webapp.services.import_services.datex2.v3_5.base_datex2_v3_5_import_service import BaseDatex2V35ImportService +from webapp.services.import_services.models import SourceInfo + + +class FlaviaDatex2ImportService(BaseDatex2V35ImportService): + source_info = SourceInfo( + uid='datex2_flavia', + name='Flavia Datex II', + public_url='https://mobilithek.info/offers/1004450433724211200', + source_url='https://mobilithek.info/offers/1004450433724211200', + attribution_contributor='Grid & Co. GmbH', + has_realtime_data=True, + ) diff --git a/webapp/services/import_services/datex2/v3_5/gp_joule_datex2_import_service.py b/webapp/services/import_services/datex2/v3_5/gp_joule_datex2_import_service.py new file mode 100644 index 00000000..dbb0f21d --- /dev/null +++ b/webapp/services/import_services/datex2/v3_5/gp_joule_datex2_import_service.py @@ -0,0 +1,32 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from webapp.services.import_services.datex2.v3_5.base_datex2_v3_5_import_service import BaseDatex2V35ImportService +from webapp.services.import_services.models import SourceInfo + + +class GpJouleDatex2ImportService(BaseDatex2V35ImportService): + source_info = SourceInfo( + uid='datex2_gp_joule', + name='GP JOULE Connect Datex II', + public_url='https://mobilithek.info/offers/997111469996658688', + source_url='https://mobilithek.info/offers/997111469996658688', + attribution_contributor='GP JOULE Connect GmbH', + attribution_license='CC-0', + has_realtime_data=True, + ) diff --git a/webapp/services/import_services/datex2/v3_5/inselwerke_datex2_import_service.py b/webapp/services/import_services/datex2/v3_5/inselwerke_datex2_import_service.py new file mode 100644 index 00000000..52b75c53 --- /dev/null +++ b/webapp/services/import_services/datex2/v3_5/inselwerke_datex2_import_service.py @@ -0,0 +1,31 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from webapp.services.import_services.datex2.v3_5.base_datex2_v3_5_import_service import BaseDatex2V35ImportService +from webapp.services.import_services.models import SourceInfo + + +class InselwerkeDatex2ImportService(BaseDatex2V35ImportService): + source_info = SourceInfo( + uid='datex2_inselwerke', + name='Inselwerke Datex II', + public_url='https://mobilithek.info/offers/981180413145477120', + source_url='https://mobilithek.info/offers/981180413145477120', + attribution_contributor='Inselwerke eG', + has_realtime_data=True, + ) diff --git a/webapp/services/import_services/datex2/v3_5/kul_rudnick_datex2_import_service.py b/webapp/services/import_services/datex2/v3_5/kul_rudnick_datex2_import_service.py new file mode 100644 index 00000000..0bb68989 --- /dev/null +++ b/webapp/services/import_services/datex2/v3_5/kul_rudnick_datex2_import_service.py @@ -0,0 +1,32 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from webapp.services.import_services.datex2.v3_5.base_datex2_v3_5_import_service import BaseDatex2V35ImportService +from webapp.services.import_services.models import SourceInfo + + +class KulRudnickDatex2ImportService(BaseDatex2V35ImportService): + source_info = SourceInfo( + uid='datex2_kul_rudnick', + name='K.u.L. Rudnick Datex II', + public_url='https://mobilithek.info/offers/983747474988376064', + source_url='https://mobilithek.info/offers/983747474988376064', + attribution_contributor='K.u.L. Rudnick Bau und Bauträger GmbH', + attribution_license='CC-0', + has_realtime_data=False, + ) diff --git a/webapp/services/import_services/datex2/v3_5/ladesonne_datex2_import_service.py b/webapp/services/import_services/datex2/v3_5/ladesonne_datex2_import_service.py new file mode 100644 index 00000000..75beed30 --- /dev/null +++ b/webapp/services/import_services/datex2/v3_5/ladesonne_datex2_import_service.py @@ -0,0 +1,31 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from webapp.services.import_services.datex2.v3_5.base_datex2_v3_5_import_service import BaseDatex2V35ImportService +from webapp.services.import_services.models import SourceInfo + + +class LadesonneDatex2ImportService(BaseDatex2V35ImportService): + source_info = SourceInfo( + uid='datex2_ladesonne', + name='Ladesonne Datex II', + public_url='https://mobilithek.info/offers/988133177339846656', + source_url='https://mobilithek.info/offers/988133177339846656', + attribution_contributor='Ladesonne GmbH & Co. KG', + has_realtime_data=True, + ) diff --git a/webapp/services/import_services/datex2/v3_5/r_m_beteiligung_datex2_import_service.py b/webapp/services/import_services/datex2/v3_5/r_m_beteiligung_datex2_import_service.py new file mode 100644 index 00000000..8c5baec6 --- /dev/null +++ b/webapp/services/import_services/datex2/v3_5/r_m_beteiligung_datex2_import_service.py @@ -0,0 +1,31 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from webapp.services.import_services.datex2.v3_5.base_datex2_v3_5_import_service import BaseDatex2V35ImportService +from webapp.services.import_services.models import SourceInfo + + +class RMBeteiligungDatex2ImportService(BaseDatex2V35ImportService): + source_info = SourceInfo( + uid='datex2_r_m_beteiligung', + name='R & M Beteiligungsgesellschaft Datex II', + public_url='https://mobilithek.info/offers/993499364131065856', + source_url='https://mobilithek.info/offers/993499364131065856', + attribution_contributor='R & M Beteiligungsgesellschaft mbH', + has_realtime_data=True, + ) diff --git a/webapp/services/import_services/datex2/v3_5/road_datex2_import_service.py b/webapp/services/import_services/datex2/v3_5/road_datex2_import_service.py new file mode 100644 index 00000000..f9f5a4c3 --- /dev/null +++ b/webapp/services/import_services/datex2/v3_5/road_datex2_import_service.py @@ -0,0 +1,31 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from webapp.services.import_services.datex2.v3_5.base_datex2_v3_5_import_service import BaseDatex2V35ImportService +from webapp.services.import_services.models import SourceInfo + + +class RoadDatex2ImportService(BaseDatex2V35ImportService): + source_info = SourceInfo( + uid='datex2_road', + name='Road Datex II', + public_url='https://mobilithek.info/offers/1021366453630017536', + source_url='https://mobilithek.info/offers/1021366453630017536', + attribution_contributor='Road B.V.', + has_realtime_data=True, + ) diff --git a/webapp/services/import_services/datex2/v3_5/sonnenstrom_saarpfalz_datex2_import_service.py b/webapp/services/import_services/datex2/v3_5/sonnenstrom_saarpfalz_datex2_import_service.py new file mode 100644 index 00000000..95599677 --- /dev/null +++ b/webapp/services/import_services/datex2/v3_5/sonnenstrom_saarpfalz_datex2_import_service.py @@ -0,0 +1,31 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from webapp.services.import_services.datex2.v3_5.base_datex2_v3_5_import_service import BaseDatex2V35ImportService +from webapp.services.import_services.models import SourceInfo + + +class SonnenstromSaarpfalzDatex2ImportService(BaseDatex2V35ImportService): + source_info = SourceInfo( + uid='datex2_sonnenstrom_saarpfalz', + name='Sonnenstrom Saarpfalz Datex II', + public_url='https://mobilithek.info/offers/979427073235566592', + source_url='https://mobilithek.info/offers/979427073235566592', + attribution_contributor='Sonnenstrom Saarpfalz OHG', + has_realtime_data=True, + ) diff --git a/webapp/services/import_services/datex2/v3_5/ulm_datex2_import_service.py b/webapp/services/import_services/datex2/v3_5/ulm_datex2_import_service.py new file mode 100644 index 00000000..85adf054 --- /dev/null +++ b/webapp/services/import_services/datex2/v3_5/ulm_datex2_import_service.py @@ -0,0 +1,32 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from webapp.services.import_services.datex2.v3_5.base_datex2_v3_5_import_service import BaseDatex2V35ImportService +from webapp.services.import_services.models import SourceInfo + + +class UlmDatex2ImportService(BaseDatex2V35ImportService): + source_info = SourceInfo( + uid='datex2_ulm', + name='Ulm Datex II', + public_url='https://mobilithek.info/offers/854410608351543296', + source_url='https://mobilithek.info/offers/854410608351543296', + attribution_contributor='Smartlab Innovationsgesellschaft mbH', + attribution_license='CC-0', + has_realtime_data=True, + ) diff --git a/webapp/services/import_services/datex2/v3_5/virta_datex2_import_service.py b/webapp/services/import_services/datex2/v3_5/virta_datex2_import_service.py new file mode 100644 index 00000000..b19ed38f --- /dev/null +++ b/webapp/services/import_services/datex2/v3_5/virta_datex2_import_service.py @@ -0,0 +1,32 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from webapp.services.import_services.datex2.v3_5.base_datex2_v3_5_import_service import BaseDatex2V35ImportService +from webapp.services.import_services.models import SourceInfo + + +class VirtaDatex2ImportService(BaseDatex2V35ImportService): + source_info = SourceInfo( + uid='datex2_virta', + name='Virta Datex II', + public_url='https://mobilithek.info/offers/1042468553977937920', + source_url='https://mobilithek.info/offers/1042468553977937920', + attribution_contributor='Virta International GmbH', + attribution_license='CC-0', + has_realtime_data=True, + ) diff --git a/webapp/services/import_services/datex2/v3_5/volkswagen_datex2_import_service.py b/webapp/services/import_services/datex2/v3_5/volkswagen_datex2_import_service.py index cbd03843..0523a1df 100644 --- a/webapp/services/import_services/datex2/v3_5/volkswagen_datex2_import_service.py +++ b/webapp/services/import_services/datex2/v3_5/volkswagen_datex2_import_service.py @@ -24,8 +24,8 @@ class VolkswagenDatex2ImportService(BaseDatex2V35ImportService): source_info = SourceInfo( uid='datex2_volkswagen', name='Volkswagen Charging Group Datex II', - public_url='https://mobilithek.info/offers/976221024898781184', - source_url='https://mobilithek.info/offers/976221024898781184', + public_url='https://mobilithek.info/offers/983008874583728128', + source_url='https://mobilithek.info/offers/983008874583728128', attribution_contributor='Volkswagen Charging Group', attribution_license='CC-0', has_realtime_data=True, diff --git a/webapp/services/import_services/datex2/v3_7/base_datex2_v3_7_import_service.py b/webapp/services/import_services/datex2/v3_7/base_datex2_v3_7_import_service.py index adcb778c..710ae850 100644 --- a/webapp/services/import_services/datex2/v3_7/base_datex2_v3_7_import_service.py +++ b/webapp/services/import_services/datex2/v3_7/base_datex2_v3_7_import_service.py @@ -34,9 +34,13 @@ from webapp.services.import_services.datex2.tariff_grouping_mixin import TariffGroupingMixin from webapp.services.import_services.exceptions import ImportException from webapp.services.import_services.models import EvseRealtimeUpdate, LocationUpdate +from webapp.shared.datex2.models import MessageContainerWrapperV37RealtimeInput from webapp.shared.datex2.v3_7.realtime.d_a_t_e_x_i_i3_d2_payload_input import ( DATEXII3D2PayloadInput as DATEXII3D2RealtimePayloadInput, ) +from webapp.shared.datex2.v3_7.realtime.payload_publication_g_input import ( + PayloadPublicationGInput as PayloadPublicationGRealtimeInput, +) from webapp.shared.datex2.v3_7.static.d_a_t_e_x_i_i3_d2_payload_input import ( DATEXII3D2PayloadInput as DATEXII3D2StaticPayloadInput, ) @@ -56,6 +60,7 @@ class RealtimeResult: class BaseDatex2V37ImportService(BaseImportService, TariffGroupingMixin, ABC): v3_7_json_static_datex_validator = DataclassValidator(DATEXII3D2StaticPayloadInput) v3_7_json_realtime_datex_validator = DataclassValidator(DATEXII3D2RealtimePayloadInput) + v3_7_json_realtime_message_container_validator = DataclassValidator(MessageContainerWrapperV37RealtimeInput) v3_7_json_static_datex_mapper = Datex2V37JSONStaticMapper() def fetch_static_data(self): @@ -151,6 +156,10 @@ def fetch_realtime_data(self): try: data, last_modified = self.request_data(subscription_id, if_modified_since) except RemoteException as e: + # Mobilithek answers with 304 Not Modified if there is no newer data + if e.http_status == 304: + last_modified = if_modified_since + break logger.error( f'Datex2 realtime request failed: {e.to_dict()}. Text: {e.data}', extra={'attributes': {'type': LogMessageType.IMPORT_SOURCE}}, @@ -201,45 +210,61 @@ def fetch_realtime_data(self): extra={'attributes': {'type': LogMessageType.IMPORT_LOCATION}}, ) - def add_realtime_data(self, data: dict, result: RealtimeResult) -> DATEXII3D2RealtimePayloadInput: + @classmethod + def get_realtime_payloads(cls, data: dict) -> list[PayloadPublicationGRealtimeInput]: + """ + Realtime data is either a MessageContainer document, as in the Mobilithek AFIR profile, or a D2Payload document. + + Raises ValidationError if the data matches neither of them. + """ + if 'messageContainer' in data: + message_container_input: MessageContainerWrapperV37RealtimeInput = ( + cls.v3_7_json_realtime_message_container_validator.validate(data) + ) + return message_container_input.messageContainer.payload + + datex_input: DATEXII3D2RealtimePayloadInput = cls.v3_7_json_realtime_datex_validator.validate(data) + if datex_input.payload is UnsetValue: + return [] + return [datex_input.payload] + + def add_realtime_data(self, data: dict, result: RealtimeResult) -> None: try: - datex_input: DATEXII3D2RealtimePayloadInput = self.v3_7_json_realtime_datex_validator.validate(data) + payloads = self.get_realtime_payloads(data) except ValidationError as e: raise ImportException( - message=f'missing payload in {self.source_info.uid} realtime data', + message=f'invalid messageContainer or payload in {self.source_info.uid} realtime data', ) from e - if datex_input.payload is UnsetValue: + if not payloads and 'messageContainer' not in data: raise ImportException( message=f'missing payload in {self.source_info.uid} realtime data', ) - payload = datex_input.payload - if ( - payload.aegiEnergyInfrastructureStatusPublication is UnsetValue - or payload.aegiEnergyInfrastructureStatusPublication.energyInfrastructureSiteStatus is UnsetValue - ): - raise ImportException( - message=f'missing aegiEnergyInfrastructureStatusPublication in {self.source_info.uid} realtime data', - ) + for payload in payloads: + if ( + payload.aegiEnergyInfrastructureStatusPublication is UnsetValue + or payload.aegiEnergyInfrastructureStatusPublication.energyInfrastructureSiteStatus is UnsetValue + ): + raise ImportException( + message=f'missing aegiEnergyInfrastructureStatusPublication in {self.source_info.uid} realtime data', + ) - for site_status in payload.aegiEnergyInfrastructureStatusPublication.energyInfrastructureSiteStatus: - if site_status.energyInfrastructureStationStatus is UnsetValue: - continue - for station_status in site_status.energyInfrastructureStationStatus: - if station_status.refillPointStatus is UnsetValue: + for site_status in payload.aegiEnergyInfrastructureStatusPublication.energyInfrastructureSiteStatus: + if site_status.energyInfrastructureStationStatus is UnsetValue: continue - for refill_point_status_g in station_status.refillPointStatus: - evse_update = self.v3_7_json_static_datex_mapper.map_energy_infrastructure_station_status( - refill_point_status_g, - ) - if evse_update is None: - result.realtime_error_count += 1 + for station_status in site_status.energyInfrastructureStationStatus: + if station_status.refillPointStatus is UnsetValue: continue - result.evse_updates_by_evse[evse_update.evse_id] = evse_update - result.realtime_success_count += 1 - - return datex_input + for refill_point_status_g in station_status.refillPointStatus: + evse_update = self.v3_7_json_static_datex_mapper.map_energy_infrastructure_station_status( + refill_point_status_g, + ) + if evse_update is None: + result.realtime_error_count += 1 + continue + result.evse_updates_by_evse[evse_update.evse_id] = evse_update + result.realtime_success_count += 1 def request_data( self, diff --git a/webapp/services/import_services/eliso/__init__.py b/webapp/services/import_services/eliso/__init__.py new file mode 100644 index 00000000..6c7e0051 --- /dev/null +++ b/webapp/services/import_services/eliso/__init__.py @@ -0,0 +1,19 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from .eliso_service import ElisoImportService diff --git a/webapp/services/import_services/eliso/eliso_service.py b/webapp/services/import_services/eliso/eliso_service.py new file mode 100644 index 00000000..20dabd03 --- /dev/null +++ b/webapp/services/import_services/eliso/eliso_service.py @@ -0,0 +1,158 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +import logging +from datetime import datetime, timezone + +from validataclass.exceptions import ValidationError +from validataclass.validators import AnythingValidator, DataclassValidator, ListValidator + +from webapp.common.error_handling.exceptions import RemoteException +from webapp.common.logging.models import LogMessageType +from webapp.common.rest.exceptions import IncompleteConfigException +from webapp.models import SourceStatus +from webapp.services.import_services.base_import_service import BaseImportService +from webapp.services.import_services.models import EvseRealtimeUpdate, LocationUpdate, SourceInfo + +from .eliso_validators import ElisoEvseStatusInput, ElisoLocationInput + +logger = logging.getLogger(__name__) + + +class ElisoImportService(BaseImportService): + """ + eliso publishes its AFIR data at the Mobilithek in a custom JSON format instead of DATEX II. Static data is a list + of locations, realtime data is a snapshot of all EVSE statuses, with one entry per EVSE and price. + """ + + source_info = SourceInfo( + uid='eliso', + name='eliso', + public_url='https://mobilithek.info/offers/843477276990078976', + source_url='https://mobilithek.info/offers/843477276990078976', + attribution_contributor='eliso GmbH', + has_realtime_data=True, + ) + + list_validator = ListValidator(AnythingValidator(allowed_types=[dict])) + location_validator = DataclassValidator(ElisoLocationInput) + evse_status_validator = DataclassValidator(ElisoEvseStatusInput) + + def fetch_static_data(self): + source = self.get_source() + static_data_updated_at = datetime.now(tz=timezone.utc) + + try: + location_dicts = self.list_validator.validate(self.request_data('static_subscription_id')) + except (RemoteException, ValidationError) as e: + logger.error( + f'{self.source_info.uid} static data has error: {e.to_dict()}', + extra={'attributes': {'type': LogMessageType.IMPORT_SOURCE}}, + ) + self.update_source(source, static_status=SourceStatus.FAILED) + return + + location_updates: list[LocationUpdate] = [] + error_count = 0 + for location_dict in location_dicts: + try: + location_input: ElisoLocationInput = self.location_validator.validate(location_dict) + except ValidationError as e: + logger.warning( + f'{self.source_info.uid} location has error: {e.to_dict()}', + extra={'attributes': {'type': LogMessageType.IMPORT_SOURCE}}, + ) + error_count += 1 + continue + location_updates.append(location_input.to_location_update(self.source_info.uid)) + + self.save_location_updates(location_updates) + + self.update_source( + source=source, + static_status=SourceStatus.ACTIVE, + static_error_count=error_count, + static_data_updated_at=static_data_updated_at, + ) + logger.info( + f'Successfully updated {self.source_info.uid} static data with {len(location_updates)} valid ' + f'locations and {error_count} failed locations.', + extra={'attributes': {'type': LogMessageType.IMPORT_LOCATION}}, + ) + + def fetch_realtime_data(self): + source = self.get_source() + if source.static_status != SourceStatus.ACTIVE: + return + + realtime_data_updated_at = datetime.now(tz=timezone.utc) + + try: + data = self.request_data('realtime_subscription_id') + if not isinstance(data, dict): + raise ValidationError(reason='realtime data is no object') + evse_status_dicts = self.list_validator.validate(data.get('evses')) + except (RemoteException, ValidationError) as e: + logger.error( + f'{self.source_info.uid} realtime data has error: {e.to_dict()}', + extra={'attributes': {'type': LogMessageType.IMPORT_SOURCE}}, + ) + self.update_source(source, realtime_status=SourceStatus.FAILED) + return + + # There is one entry per EVSE and price, all with the same status + evse_updates_by_uid: dict[str, EvseRealtimeUpdate] = {} + error_count = 0 + for evse_status_dict in evse_status_dicts: + try: + evse_status_input: ElisoEvseStatusInput = self.evse_status_validator.validate(evse_status_dict) + except ValidationError as e: + logger.warning( + f'{self.source_info.uid} EVSE status has error: {e.to_dict()}', + extra={'attributes': {'type': LogMessageType.IMPORT_SOURCE}}, + ) + error_count += 1 + continue + evse_updates_by_uid[evse_status_input.evseId] = evse_status_input.to_evse_realtime_update() + + self.save_evse_updates(list(evse_updates_by_uid.values())) + + self.update_source( + source=source, + realtime_status=SourceStatus.ACTIVE, + realtime_error_count=error_count, + realtime_data_updated_at=realtime_data_updated_at, + ) + logger.info( + f'Successfully updated {self.source_info.uid} realtime data with {len(evse_updates_by_uid)} valid EVSEs ' + f'and {error_count} failed EVSEs.', + extra={'attributes': {'type': LogMessageType.IMPORT_LOCATION}}, + ) + + def request_data(self, subscription_config_key: str) -> dict | list: + subscription_id: int | None = self.config.get(subscription_config_key) + if subscription_id is None: + raise IncompleteConfigException(message=f'Missing {subscription_config_key} in config') + + return self.json_request( + url=f'https://mobilithek.info:8443/mobilithek/api/v1.0/subscription?subscriptionID={subscription_id}', + cert=( + self.config.get('mobilithek_cert_path'), + self.config.get('mobilithek_key_path'), + ), + ) diff --git a/webapp/services/import_services/eliso/eliso_validators.py b/webapp/services/import_services/eliso/eliso_validators.py new file mode 100644 index 00000000..36bf75ee --- /dev/null +++ b/webapp/services/import_services/eliso/eliso_validators.py @@ -0,0 +1,176 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from datetime import datetime +from decimal import Decimal +from enum import Enum + +from pycountry import countries +from validataclass.dataclasses import Default, validataclass +from validataclass.validators import ( + DataclassValidator, + DateTimeValidator, + EnumValidator, + FloatToDecimalValidator, + IntegerValidator, + ListValidator, + StringValidator, +) + +from webapp.models.connector import ConnectorFormat, ConnectorType, PowerType +from webapp.models.evse import EvseStatus +from webapp.services.import_services.models import ( + BusinessUpdate, + ChargingStationUpdate, + ConnectorUpdate, + EvseRealtimeUpdate, + EvseUpdate, + LocationUpdate, +) + + +class ElisoConnectorType(Enum): + CCS = 'Combo2/CCS (DC)' + TYPE_2 = 'Type 2 (AC)' + + def to_connector_type(self) -> ConnectorType: + return { + self.CCS: ConnectorType.IEC_62196_T2_COMBO, + self.TYPE_2: ConnectorType.IEC_62196_T2, + }.get(self) + + def to_connector_format(self) -> ConnectorFormat: + return { + self.CCS: ConnectorFormat.CABLE, + self.TYPE_2: ConnectorFormat.SOCKET, + }.get(self) + + +class ElisoOperationalStatus(Enum): + OPERATIONAL = 'Operational' + NON_OPERATIONAL = 'Non-operational' + + +class ElisoAvailabilityStatus(Enum): + IN_USE = 'In use' + NOT_IN_USE = 'Not in use' + + +@validataclass +class ElisoConnectorInput: + maxPower: int = IntegerValidator(min_value=0) # in kW + powerType: PowerType = EnumValidator(PowerType) + type_of_connector: ElisoConnectorType = EnumValidator(ElisoConnectorType) + + +@validataclass +class ElisoEvseInput: + evseId: str = StringValidator(min_length=1, max_length=48) + connectors: list[ElisoConnectorInput] = ListValidator(DataclassValidator(ElisoConnectorInput), min_length=1) + + +@validataclass +class ElisoCoordinatesInput: + latitude: Decimal = FloatToDecimalValidator(min_value=-90, max_value=90, allow_integers=True) + longitude: Decimal = FloatToDecimalValidator(min_value=-180, max_value=180, allow_integers=True) + + +@validataclass +class ElisoLocationInput: + address: str = StringValidator(min_length=1) + postalCode: str = StringValidator(min_length=1) + city: str = StringValidator(min_length=1) + country_iso_3166_alpha_2: str = StringValidator(min_length=2, max_length=2) + coordinates: ElisoCoordinatesInput = DataclassValidator(ElisoCoordinatesInput) + operator_name: str = StringValidator(min_length=1) + hotline_number: str | None = StringValidator(), Default(None) + opening_time: str | None = StringValidator(), Default(None) + mobilithek_last_updated_dts: datetime = DateTimeValidator() + evses: list[ElisoEvseInput] = ListValidator(DataclassValidator(ElisoEvseInput), min_length=1) + + def to_location_update(self, source_uid: str) -> LocationUpdate: + # eliso does not provide location ids, so we use the lowest EVSE id as stable identifier + uid = min(evse.evseId for evse in self.evses) + return LocationUpdate( + uid=uid, + source=source_uid, + time_zone='Europe/Berlin', + name=self.address, + address=self.address, + postal_code=self.postalCode, + city=self.city, + country=countries.get(alpha_2=self.country_iso_3166_alpha_2).alpha_3, + lat=self.coordinates.latitude, + lon=self.coordinates.longitude, + operator=BusinessUpdate(name=self.operator_name), + help_phone=self.hotline_number, + twentyfourseven=True if self.opening_time == '24/7' else None, + last_updated=self.mobilithek_last_updated_dts, + charging_pool=[ + ChargingStationUpdate( + uid=uid, + last_updated=self.mobilithek_last_updated_dts, + evses=[ + EvseUpdate( + uid=evse.evseId, + evse_id=evse.evseId, + status=EvseStatus.UNKNOWN, + last_updated=self.mobilithek_last_updated_dts, + connectors=[ + ConnectorUpdate( + uid=str(index), + standard=connector.type_of_connector.to_connector_type(), + format=connector.type_of_connector.to_connector_format(), + power_type=connector.powerType, + max_electric_power=connector.maxPower * 1000, + last_updated=self.mobilithek_last_updated_dts, + ) + for index, connector in enumerate(evse.connectors, start=1) + ], + ) + for evse in self.evses + ], + ), + ], + ) + + +@validataclass +class ElisoEvseStatusInput: + evseId: str = StringValidator(min_length=1, max_length=48) + operational_status: ElisoOperationalStatus = EnumValidator(ElisoOperationalStatus) + availability_status: ElisoAvailabilityStatus | None = EnumValidator(ElisoAvailabilityStatus), Default(None) + mobilithek_last_updated_dts: datetime = DateTimeValidator() + + def to_evse_realtime_update(self) -> EvseRealtimeUpdate: + if self.operational_status == ElisoOperationalStatus.NON_OPERATIONAL: + status = EvseStatus.INOPERATIVE + elif self.availability_status == ElisoAvailabilityStatus.IN_USE: + status = EvseStatus.CHARGING + elif self.availability_status == ElisoAvailabilityStatus.NOT_IN_USE: + status = EvseStatus.AVAILABLE + else: + status = EvseStatus.UNKNOWN + + return EvseRealtimeUpdate( + uid=self.evseId, + evse_id=self.evseId, + status=status, + last_updated=self.mobilithek_last_updated_dts, + status_last_updated=self.mobilithek_last_updated_dts, + ) diff --git a/webapp/services/import_services/import_services.py b/webapp/services/import_services/import_services.py index 43c58a92..d7694412 100644 --- a/webapp/services/import_services/import_services.py +++ b/webapp/services/import_services/import_services.py @@ -35,31 +35,49 @@ from webapp.services.import_services.base_import_service import BaseImportService from webapp.services.import_services.bnetza import BnetzaApiImportService, BnetzaExcelImportService from webapp.services.import_services.datex2 import ( + AhBeteiligungDatex2ImportService, AmpecoDatex2ImportService, + AudiDatex2ImportService, ChargecloudDatex2ImportService, + EaazeDatex2ImportService, EClearingNetDatex2ImportService, EcoMovementDatex2ImportService, EluMobilityDatex2ImportService, EnBWDatex2ImportService, + EnerandoDatex2ImportService, EnioDatex2ImportService, ERoundDatex2ImportService, + EulektroDatex2ImportService, + EvPriceDatex2ImportService, + EweDatex2ImportService, + FlaviaDatex2ImportService, GiroEDatex2ImportService, + GpJouleDatex2ImportService, GridAndCoDatex2ImportService, InergiaDatex2ImportService, + InselwerkeDatex2ImportService, + KulRudnickDatex2ImportService, LadebusinessDatex2ImportService, + LadesonneDatex2ImportService, LichtblickDatex2ImportService, MidorionDatex2ImportService, MontaDatex2ImportService, MsuDatex2ImportService, PumpDatex2ImportService, QwelloDatex2ImportService, + RMBeteiligungDatex2ImportService, + RoadDatex2ImportService, SmatricsDatex2ImportService, + SonnenstromSaarpfalzDatex2ImportService, TaubertDatex2ImportService, TeslaDatex2ImportService, + UlmDatex2ImportService, VaylensDatex2ImportService, + VirtaDatex2ImportService, VolkswagenDatex2ImportService, WirelaneDatex2ImportService, ) +from webapp.services.import_services.eliso import ElisoImportService from webapp.services.import_services.giroe import GiroeImportService from webapp.services.import_services.goldbeck_ipcm import HeilbronnNeckarbogenImportService from webapp.services.import_services.lichtblick import LichtblickImportService @@ -82,25 +100,38 @@ class ImportServices(BaseService): importer_by_uid: dict[str, BaseImportService] = {} importer_classes: list[type[BaseImportService]] = [ + AhBeteiligungDatex2ImportService, AlbwerkOchpImportService, AmpecoDatex2ImportService, + AudiDatex2ImportService, BnetzaApiImportService, BnetzaExcelImportService, ChargecloudDatex2ImportService, + EaazeDatex2ImportService, EaazePbwImportService, EClearingNetDatex2ImportService, EcoMovementDatex2ImportService, + ElisoImportService, EluMobilityDatex2ImportService, EnBWDatex2ImportService, + EnerandoDatex2ImportService, EnioDatex2ImportService, ERoundDatex2ImportService, + EulektroDatex2ImportService, + EvPriceDatex2ImportService, + EweDatex2ImportService, + FlaviaDatex2ImportService, GiroEDatex2ImportService, GiroeImportService, + GpJouleDatex2ImportService, GridAndCoDatex2ImportService, HeilbronnNeckarbogenImportService, InergiaDatex2ImportService, + InselwerkeDatex2ImportService, + KulRudnickDatex2ImportService, LadebusinessDatex2ImportService, LadenetzOchpImportService, + LadesonneDatex2ImportService, LichtblickDatex2ImportService, LichtblickImportService, LudwigsburgImportService, @@ -111,13 +142,18 @@ class ImportServices(BaseService): PforzheimImportService, PumpDatex2ImportService, QwelloDatex2ImportService, + RMBeteiligungDatex2ImportService, + RoadDatex2ImportService, SmatricsDatex2ImportService, + SonnenstromSaarpfalzDatex2ImportService, StadtnaviImportService, SWStuttgartImportService, TaubertDatex2ImportService, TeslaDatex2ImportService, TuebingenImportService, + UlmDatex2ImportService, VaylensDatex2ImportService, + VirtaDatex2ImportService, VolkswagenDatex2ImportService, WirelaneDatex2ImportService, ] diff --git a/webapp/shared/datex2/models.py b/webapp/shared/datex2/models.py index 15f7780c..4caaaa6d 100644 --- a/webapp/shared/datex2/models.py +++ b/webapp/shared/datex2/models.py @@ -21,7 +21,6 @@ from validataclass.dataclasses import Default, ValidataclassMixin, validataclass from validataclass.validators import ( - AnythingValidator, DataclassValidator, DateTimeValidator, EnumValidator, @@ -110,7 +109,7 @@ class DynamicInformationInput(ValidataclassMixin): exchangeStatus: ExchangeStatusEnumGInput = DataclassValidator(ExchangeStatusEnumGInput) messageGenerationTimestamp: datetime = DateTimeValidator() exDynamicInformationExtensionG: ExtensionTypeGInput | None = ( - Noneable(AnythingValidator(allowed_types=[ExtensionTypeGInput])), + Noneable(DataclassValidator(ExtensionTypeGInput)), Default(None), ) @@ -163,3 +162,30 @@ class MessageContainerWrapperInput(ValidataclassMixin): Noneable(DataclassValidator(ExchangeInformationInput)), Default(None), ) + + +@validataclass(reject_unknown_fields=True) +class MessageContainerV37RealtimeInput(ValidataclassMixin): + payload: list[PayloadPublicationGV37RealtimeInput] = ( + ListValidator(DataclassValidator(PayloadPublicationGV37RealtimeInput)), + Default([]), + ) + exchangeInformation: ExchangeInformationInput = DataclassValidator(ExchangeInformationInput) + conMessageContainerExtensionG: ExtensionTypeGInput | None = ( + Noneable(DataclassValidator(ExtensionTypeGInput)), + Default(None), + ) + + +@validataclass(reject_unknown_fields=True) +class MessageContainerWrapperV37RealtimeInput(ValidataclassMixin): + """ + DATEX II v3.7 realtime data as MessageContainer document, as used in the Mobilithek AFIR profile. Unlike + MessageContainerWrapperInput, unknown fields are rejected, so payloads outside the messageContainer are invalid. + """ + + messageContainer: MessageContainerV37RealtimeInput = DataclassValidator(MessageContainerV37RealtimeInput) + exchangeInformation: ExchangeInformationInput | None = ( + Noneable(DataclassValidator(ExchangeInformationInput)), + Default(None), + ) From 4637a6026225886f047ae8fedabec02a1ee90bbb Mon Sep 17 00:00:00 2001 From: Ernesto Ruge Date: Sun, 1 Nov 2026 08:07:48 +0100 Subject: [PATCH 2/2] mobilithek helper --- README.md | 6 + dev/mobilithek_helper/list_charging_offers.py | 266 +++++++++++++++ dev/mobilithek_helper/list_subscriptions.py | 125 +++++++ dev/mobilithek_helper/mobilithek_client.py | 124 +++++++ dev/mobilithek_helper/send_reminder.py | 170 ++++++++++ dev/mobilithek_helper/subscribe.py | 39 +++ dev/mobilithek_helper/unsubscribe.py | 38 +++ dev/mobilithek_helper/validate_datex2.py | 304 ++++++++++++++++++ 8 files changed, 1072 insertions(+) create mode 100644 dev/mobilithek_helper/list_charging_offers.py create mode 100644 dev/mobilithek_helper/list_subscriptions.py create mode 100644 dev/mobilithek_helper/mobilithek_client.py create mode 100644 dev/mobilithek_helper/send_reminder.py create mode 100644 dev/mobilithek_helper/subscribe.py create mode 100644 dev/mobilithek_helper/unsubscribe.py create mode 100644 dev/mobilithek_helper/validate_datex2.py diff --git a/README.md b/README.md index 95e43522..3015d08c 100644 --- a/README.md +++ b/README.md @@ -79,8 +79,14 @@ python dev/mobilithek_helper/list_charging_offers.py # all charging data offer python dev/mobilithek_helper/list_subscriptions.py # all subscriptions of the account python dev/mobilithek_helper/subscribe.py [ ...] python dev/mobilithek_helper/unsubscribe.py [ ...] +python dev/mobilithek_helper/send_reminder.py [--dry-run] --to [ ...] ``` +`send_reminder.py` sends a German plain text email to the contact person of a provider, asking to approve pending +subscriptions. The Mobilithek does not expose contact addresses of other organisations, so the recipient has to be given +explicitly. It reads `SMTP_HOST`, `SMTP_PORT` (465 uses SSL, otherwise STARTTLS), `SMTP_USER`, `SMTP_PASSWORD` and +`SMTP_FROM` from `.env`. + `validate_datex2.py` validates the data of a subscription against the DATEX II v3.5 and v3.7 import services without touching the database. It shows which base class fits, why an import fails and how many realtime EVSE statuses match static EVSEs. It needs the webapp and the Mobilithek machine certificate in `files/`, so it runs in the flask container: diff --git a/dev/mobilithek_helper/list_charging_offers.py b/dev/mobilithek_helper/list_charging_offers.py new file mode 100644 index 00000000..cd3423b2 --- /dev/null +++ b/dev/mobilithek_helper/list_charging_offers.py @@ -0,0 +1,266 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Writes all charging data offers of the Mobilithek as CSV, with the subscription status of the Mobilithek account and the +OCPDB import service covering the offer, if there is one. Afterwards, it prints the ids of all brokered charging data +offers without an active or requested subscription, which can be passed to subscribe.py. + +Usage: python dev/mobilithek_helper/list_charging_offers.py [] +Without output file, the CSV is written to temp/mobilithek_charging_offers.csv. +""" + +import csv +import sys +from collections import Counter, defaultdict +from pathlib import Path + +from mobilithek_client import MobilithekClient + +DEFAULT_OUTPUT_PATH = Path(__file__).parents[2] / 'temp' / 'mobilithek_charging_offers.csv' + +CHARGING_CATEGORY = 'https://w3id.org/mdp/schema/data_categories#FILLING_AND_CHARGING_STATIONS' + +# Charging data offers which the publisher filed in another data category +EXTRA_DATA_OFFER_IDS: list[str] = [ + '974006122901856256', # AFIR-recharging-dyn-Eulektro, filed as PARKING_AND_REST_AREA_INFORMATION +] + +MATCH_MOBILITHEK = 'Mobilithek import service' +MATCH_OTHER_SOURCE = 'other source, same data' +MATCH_OTHER_SOURCE_LIKELY = 'other source, likely same data' +MATCH_OWN_PUBLICATION = 'own publication' + +# The Mobilithek import services just know their subscription ids from config, and the other import services fetch +# their data from the original source, so this mapping has to be maintained by hand. The 'likely' matches are guessed +# from publisher and description of the offer. +CONVERTER_BY_DATA_OFFER_ID: dict[str, tuple[str, str]] = { + '972837891969273856': ('datex2_ampeco', MATCH_MOBILITHEK), + '972842599324557312': ('datex2_ampeco', MATCH_MOBILITHEK), + '978597062404620288': ('datex2_chargecloud', MATCH_MOBILITHEK), + '980862594474274816': ('datex2_chargecloud', MATCH_MOBILITHEK), + '996825300704600064': ('datex2_e_clearing_net', MATCH_MOBILITHEK), + '996823601386508288': ('datex2_e_clearing_net', MATCH_MOBILITHEK), + '907574882292453376': ('datex2_enbw', MATCH_MOBILITHEK), + '907575401287241728': ('datex2_enbw', MATCH_MOBILITHEK), + '961625658278940672': ('datex2_eround', MATCH_MOBILITHEK), + '961629419076456448': ('datex2_eround', MATCH_MOBILITHEK), + '954064102947180544': ('datex2_ecomovement', MATCH_MOBILITHEK), + '955166494396665856': ('datex2_ecomovement', MATCH_MOBILITHEK), + '936298491949047808': ('datex2_elu_mobility', MATCH_MOBILITHEK), + '971513500454850560': ('datex2_elu_mobility', MATCH_MOBILITHEK), + '963766220171735040': ('datex2_enio', MATCH_MOBILITHEK), + '968541134128902144': ('datex2_enio', MATCH_MOBILITHEK), + # giroe fetches the same data directly from GLS Mobility + '980559859451379712': ('datex2_giroe; giroe', MATCH_MOBILITHEK), + '980563757096464384': ('datex2_giroe; giroe', MATCH_MOBILITHEK), + '984104561811357696': ('datex2_gridandco', MATCH_MOBILITHEK), + '984103903968534528': ('datex2_gridandco', MATCH_MOBILITHEK), + '962721207430316032': ('datex2_lichtblick', MATCH_MOBILITHEK), + '962731482363617280': ('datex2_lichtblick', MATCH_MOBILITHEK), + '979435003058024448': ('datex2_midorion', MATCH_MOBILITHEK), + '979433370303545344': ('datex2_midorion', MATCH_MOBILITHEK), + '963836072152719360': ('datex2_monta', MATCH_MOBILITHEK), + '963870983660167168': ('datex2_monta', MATCH_MOBILITHEK), + '970305056590979072': ('datex2_msu', MATCH_MOBILITHEK), + '970388804493828096': ('datex2_msu', MATCH_MOBILITHEK), + '969322788846231552': ('datex2_pump', MATCH_MOBILITHEK), + '972963216296222720': ('datex2_qwello', MATCH_MOBILITHEK), + '972966368902897664': ('datex2_qwello', MATCH_MOBILITHEK), + '980809280034390016': ('datex2_smatrics', MATCH_MOBILITHEK), + '961319990963605504': ('datex2_smatrics', MATCH_MOBILITHEK), + '1000749522967314432': ('datex2_taubert', MATCH_MOBILITHEK), + '1000746576770711552': ('datex2_taubert', MATCH_MOBILITHEK), + '953828817873125376': ('datex2_tesla', MATCH_MOBILITHEK), + '953843379766972416': ('datex2_tesla', MATCH_MOBILITHEK), + '979363267193040896': ('datex2_vaylens', MATCH_MOBILITHEK), + '979364650281549824': ('datex2_vaylens', MATCH_MOBILITHEK), + '983008874583728128': ('datex2_volkswagen', MATCH_MOBILITHEK), + '983006934617260032': ('datex2_volkswagen', MATCH_MOBILITHEK), + '869246425829892096': ('datex2_wirelane', MATCH_MOBILITHEK), + '876587237907525632': ('datex2_wirelane', MATCH_MOBILITHEK), + '998571342932365312': ('datex2_audi', MATCH_MOBILITHEK), + '998567365272563712': ('datex2_audi', MATCH_MOBILITHEK), + # eaaze_pbw fetches the PBW part of the EAAZE data via OCPI + '980426161955500032': ('datex2_eaaze; eaaze_pbw', MATCH_MOBILITHEK), + '980425456721260544': ('datex2_eaaze; eaaze_pbw', MATCH_MOBILITHEK), + '1037036344672452608': ('datex2_enerando', MATCH_MOBILITHEK), + '1037038702374617088': ('datex2_enerando', MATCH_MOBILITHEK), + '973938793916375040': ('datex2_eulektro', MATCH_MOBILITHEK), + '974006122901856256': ('datex2_eulektro', MATCH_MOBILITHEK), + '993495185148837888': ('datex2_ah_beteiligung', MATCH_MOBILITHEK), + '993494022504546304': ('datex2_ah_beteiligung', MATCH_MOBILITHEK), + '981180413145477120': ('datex2_inselwerke', MATCH_MOBILITHEK), + '981170994693238784': ('datex2_inselwerke', MATCH_MOBILITHEK), + '988133177339846656': ('datex2_ladesonne', MATCH_MOBILITHEK), + '988137788909801472': ('datex2_ladesonne', MATCH_MOBILITHEK), + '993499364131065856': ('datex2_r_m_beteiligung', MATCH_MOBILITHEK), + '993498817080557568': ('datex2_r_m_beteiligung', MATCH_MOBILITHEK), + '979427073235566592': ('datex2_sonnenstrom_saarpfalz', MATCH_MOBILITHEK), + '979428785661079552': ('datex2_sonnenstrom_saarpfalz', MATCH_MOBILITHEK), + '989311807176560640': ('datex2_ev_price', MATCH_MOBILITHEK), + '989311073915731968': ('datex2_ev_price', MATCH_MOBILITHEK), + '1024264572575293440': ('datex2_ewe', MATCH_MOBILITHEK), + '1006184570533171200': ('datex2_ewe', MATCH_MOBILITHEK), + '1004450433724211200': ('datex2_flavia', MATCH_MOBILITHEK), + '1004756865530970112': ('datex2_flavia', MATCH_MOBILITHEK), + '997111469996658688': ('datex2_gp_joule', MATCH_MOBILITHEK), + '997190851637440512': ('datex2_gp_joule', MATCH_MOBILITHEK), + '1043944773714128896': ('datex2_inergia', MATCH_MOBILITHEK), + '1043947032435105792': ('datex2_inergia', MATCH_MOBILITHEK), + '983747474988376064': ('datex2_kul_rudnick', MATCH_MOBILITHEK), + '1021366453630017536': ('datex2_road', MATCH_MOBILITHEK), + '1021364001002217472': ('datex2_road', MATCH_MOBILITHEK), + '1042468553977937920': ('datex2_virta', MATCH_MOBILITHEK), + '1042472664437800960': ('datex2_virta', MATCH_MOBILITHEK), + '854410608351543296': ('datex2_ulm', MATCH_MOBILITHEK), + '854416606814023680': ('datex2_ulm', MATCH_MOBILITHEK), + # eliso uses a custom JSON format instead of DATEX II + '843477276990078976': ('eliso', MATCH_MOBILITHEK), + '843502085052710912': ('eliso', MATCH_MOBILITHEK), + '951517095896416256': ('bnetza_api', MATCH_OTHER_SOURCE_LIKELY), + '842113170303512576': ('bnetza_excel', MATCH_OTHER_SOURCE_LIKELY), + # Bundled data of MobiData BW, which is the output of OCPDB itself + '-2035239451501556619': ('', MATCH_OWN_PUBLICATION), + '980084361378099200': ('', MATCH_OWN_PUBLICATION), + '981124859836461056': ('', MATCH_OWN_PUBLICATION), +} + +# Offers in the charging category which are no charging data sources, so we don't subscribe to them +IGNORE_REASON_BY_DATA_OFFER_ID: dict[str, str] = { + '935580278881759232': 'fuel station', + '937348385824550912': 'fuel station', + '753618942489935872': 'fuel station', + '834870693179846656': 'fuel station', + '968150797115953152': 'test data', + '981218494322012160': 'test data', + '981223821767557120': 'test data', + '1006136734156312576': 'test data', + '755033335157133312': 'statistical analysis', + '1033716358314917888': 'organisation data only', + '980084361378099200': 'own publication', + '981124859836461056': 'own publication', +} + +# Subscriptions in these states don't need a new subscription +OPEN_SUBSCRIPTION_STATUSES = ('ACTIVE', 'REQUESTED') + +FIELDS = [ + 'publicationId', + 'title', + 'publisher', + 'dataModel', + 'license', + 'mobilithekBrokering', + 'region', + 'subscriptionStatus', + 'subscriptionId', + 'converter', + 'converterMatch', + 'ignoreReason', + 'url', +] + + +def short_uri(uri: str | None) -> str: + """ + Strips vocabulary URIs like https://w3id.org/mdp/schema/license#NO_LICENSE down to NO_LICENSE + """ + if not uri: + return '' + return uri.rsplit('#', 1)[-1] + + +def get_row(data_offer: dict, subscriptions: list[dict]) -> dict: + publication_id = data_offer['publicationId'] + converter, converter_match = CONVERTER_BY_DATA_OFFER_ID.get(publication_id, ('', '')) + data_models = { + short_uri(content_data.get('dataModel')) + for content_data in data_offer.get('contentData') or [] + if content_data.get('dataModel') + } + return { + 'publicationId': publication_id, + 'title': data_offer['title'].strip(), + 'publisher': data_offer['agents']['publisher']['name'], + 'dataModel': '; '.join(sorted(data_models)), + 'license': short_uri(data_offer.get('rights')), + # Brokered offers are delivered by the Mobilithek itself and can be pulled with the machine certificate + 'mobilithekBrokering': data_offer.get('mdpBrokering'), + 'region': '; '.join(nuts['name'] for nuts in (data_offer.get('spatialCoverage') or {}).get('nuts') or []), + 'subscriptionStatus': '; '.join(subscription['contractStatus'] for subscription in subscriptions), + 'subscriptionId': '; '.join(subscription['id'] for subscription in subscriptions), + 'converter': converter, + 'converterMatch': converter_match, + 'ignoreReason': IGNORE_REASON_BY_DATA_OFFER_ID.get(publication_id, ''), + 'url': f'https://mobilithek.info/offers/{publication_id}', + } + + +def is_missing_subscription(row: dict) -> bool: + return ( + row['mobilithekBrokering'] is True + and not row['ignoreReason'] + and not any(status in OPEN_SUBSCRIPTION_STATUSES for status in row['subscriptionStatus'].split('; ')) + ) + + +def list_charging_offers(output_path: Path): + client = MobilithekClient() + data_offers = client.search_data_offers([CHARGING_CATEGORY]) + known_data_offer_ids = {data_offer['publicationId'] for data_offer in data_offers} + data_offers += [ + client.get_data_offer_by_id(data_offer_id) + for data_offer_id in EXTRA_DATA_OFFER_IDS + if data_offer_id not in known_data_offer_ids + ] + data_offers.sort(key=lambda data_offer: (data_offer['agents']['publisher']['name'], data_offer['title'])) + + subscriptions_by_data_offer_id: dict[str, list[dict]] = defaultdict(list) + for subscription in client.get_subscriptions(): + subscriptions_by_data_offer_id[subscription['dataOfferId']].append(subscription) + + rows = [ + get_row(data_offer, subscriptions_by_data_offer_id[data_offer['publicationId']]) for data_offer in data_offers + ] + + output_path.parent.mkdir(parents=True, exist_ok=True) + with output_path.open('w', newline='', encoding='utf-8') as output_file: + writer = csv.DictWriter(output_file, fieldnames=FIELDS) + writer.writeheader() + writer.writerows(rows) + + print(f'{len(rows)} charging data offers written to {output_path}') + overview = Counter( + ( + 'converter' if row['converter'] or row['converterMatch'] else 'no converter', + row['subscriptionStatus'] or 'no subscription', + ) + for row in rows + ) + for (converter, subscription_status), count in sorted(overview.items()): + print(f' {converter}, {subscription_status}: {count}') + + missing_rows = [row for row in rows if is_missing_subscription(row)] + print(f'{len(missing_rows)} brokered charging data offers without active or requested subscription:') + for row in missing_rows: + print(f' {row["publicationId"]} {row["publisher"]}: {row["title"]}') + if missing_rows: + print(f'subscribe with: subscribe.py {" ".join(row["publicationId"] for row in missing_rows)}') + + +if __name__ == '__main__': + list_charging_offers(Path(sys.argv[1]) if len(sys.argv) > 1 else DEFAULT_OUTPUT_PATH) diff --git a/dev/mobilithek_helper/list_subscriptions.py b/dev/mobilithek_helper/list_subscriptions.py new file mode 100644 index 00000000..f72448c1 --- /dev/null +++ b/dev/mobilithek_helper/list_subscriptions.py @@ -0,0 +1,125 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Writes all subscriptions of the Mobilithek account as CSV, including their status (REQUESTED, ACTIVE, INACTIVE) and +metadata of the subscribed data offer like licence, category and data model. For restricted or withdrawn data offers, +only parts of the metadata are available, see column metadataComplete. + +Usage: python dev/mobilithek_helper/list_subscriptions.py [] +Without output file, the CSV is written to temp/mobilithek_subscriptions.csv. +""" + +import csv +import sys +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +from mobilithek_client import MobilithekClient + +DEFAULT_OUTPUT_PATH = Path(__file__).parents[2] / 'temp' / 'mobilithek_subscriptions.csv' + +SUBSCRIPTION_FIELDS = [ + 'id', + 'dataOfferId', + 'dataOfferTitle', + 'providerOrgName', + 'contractStatus', + 'providerApproved', + 'napApproved', + 'validFrom', + 'validTo', + 'deliveryMode', + 'currentAccessCount', + 'maxAccessCount', +] + +DATA_OFFER_FIELDS = [ + 'metadataComplete', + 'license', + 'standardLicense', + 'customLicense', + 'dataCategory', + 'dataCategoryDetail', + 'themes', + 'transportMode', + 'dataModel', + 'accrualPeriodicity', + 'spatialCoverage', + 'keywords', + 'dataOfferModified', + 'irregularSince', +] + + +def short_uri(uri: str | None) -> str: + """ + Strips vocabulary URIs like https://w3id.org/mdp/schema/license#NO_LICENSE down to NO_LICENSE + """ + if not uri: + return '' + return uri.rsplit('#', 1)[-1] if '#' in uri else uri.rsplit('/', 1)[-1] + + +def join_values(values: list | None) -> str: + return '; '.join(str(value) for value in values or [] if value) + + +def get_data_offer_row(data_offer: dict, metadata_complete: bool) -> dict: + custom_license = data_offer.get('customLicenseAsResource') or {} + return { + 'metadataComplete': metadata_complete, + 'license': short_uri(data_offer.get('rights')), + 'standardLicense': data_offer.get('standardLicense') or '', + 'customLicense': (custom_license.get('instance') or {}).get('fileName') or '', + 'dataCategory': short_uri(data_offer.get('dataCategory')), + 'dataCategoryDetail': join_values([short_uri(uri) for uri in data_offer.get('dataCategoryDetail') or []]), + 'themes': join_values([short_uri(uri) for uri in data_offer.get('themes') or []]), + 'transportMode': join_values([short_uri(uri) for uri in data_offer.get('transportMode') or []]), + 'dataModel': join_values([short_uri(item.get('dataModel')) for item in data_offer.get('contentData') or []]), + 'accrualPeriodicity': short_uri(data_offer.get('accrualPeriodicity')), + 'spatialCoverage': join_values( + [nuts.get('name') for nuts in (data_offer.get('spatialCoverage') or {}).get('nuts') or []], + ), + 'keywords': join_values(data_offer.get('keyword')), + 'dataOfferModified': data_offer.get('modified') or '', + 'irregularSince': data_offer.get('irregularSince') or '', + } + + +def list_subscriptions(output_path: Path): + client = MobilithekClient() + subscriptions = client.get_subscriptions() + subscriptions.sort(key=lambda subscription: (subscription['providerOrgName'] or '', subscription['dataOfferTitle'])) + + # One request per subscription, so we parallelize a bit + with ThreadPoolExecutor(max_workers=8) as executor: + data_offers = executor.map(client.get_data_offer, subscriptions) + rows = [ + {**subscription, **get_data_offer_row(data_offer, metadata_complete)} + for subscription, (data_offer, metadata_complete) in zip(subscriptions, data_offers, strict=True) + ] + + with output_path.open('w', newline='', encoding='utf-8') as output_file: + writer = csv.DictWriter(output_file, fieldnames=SUBSCRIPTION_FIELDS + DATA_OFFER_FIELDS, extrasaction='ignore') + writer.writeheader() + writer.writerows(rows) + + print(f'{len(rows)} subscriptions written to {output_path}') + + +if __name__ == '__main__': + list_subscriptions(Path(sys.argv[1]) if len(sys.argv) > 1 else DEFAULT_OUTPUT_PATH) diff --git a/dev/mobilithek_helper/mobilithek_client.py b/dev/mobilithek_helper/mobilithek_client.py new file mode 100644 index 00000000..6917b504 --- /dev/null +++ b/dev/mobilithek_helper/mobilithek_client.py @@ -0,0 +1,124 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +import os +from pathlib import Path + +import requests +from dotenv import load_dotenv + +TOKEN_URL = 'https://mobilithek.info/auth/realms/MDP/protocol/openid-connect/token' +CONTRACTS_URL = 'https://mobilithek.info/mdp-api/mdp-msa-contracts/v1/contracts' +OFFERS_URL = 'https://mobilithek.info/mdp-api/mdp-msa-metadata/v2/offers' + + +class MobilithekClient: + """ + Client for the (undocumented) API behind the Mobilithek web GUI. Managing subscriptions needs a user login, the + machine certificate is not sufficient. Credentials are read from MOBILITHEK_USER and MOBILITHEK_PASSWORD in .env. + """ + + def __init__(self): + load_dotenv(Path(__file__).parents[2] / '.env') + response = requests.post( + TOKEN_URL, + data={ + 'grant_type': 'password', + 'client_id': 'Platform', + 'username': os.environ['MOBILITHEK_USER'], + 'password': os.environ['MOBILITHEK_PASSWORD'], + }, + timeout=30, + ) + response.raise_for_status() + + self.session = requests.Session() + self.session.headers['Authorization'] = f'Bearer {response.json()["access_token"]}' + + def get_subscriptions(self) -> list[dict]: + subscriptions: list[dict] = [] + page = 0 + while True: + response = self.session.post( + f'{CONTRACTS_URL}/search', + json={}, + params={'page': page, 'size': 100}, + timeout=30, + ) + response.raise_for_status() + data = response.json() + subscriptions += data['content'] + if data['last']: + return subscriptions + page += 1 + + def search_data_offers(self, categories: list[str]) -> list[dict]: + """ + Returns all data offers in the given data categories, e.g. data_categories#FILLING_AND_CHARGING_STATIONS. + Logged in, this includes restricted offers which are not visible in the public search. + """ + data_offers: list[dict] = [] + page = 0 + while True: + # Without explicit sorting, the order is not stable between pages, so offers get duplicated or skipped + response = self.session.post( + f'{OFFERS_URL}/search', + json={'categories': categories}, + params={'page': page, 'size': 1000, 'sort': 'created,desc'}, + timeout=60, + ) + response.raise_for_status() + data = response.json()['dataOffers'] + data_offers += data['content'] + if data['last']: + return data_offers + page += 1 + + def get_data_offer_by_id(self, data_offer_id: str) -> dict: + response = self.session.get(f'{OFFERS_URL}/{data_offer_id}', timeout=30) + response.raise_for_status() + return response.json() + + def get_data_offer(self, subscription: dict) -> tuple[dict, bool]: + """ + Returns the data offer of a subscription and whether its metadata is complete. Restricted or withdrawn offers + are just available via the subscription, which only contains parts of the metadata, e.g. the licence. + """ + response = self.session.get(f'{OFFERS_URL}/{subscription["dataOfferId"]}', timeout=30) + if response.ok: + return response.json(), True + if response.status_code not in (403, 404): + response.raise_for_status() + + response = self.session.get(f'{CONTRACTS_URL}/{subscription["id"]}/with-offer', timeout=30) + response.raise_for_status() + return response.json()['dataOffer'], False + + def subscribe(self, data_offer_id: str) -> str: + response = self.session.post( + CONTRACTS_URL, + json={'dataOfferId': data_offer_id, 'relevantToMDVPBefG': False}, + timeout=30, + ) + response.raise_for_status() + # The API returns the new subscription id as plain body + return response.text + + def unsubscribe(self, subscription_id: str): + response = self.session.delete(f'{CONTRACTS_URL}/{subscription_id}', timeout=30) + response.raise_for_status() diff --git a/dev/mobilithek_helper/send_reminder.py b/dev/mobilithek_helper/send_reminder.py new file mode 100644 index 00000000..1fca4ca4 --- /dev/null +++ b/dev/mobilithek_helper/send_reminder.py @@ -0,0 +1,170 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Sends a German plain text reminder to the contact person of a data provider, asking to approve our pending +subscriptions of the given charging data offers. All data offers have to belong to the same provider and have a +REQUESTED subscription. + +The Mobilithek does not expose contact email addresses of other organisations, so the recipient has to be given +explicitly. SMTP credentials are read from .env: SMTP_HOST, SMTP_PORT (465 uses SSL, everything else STARTTLS), +SMTP_USER, SMTP_PASSWORD and SMTP_FROM. + +Usage: python dev/mobilithek_helper/send_reminder.py [--dry-run] --to [--to ...] [--cc ...] + [ ...] +""" + +import argparse +import os +import smtplib +import ssl +import sys +import textwrap +from datetime import datetime +from email.message import EmailMessage +from email.utils import formatdate, make_msgid + +from mobilithek_client import MobilithekClient + +SUBJECT = 'Mobilithek: Bitte um Freigabe unserer Abonnement-Anfrage' + + +def format_request_date(subscription: dict) -> str: + if not subscription.get('createdAt'): + return 'Datum unbekannt' + return datetime.fromisoformat(subscription['createdAt'].replace('Z', '+00:00')).strftime('%d.%m.%Y') + + +def build_body(provider_name: str, subscriptions: list[dict]) -> str: + plural = len(subscriptions) > 1 + offer_lines = '\n'.join( + f'- {subscription["dataOfferTitle"].strip()} (angefragt am {format_request_date(subscription)})\n' + f' https://mobilithek.info/offers/{subscription["dataOfferId"]}' + for subscription in subscriptions + ) + sender_name = f'{subscriptions[0]["subscriberUserFirstName"]} {subscriptions[0]["subscriberUserLastName"]}'.strip() + + paragraphs = [ + 'Guten Tag,', + ( + 'im Auftrag der NVBW – Nahverkehrsgesellschaft Baden-Württemberg mbH betreiben wir die ' + 'Ladeinfrastruktur-Datenbank von MobiData BW, der Mobilitätsdatenplattform des Landes Baden-Württemberg. ' + 'Dort führen wir die Daten öffentlich zugänglicher Ladepunkte aus vielen Quellen zusammen und stellen sie ' + 'als Open Data für Apps, Routenplaner und Reiseinformation bereit:' + ), + 'https://mobidata-bw.de', + ( + f'Gerne würden wir prüfen, ob sich auch die Daten von {provider_name} für eine Aufnahme eignen. Dazu ' + f'haben wir über die Mobilithek {"folgende Datenangebote" if plural else "folgendes Datenangebot"} ' + 'abonniert:' + ), + offer_lines, + ( + f'{"Diese Anfragen sind" if plural else "Diese Anfrage ist"} bisher noch nicht freigegeben. Wir würden uns ' + f'sehr freuen, wenn Sie sie in der Mobilithek unter „Organisation“ → „Abonnements“ ' + 'bestätigen könnten:' + ), + 'https://mobilithek.info/organisation/subscriptions', + ( + 'Zum Hintergrund: Nach Art. 20 der Verordnung (EU) 2023/1804 über den Aufbau der Infrastruktur für ' + 'alternative Kraftstoffe (AFIR) müssen Betreiber öffentlich zugänglicher Ladepunkte statische und dynamische ' + 'Daten zu ihren Ladepunkten kostenlos und diskriminierungsfrei über den nationalen Zugangspunkt ' + 'bereitstellen. In Deutschland ist das die Mobilithek. Solange ein Abonnement nicht freigegeben ist, sind die ' + 'Daten für Nutzer jedoch nicht abrufbar.' + ), + ( + 'Falls Sie Fragen haben oder die Anfrage an eine andere Stelle in Ihrem Haus gehen sollte, melden Sie sich ' + 'gerne.' + ), + f'Vielen Dank und viele Grüße\n{sender_name}', + ] + # The offer list is already formatted, everything else is wrapped for plain text email clients + return ( + '\n\n'.join( + paragraph if paragraph == offer_lines or '\n' in paragraph else textwrap.fill(paragraph, width=72) + for paragraph in paragraphs + ) + + '\n' + ) + + +def get_pending_subscriptions(client: MobilithekClient, data_offer_ids: list[str]) -> list[dict]: + subscriptions_by_data_offer_id = { + subscription['dataOfferId']: subscription + for subscription in client.get_subscriptions() + if subscription['contractStatus'] == 'REQUESTED' + } + missing = [data_offer_id for data_offer_id in data_offer_ids if data_offer_id not in subscriptions_by_data_offer_id] + if missing: + sys.exit(f'No REQUESTED subscription for data offer(s): {", ".join(missing)}') + + subscriptions = [subscriptions_by_data_offer_id[data_offer_id] for data_offer_id in data_offer_ids] + providers = {subscription['providerOrgName'] for subscription in subscriptions} + if len(providers) > 1: + sys.exit(f'Data offers belong to different providers: {", ".join(sorted(providers))}') + return subscriptions + + +def send(message: EmailMessage): + host = os.environ['SMTP_HOST'] + port = int(os.environ.get('SMTP_PORT', '587')) + context = ssl.create_default_context() + if port == 465: + smtp = smtplib.SMTP_SSL(host, port, context=context, timeout=30) + else: + smtp = smtplib.SMTP(host, port, timeout=30) + smtp.starttls(context=context) + with smtp: + smtp.login(os.environ['SMTP_USER'], os.environ['SMTP_PASSWORD']) + smtp.send_message(message) + + +def send_reminder(data_offer_ids: list[str], to: list[str], cc: list[str], dry_run: bool): + # MobilithekClient loads .env, so the SMTP variables are available afterwards + client = MobilithekClient() + subscriptions = get_pending_subscriptions(client, data_offer_ids) + + message = EmailMessage() + message['Subject'] = SUBJECT + message['From'] = os.environ.get('SMTP_FROM', '' if dry_run else os.environ['SMTP_FROM']) + message['To'] = ', '.join(to) + if cc: + message['Cc'] = ', '.join(cc) + message['Date'] = formatdate(localtime=True) + message['Message-ID'] = make_msgid() + message.set_content(build_body(subscriptions[0]['providerOrgName'], subscriptions), charset='utf-8') + + if dry_run: + print(render(message)) + return + + send(message) + print(f'Reminder for {len(subscriptions)} data offer(s) sent to {", ".join(to + cc)}') + + +def render(message: EmailMessage) -> str: + headers = '\n'.join(f'{key}: {value}' for key, value in message.items() if key in ('From', 'To', 'Cc', 'Subject')) + return f'{headers}\n\n{message.get_content()}' + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Sends a reminder to approve pending Mobilithek subscriptions.') + parser.add_argument('data_offer_ids', nargs='+', metavar='data offer id') + parser.add_argument('--to', action='append', required=True, help='recipient, can be given multiple times') + parser.add_argument('--cc', action='append', default=[], help='cc recipient, can be given multiple times') + parser.add_argument('--dry-run', action='store_true', help='print the email instead of sending it') + args = parser.parse_args() + send_reminder(args.data_offer_ids, args.to, args.cc, args.dry_run) diff --git a/dev/mobilithek_helper/subscribe.py b/dev/mobilithek_helper/subscribe.py new file mode 100644 index 00000000..228f83c3 --- /dev/null +++ b/dev/mobilithek_helper/subscribe.py @@ -0,0 +1,39 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Requests a subscription for each given Mobilithek data offer id. New subscriptions start as REQUESTED and have to be +approved by the data provider. + +Usage: python dev/mobilithek_helper/subscribe.py [ ...] +""" + +import sys + +from mobilithek_client import MobilithekClient + + +def subscribe(data_offer_ids: list[str]): + client = MobilithekClient() + for data_offer_id in data_offer_ids: + subscription_id = client.subscribe(data_offer_id) + print(f'data offer {data_offer_id}: subscription {subscription_id}') + + +if __name__ == '__main__': + if len(sys.argv) < 2: + sys.exit(f'Usage: {sys.argv[0]} [ ...]') + subscribe(sys.argv[1:]) diff --git a/dev/mobilithek_helper/unsubscribe.py b/dev/mobilithek_helper/unsubscribe.py new file mode 100644 index 00000000..a0e65d92 --- /dev/null +++ b/dev/mobilithek_helper/unsubscribe.py @@ -0,0 +1,38 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Deletes each given Mobilithek subscription. + +Usage: python dev/mobilithek_helper/unsubscribe.py [ ...] +""" + +import sys + +from mobilithek_client import MobilithekClient + + +def unsubscribe(subscription_ids: list[str]): + client = MobilithekClient() + for subscription_id in subscription_ids: + client.unsubscribe(subscription_id) + print(f'subscription {subscription_id}: deleted') + + +if __name__ == '__main__': + if len(sys.argv) < 2: + sys.exit(f'Usage: {sys.argv[0]} [ ...]') + unsubscribe(sys.argv[1:]) diff --git a/dev/mobilithek_helper/validate_datex2.py b/dev/mobilithek_helper/validate_datex2.py new file mode 100644 index 00000000..83432cc4 --- /dev/null +++ b/dev/mobilithek_helper/validate_datex2.py @@ -0,0 +1,304 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Validates DATEX II static and realtime data of a Mobilithek subscription against the validators and mappers of the +DATEX II v3.5 and v3.7 import services, without touching the database. This shows which import service base class +fits the data and why an import fails. The static data is validated as a whole, so a single invalid site lets the +whole import fail. Realtime EVSE references are matched against the EVSE ids of the static data. + +Each argument is either a local file or a Mobilithek subscription id, which is fetched with the machine certificate. +This script needs the webapp, so it has to run in the flask container: + +Usage: docker compose run --rm flask python dev/mobilithek_helper/validate_datex2.py [] [--json] +""" + +import json +import sys +from collections import Counter +from dataclasses import dataclass, field +from pathlib import Path + +import requests +from validataclass.exceptions import ValidationError +from validataclass.helpers import UnsetValue +from validataclass.validators import DataclassValidator + +sys.path.insert(0, str(Path(__file__).parents[2])) + +from webapp.services.import_services.datex2.v3_5.base_datex2_v3_5_import_service import ( # noqa: E402 + BaseDatex2V35ImportService, +) +from webapp.services.import_services.datex2.v3_5.datex2_v3_5_json_static_mapper import ( # noqa: E402 + Datex2V35JSONStaticMapper, +) +from webapp.services.import_services.datex2.v3_7.base_datex2_v3_7_import_service import ( # noqa: E402 + BaseDatex2V37ImportService, +) +from webapp.services.import_services.datex2.v3_7.datex2_v3_7_json_static_mapper import ( # noqa: E402 + Datex2V37JSONStaticMapper, +) +from webapp.shared.datex2.models import MessageContainerWrapperInput # noqa: E402 +from webapp.shared.datex2.v3_5_json_static.models.d_a_t_e_x_i_i3_d2_payload_input import ( # noqa: E402 + DATEXII3D2PayloadInput as V35StaticPayloadInput, +) +from webapp.shared.datex2.v3_5_json_static.models.energy_infrastructure_site_input import ( # noqa: E402 + EnergyInfrastructureSiteInput as V35SiteInput, +) +from webapp.shared.datex2.v3_7.static.d_a_t_e_x_i_i3_d2_payload_input import ( # noqa: E402 + DATEXII3D2PayloadInput as V37StaticPayloadInput, +) +from webapp.shared.datex2.v3_7.static.energy_infrastructure_site_input import ( # noqa: E402 + EnergyInfrastructureSiteInput as V37SiteInput, +) + +SUBSCRIPTION_URL = 'https://mobilithek.info:8443/mobilithek/api/v1.0/subscription' +CERT = ( + str(Path(__file__).parents[2] / 'files' / 'mobilithek-binary-butterfly.crt.pem'), + str(Path(__file__).parents[2] / 'files' / 'mobilithek-binary-butterfly.key.pem'), +) + +VERSIONS = { + '3.5': { + 'static_validator': DataclassValidator(V35StaticPayloadInput), + 'site_validator': DataclassValidator(V35SiteInput), + 'realtime_payloads': lambda data: get_v3_5_realtime_payloads(data), + 'mapper': Datex2V35JSONStaticMapper(), + }, + '3.7': { + 'static_validator': DataclassValidator(V37StaticPayloadInput), + 'site_validator': DataclassValidator(V37SiteInput), + 'realtime_payloads': BaseDatex2V37ImportService.get_realtime_payloads, + 'mapper': Datex2V37JSONStaticMapper(), + }, +} + + +@dataclass +class StaticResult: + valid: bool = False + site_count: int = 0 + invalid_site_count: int = 0 + location_count: int = 0 + unmapped_site_count: int = 0 + evse_count: int = 0 + errors: list[str] = field(default_factory=list) + evse_uids: set[str] = field(default_factory=set) + + +@dataclass +class RealtimeResult: + valid: bool = False + status_count: int = 0 + unmapped_status_count: int = 0 + matched_count: int = 0 + errors: list[str] = field(default_factory=list) + + +def load_data(source: str) -> dict | None: + """ + Loads a local file or fetches a Mobilithek subscription. Returns None for empty responses. + """ + if Path(source).is_file(): + raw = Path(source).read_bytes() + else: + response = requests.get(SUBSCRIPTION_URL, params={'subscriptionID': source}, cert=CERT, timeout=300) + response.raise_for_status() + raw = response.content + if not raw.strip(): + return None + return json.loads(raw) + + +def flatten_errors(error: dict, path: str = '') -> list[str]: + """ + Flattens nested validataclass errors to paths, with list indices replaced by [*] + """ + errors: list[str] = [] + for key, child in (error.get('field_errors') or {}).items(): + errors += flatten_errors(child, f'{path}.{key}') + for child in (error.get('item_errors') or {}).values(): + errors += flatten_errors(child, f'{path}[*]') + if not errors: + reason = error.get('reason') or error.get('code') + errors.append(f'{path or "."}: {reason}') + return errors + + +def summarize_errors(errors: list[str], limit: int = 8) -> list[str]: + return [f'{count}x {error}' for error, count in Counter(errors).most_common(limit)] + + +def get_sites(data: dict) -> list[dict]: + publication = (data.get('payload') or {}).get('aegiEnergyInfrastructureTablePublication') or {} + return [ + site + for table in publication.get('energyInfrastructureTable') or [] + for site in table.get('energyInfrastructureSite') or [] + ] + + +def validate_static(data: dict, version: str) -> StaticResult: + config = VERSIONS[version] + result = StaticResult() + # Same hack as in the import services to support an invalid MessageContainer + if 'messageContainer' in data: + data = {'payload': data['messageContainer']['payload'][0]} + + sites = get_sites(data) + result.site_count = len(sites) + site_errors: list[str] = [] + for site in sites: + try: + config['site_validator'].validate(site) + except ValidationError as e: + result.invalid_site_count += 1 + site_errors += flatten_errors(e.to_dict(), 'site') + + try: + datex_input = config['static_validator'].validate(data) + except ValidationError as e: + result.errors = summarize_errors(site_errors or flatten_errors(e.to_dict())) + return result + + publication = datex_input.payload.aegiEnergyInfrastructureTablePublication + if publication is UnsetValue or not publication.energyInfrastructureTable: + result.errors = ['missing aegiEnergyInfrastructureTablePublication'] + return result + if version == '3.7' and len(publication.energyInfrastructureTable) != 1: + result.errors = [ + f'{len(publication.energyInfrastructureTable)} energyInfrastructureTables, 3.7 needs exactly one' + ] + return result + + result.valid = True + for table in publication.energyInfrastructureTable: + for site in table.energyInfrastructureSite: + location_update = config['mapper'].map_energy_infrastructure_site_to_location('validation', site) + if location_update is None: + result.unmapped_site_count += 1 + continue + result.location_count += 1 + for charging_station in location_update.charging_pool: + for evse in charging_station.evses: + result.evse_count += 1 + result.evse_uids.add(evse.uid) + return result + + +def get_v3_5_realtime_payloads(data: dict) -> list: + """ + Same envelope handling as BaseDatex2V35ImportService.fetch_realtime_data + """ + datex_input = DataclassValidator(MessageContainerWrapperInput).validate( + BaseDatex2V35ImportService.wrap_realtime_payload(data), + ) + if not datex_input.messageContainer or not datex_input.messageContainer.payload: + raise ValidationError(reason='missing messageContainer payload') + return [datex_input.messageContainer.payload[0]] + + +def validate_realtime(data: dict, version: str, evse_uids: set[str]) -> RealtimeResult: + config = VERSIONS[version] + result = RealtimeResult() + try: + payloads = config['realtime_payloads'](data) + except ValidationError as e: + result.errors = summarize_errors(flatten_errors(e.to_dict())) + return result + + for payload in payloads: + if payload is UnsetValue or payload.aegiEnergyInfrastructureStatusPublication is UnsetValue: + result.errors = ['missing aegiEnergyInfrastructureStatusPublication'] + return result + + result.valid = True + for payload in payloads: + for site_status in payload.aegiEnergyInfrastructureStatusPublication.energyInfrastructureSiteStatus or []: + if site_status.energyInfrastructureStationStatus is UnsetValue: + continue + for station_status in site_status.energyInfrastructureStationStatus: + if station_status.refillPointStatus is UnsetValue: + continue + for refill_point_status_g in station_status.refillPointStatus: + result.status_count += 1 + evse_update = config['mapper'].map_energy_infrastructure_station_status(refill_point_status_g) + if evse_update is None: + result.unmapped_status_count += 1 + elif evse_update.evse_id in evse_uids: + result.matched_count += 1 + return result + + +def validate(static_source: str, realtime_source: str | None) -> dict: + report: dict = {} + static_data = load_data(static_source) + realtime_data = load_data(realtime_source) if realtime_source else None + if static_data is None: + report['static'] = 'empty response' + if realtime_source and realtime_data is None: + report['realtime'] = 'empty response' + + for version in VERSIONS: + version_report: dict = {} + evse_uids: set[str] = set() + if static_data is not None: + static_result = validate_static(static_data, version) + evse_uids = static_result.evse_uids + version_report['static'] = { + key: value for key, value in static_result.__dict__.items() if key != 'evse_uids' + } + if realtime_data is not None: + version_report['realtime'] = validate_realtime(realtime_data, version, evse_uids).__dict__ + report[version] = version_report + return report + + +def print_report(report: dict): + for key in ('static', 'realtime'): + if isinstance(report.get(key), str): + print(f'{key}: {report[key]}') + for version in VERSIONS: + static = report[version].get('static') + if static: + state = 'valid' if static['valid'] else 'INVALID' + print( + f'v{version} static {state}: {static["site_count"]} sites, {static["invalid_site_count"]} invalid, ' + f'{static["location_count"]} locations mapped, {static["unmapped_site_count"]} unmapped, ' + f'{static["evse_count"]} EVSEs', + ) + for error in static['errors']: + print(f' {error}') + realtime = report[version].get('realtime') + if realtime: + state = 'valid' if realtime['valid'] else 'INVALID' + print( + f'v{version} realtime {state}: {realtime["status_count"]} statuses, ' + f'{realtime["unmapped_status_count"]} unmapped, {realtime["matched_count"]} matching static EVSEs', + ) + for error in realtime['errors']: + print(f' {error}') + + +if __name__ == '__main__': + arguments = [argument for argument in sys.argv[1:] if argument != '--json'] + if not 1 <= len(arguments) <= 2: + sys.exit(f'Usage: {sys.argv[0]} [] [--json]') + validation_report = validate(arguments[0], arguments[1] if len(arguments) > 1 else None) + if '--json' in sys.argv: + print(json.dumps(validation_report)) + else: + print_report(validation_report)