diff --git a/README.md b/README.md index 70aab853..3015d08c 100644 --- a/README.md +++ b/README.md @@ -5,45 +5,95 @@ OCPI-style JSON-API and a vector tile server. ## Data sources -| name | uid | realtime | credentials | comment | -|-------------------------------------|-------------------------|----------|-------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Bundesnetzagentur: API | bnetza_api | false | false | Additional config `ignore_operators:: list[str]` is supported, which will ignore given operators during import. Set to weekly download, as it does not change so often. | -| Bundesnetzagentur: Excel | bnetza_excel | false | false | | -| chargecloud: Stadtwerke Pforzheim | chargecloud_pforzheim | true | false | | -| chargecloud: Stadtwerke Stuttgart | chargecloud_stuttgart | true | false | | -| chargecloud: Stadtwerke Tübingen | chargecloud_tuebingen | true | false | | -| chargecloud: Stadtwerke Ludwigsburg | chargecloud_ludwigsburg | true | true | | -| Datex2: Ampeco | datex2_ampeco | true | true | Via Mobilithek, currently disfunctional | -| Datex2: Chargecloud | datex2_chargecloud | true | true | Via Mobilithek | -| Datex2: e-clearing.net | datex2_e_clearing_net | true | true | Via Mobilithek | -| Datex2: EcoMovement | datex2_ecomovement | true | true | Via Mobilithek | -| Datex2: Elu Mobility | datex2_elu_mobility | true | true | Via Mobilithek, currently disfunctional | -| Datex2: EnBW | datex2_enbw | true | true | Via Mobilithek | -| Datex2: enio | datex2_enio | true | true | Via Mobilithek | -| Datex2: eRound | datex2_eround | true | true | Via Mobilithek | -| Datex2: Grid&Co | datex2_gridandco | true | true | Via Mobilithek, currently disfunctional | -| Datex2: Inergia | datex2_inergia | true | true | Via Mobilithek | -| Datex2: ladebusiness | datex2_ladebusiness | true | true | Via Mobilithek, currently disfunctional | -| Datex2: Lichtblick | datex2_lichtblick | true | true | Via Mobilithek | -| Datex2: Midorion | datex2_midorion | true | true | Via Mobilithek | -| Datex2: Monta | datex2_monta | true | true | Via Mobilithek, currently disfunctional | -| Datex2: msu | datex2_msu | true | true | Via Mobilithek | -| Datex2: Pump | datex2_pump | false | true | Via Mobilithek, currently disfunctional | -| Datex2: Qwello | datex2_qwello | true | true | Via Mobilithek | -| Datex2: Smatrics | datex2_smatrics | true | true | Via Mobilithek, currently disfunctional | -| Datex2: Taubert Consulting | datex2_taubert | true | true | Via Mobilithek | -| Datex2: Tesla | datex2_tesla | true | true | Via Mobilithek | -| Datex2: Vaylens | datex2_vaylens | true | true | Via Mobilithek, currently disfunctional | -| Datex2: Volkswagen Charging Group | datex2_volkswagen | true | true | Via Mobilithek | -| Datex2: Wirelane | datex2_wirelane | true | true | Via Mobilithek | -| PBW | eaaze_pbw | true | true | | -| Giro-e | giroe | true | true | | -| Heilbronn Heckarbogen | heilbronn_neckarbogen | true | true | | -| Lichtblick | lichtblick | true | true | Currently dysfunctional | -| OCHP: Albwerk | ochp_albwerk | true | true | | -| OCHP: Ladenetz | ochp_ladenetz | true | true | | -| OCPI: Stadtnavi | ocpi_stadtnavi | true | false | | -| OpenData Swiss | opendata_swiss | true | false | | +Mobilithek sources were last validated on 2026-10-28 with `dev/mobilithek_helper/validate_datex2.py`. DATEX II data +has to be valid against the DATEX II schema, otherwise the import fails as a whole. See +[Mobilithek helper scripts](#mobilithek-helper-scripts) for details. + +| name | uid | realtime | credentials | comment | +|----------------------------------------|------------------------------|----------|-------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Bundesnetzagentur: API | bnetza_api | false | false | Additional config `ignore_operators:: list[str]` is supported, which will ignore given operators during import. Set to weekly download, as it does not change so often. | +| Bundesnetzagentur: Excel | bnetza_excel | false | false | | +| chargecloud: Stadtwerke Ludwigsburg | chargecloud_ludwigsburg | true | true | | +| chargecloud: Stadtwerke Pforzheim | chargecloud_pforzheim | true | false | | +| chargecloud: Stadtwerke Stuttgart | chargecloud_stuttgart | true | false | | +| chargecloud: Stadtwerke Tübingen | chargecloud_tuebingen | true | false | | +| Datex2: AH Beteiligungsgesellschaft | datex2_ah_beteiligung | true | true | | +| Datex2: Ampeco | datex2_ampeco | true | true | Static data is schema-invalid (organisation names and identifiers, operating hours) | +| Datex2: Audi charging hub | datex2_audi | true | true | Static data is schema-invalid (energy rates without `lastUpdated`), realtime data is empty | +| Datex2: Chargecloud | datex2_chargecloud | true | true | | +| Datex2: e-clearing.net | datex2_e_clearing_net | true | true | | +| Datex2: EAAZE | datex2_eaaze | true | true | | +| Datex2: EcoMovement | datex2_ecomovement | true | true | | +| Datex2: Elu Mobility | datex2_elu_mobility | true | true | Static data is schema-invalid (`pointLocation` instead of `locPointLocation`) | +| Datex2: EnBW | datex2_enbw | true | true | | +| Datex2: ENERANDO | datex2_enerando | true | true | | +| Datex2: enio | datex2_enio | true | true | | +| Datex2: eRound | datex2_eround | true | true | | +| Datex2: Eulektro | datex2_eulektro | true | true | | +| Datex2: EV Price | datex2_ev_price | true | true | | +| Datex2: EWE | datex2_ewe | true | true | | +| Datex2: Flavia | datex2_flavia | true | true | | +| Datex2: GLS Mobility | datex2_giroe | true | true | Realtime data is a delta pull and was empty on 2026-10-29, so EVSE matching is unverified | +| Datex2: GP JOULE Connect | datex2_gp_joule | true | true | | +| Datex2: Grid&Co | datex2_gridandco | true | true | Static data is schema-invalid (external identifier without identifier) | +| Datex2: Inergia | datex2_inergia | true | true | Via Mobilithek | +| Datex2: Inselwerke | datex2_inselwerke | true | true | | +| Datex2: K.u.L. Rudnick | datex2_kul_rudnick | false | true | No data published | +| Datex2: ladebusiness | datex2_ladebusiness | true | true | Data offer was withdrawn | +| Datex2: Ladesonne | datex2_ladesonne | true | true | | +| Datex2: Lichtblick | datex2_lichtblick | true | true | | +| Datex2: Midorion | datex2_midorion | true | true | | +| Datex2: Monta | datex2_monta | true | true | Static data is schema-invalid (times without seconds and offset) | +| Datex2: msu | datex2_msu | true | true | | +| Datex2: Pump | datex2_pump | false | true | Static data is schema-invalid (`versionG` at energy rates) | +| Datex2: Qwello | datex2_qwello | true | true | | +| Datex2: R & M Beteiligungsgesellschaft | datex2_r_m_beteiligung | true | true | | +| Datex2: Road | datex2_road | true | true | | +| Datex2: Smatrics | datex2_smatrics | true | true | | +| Datex2: Sonnenstrom Saarpfalz | datex2_sonnenstrom_saarpfalz | true | true | | +| Datex2: Taubert Consulting | datex2_taubert | true | true | | +| Datex2: Tesla | datex2_tesla | true | true | | +| Datex2: Ulm | datex2_ulm | true | true | | +| Datex2: Vaylens | datex2_vaylens | true | true | Static data is schema-invalid (times without seconds and offset) | +| Datex2: Virta | datex2_virta | true | true | | +| Datex2: Volkswagen Charging Group | datex2_volkswagen | true | true | Realtime data is outdated since 2026-10-02 | +| Datex2: Wirelane | datex2_wirelane | true | true | | +| eliso | eliso | true | true | Custom JSON format instead of DATEX II | +| Giro-e | giroe | true | true | | +| Heilbronn Heckarbogen | heilbronn_neckarbogen | true | true | | +| Lichtblick | lichtblick | true | true | Currently dysfunctional | +| OCHP: Albwerk | ochp_albwerk | true | true | | +| OCHP: Ladenetz | ochp_ladenetz | true | true | | +| OCPI: Stadtnavi | ocpi_stadtnavi | true | false | | +| OpenData Swiss | opendata_swiss | true | false | | +| PBW | eaaze_pbw | true | true | | + + +### Mobilithek helper scripts + +`dev/mobilithek_helper` contains scripts to manage the Mobilithek subscriptions of the charging data sources. Managing +subscriptions needs a user login, so the scripts read `MOBILITHEK_USER` and `MOBILITHEK_PASSWORD` from `.env`. + +```bash +python dev/mobilithek_helper/list_charging_offers.py # all charging data offers with subscription and import service +python dev/mobilithek_helper/list_subscriptions.py # all subscriptions of the account +python dev/mobilithek_helper/subscribe.py [ ...] +python dev/mobilithek_helper/unsubscribe.py [ ...] +python dev/mobilithek_helper/send_reminder.py [--dry-run] --to [ ...] +``` + +`send_reminder.py` sends a German plain text email to the contact person of a provider, asking to approve pending +subscriptions. The Mobilithek does not expose contact addresses of other organisations, so the recipient has to be given +explicitly. It reads `SMTP_HOST`, `SMTP_PORT` (465 uses SSL, otherwise STARTTLS), `SMTP_USER`, `SMTP_PASSWORD` and +`SMTP_FROM` from `.env`. + +`validate_datex2.py` validates the data of a subscription against the DATEX II v3.5 and v3.7 import services without +touching the database. It shows which base class fits, why an import fails and how many realtime EVSE statuses match +static EVSEs. It needs the webapp and the Mobilithek machine certificate in `files/`, so it runs in the flask container: + +```bash +docker compose run --rm flask python dev/mobilithek_helper/validate_datex2.py [] +``` ### Duplicate matching diff --git a/dev/mobilithek_helper/list_charging_offers.py b/dev/mobilithek_helper/list_charging_offers.py new file mode 100644 index 00000000..cd3423b2 --- /dev/null +++ b/dev/mobilithek_helper/list_charging_offers.py @@ -0,0 +1,266 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Writes all charging data offers of the Mobilithek as CSV, with the subscription status of the Mobilithek account and the +OCPDB import service covering the offer, if there is one. Afterwards, it prints the ids of all brokered charging data +offers without an active or requested subscription, which can be passed to subscribe.py. + +Usage: python dev/mobilithek_helper/list_charging_offers.py [] +Without output file, the CSV is written to temp/mobilithek_charging_offers.csv. +""" + +import csv +import sys +from collections import Counter, defaultdict +from pathlib import Path + +from mobilithek_client import MobilithekClient + +DEFAULT_OUTPUT_PATH = Path(__file__).parents[2] / 'temp' / 'mobilithek_charging_offers.csv' + +CHARGING_CATEGORY = 'https://w3id.org/mdp/schema/data_categories#FILLING_AND_CHARGING_STATIONS' + +# Charging data offers which the publisher filed in another data category +EXTRA_DATA_OFFER_IDS: list[str] = [ + '974006122901856256', # AFIR-recharging-dyn-Eulektro, filed as PARKING_AND_REST_AREA_INFORMATION +] + +MATCH_MOBILITHEK = 'Mobilithek import service' +MATCH_OTHER_SOURCE = 'other source, same data' +MATCH_OTHER_SOURCE_LIKELY = 'other source, likely same data' +MATCH_OWN_PUBLICATION = 'own publication' + +# The Mobilithek import services just know their subscription ids from config, and the other import services fetch +# their data from the original source, so this mapping has to be maintained by hand. The 'likely' matches are guessed +# from publisher and description of the offer. +CONVERTER_BY_DATA_OFFER_ID: dict[str, tuple[str, str]] = { + '972837891969273856': ('datex2_ampeco', MATCH_MOBILITHEK), + '972842599324557312': ('datex2_ampeco', MATCH_MOBILITHEK), + '978597062404620288': ('datex2_chargecloud', MATCH_MOBILITHEK), + '980862594474274816': ('datex2_chargecloud', MATCH_MOBILITHEK), + '996825300704600064': ('datex2_e_clearing_net', MATCH_MOBILITHEK), + '996823601386508288': ('datex2_e_clearing_net', MATCH_MOBILITHEK), + '907574882292453376': ('datex2_enbw', MATCH_MOBILITHEK), + '907575401287241728': ('datex2_enbw', MATCH_MOBILITHEK), + '961625658278940672': ('datex2_eround', MATCH_MOBILITHEK), + '961629419076456448': ('datex2_eround', MATCH_MOBILITHEK), + '954064102947180544': ('datex2_ecomovement', MATCH_MOBILITHEK), + '955166494396665856': ('datex2_ecomovement', MATCH_MOBILITHEK), + '936298491949047808': ('datex2_elu_mobility', MATCH_MOBILITHEK), + '971513500454850560': ('datex2_elu_mobility', MATCH_MOBILITHEK), + '963766220171735040': ('datex2_enio', MATCH_MOBILITHEK), + '968541134128902144': ('datex2_enio', MATCH_MOBILITHEK), + # giroe fetches the same data directly from GLS Mobility + '980559859451379712': ('datex2_giroe; giroe', MATCH_MOBILITHEK), + '980563757096464384': ('datex2_giroe; giroe', MATCH_MOBILITHEK), + '984104561811357696': ('datex2_gridandco', MATCH_MOBILITHEK), + '984103903968534528': ('datex2_gridandco', MATCH_MOBILITHEK), + '962721207430316032': ('datex2_lichtblick', MATCH_MOBILITHEK), + '962731482363617280': ('datex2_lichtblick', MATCH_MOBILITHEK), + '979435003058024448': ('datex2_midorion', MATCH_MOBILITHEK), + '979433370303545344': ('datex2_midorion', MATCH_MOBILITHEK), + '963836072152719360': ('datex2_monta', MATCH_MOBILITHEK), + '963870983660167168': ('datex2_monta', MATCH_MOBILITHEK), + '970305056590979072': ('datex2_msu', MATCH_MOBILITHEK), + '970388804493828096': ('datex2_msu', MATCH_MOBILITHEK), + '969322788846231552': ('datex2_pump', MATCH_MOBILITHEK), + '972963216296222720': ('datex2_qwello', MATCH_MOBILITHEK), + '972966368902897664': ('datex2_qwello', MATCH_MOBILITHEK), + '980809280034390016': ('datex2_smatrics', MATCH_MOBILITHEK), + '961319990963605504': ('datex2_smatrics', MATCH_MOBILITHEK), + '1000749522967314432': ('datex2_taubert', MATCH_MOBILITHEK), + '1000746576770711552': ('datex2_taubert', MATCH_MOBILITHEK), + '953828817873125376': ('datex2_tesla', MATCH_MOBILITHEK), + '953843379766972416': ('datex2_tesla', MATCH_MOBILITHEK), + '979363267193040896': ('datex2_vaylens', MATCH_MOBILITHEK), + '979364650281549824': ('datex2_vaylens', MATCH_MOBILITHEK), + '983008874583728128': ('datex2_volkswagen', MATCH_MOBILITHEK), + '983006934617260032': ('datex2_volkswagen', MATCH_MOBILITHEK), + '869246425829892096': ('datex2_wirelane', MATCH_MOBILITHEK), + '876587237907525632': ('datex2_wirelane', MATCH_MOBILITHEK), + '998571342932365312': ('datex2_audi', MATCH_MOBILITHEK), + '998567365272563712': ('datex2_audi', MATCH_MOBILITHEK), + # eaaze_pbw fetches the PBW part of the EAAZE data via OCPI + '980426161955500032': ('datex2_eaaze; eaaze_pbw', MATCH_MOBILITHEK), + '980425456721260544': ('datex2_eaaze; eaaze_pbw', MATCH_MOBILITHEK), + '1037036344672452608': ('datex2_enerando', MATCH_MOBILITHEK), + '1037038702374617088': ('datex2_enerando', MATCH_MOBILITHEK), + '973938793916375040': ('datex2_eulektro', MATCH_MOBILITHEK), + '974006122901856256': ('datex2_eulektro', MATCH_MOBILITHEK), + '993495185148837888': ('datex2_ah_beteiligung', MATCH_MOBILITHEK), + '993494022504546304': ('datex2_ah_beteiligung', MATCH_MOBILITHEK), + '981180413145477120': ('datex2_inselwerke', MATCH_MOBILITHEK), + '981170994693238784': ('datex2_inselwerke', MATCH_MOBILITHEK), + '988133177339846656': ('datex2_ladesonne', MATCH_MOBILITHEK), + '988137788909801472': ('datex2_ladesonne', MATCH_MOBILITHEK), + '993499364131065856': ('datex2_r_m_beteiligung', MATCH_MOBILITHEK), + '993498817080557568': ('datex2_r_m_beteiligung', MATCH_MOBILITHEK), + '979427073235566592': ('datex2_sonnenstrom_saarpfalz', MATCH_MOBILITHEK), + '979428785661079552': ('datex2_sonnenstrom_saarpfalz', MATCH_MOBILITHEK), + '989311807176560640': ('datex2_ev_price', MATCH_MOBILITHEK), + '989311073915731968': ('datex2_ev_price', MATCH_MOBILITHEK), + '1024264572575293440': ('datex2_ewe', MATCH_MOBILITHEK), + '1006184570533171200': ('datex2_ewe', MATCH_MOBILITHEK), + '1004450433724211200': ('datex2_flavia', MATCH_MOBILITHEK), + '1004756865530970112': ('datex2_flavia', MATCH_MOBILITHEK), + '997111469996658688': ('datex2_gp_joule', MATCH_MOBILITHEK), + '997190851637440512': ('datex2_gp_joule', MATCH_MOBILITHEK), + '1043944773714128896': ('datex2_inergia', MATCH_MOBILITHEK), + '1043947032435105792': ('datex2_inergia', MATCH_MOBILITHEK), + '983747474988376064': ('datex2_kul_rudnick', MATCH_MOBILITHEK), + '1021366453630017536': ('datex2_road', MATCH_MOBILITHEK), + '1021364001002217472': ('datex2_road', MATCH_MOBILITHEK), + '1042468553977937920': ('datex2_virta', MATCH_MOBILITHEK), + '1042472664437800960': ('datex2_virta', MATCH_MOBILITHEK), + '854410608351543296': ('datex2_ulm', MATCH_MOBILITHEK), + '854416606814023680': ('datex2_ulm', MATCH_MOBILITHEK), + # eliso uses a custom JSON format instead of DATEX II + '843477276990078976': ('eliso', MATCH_MOBILITHEK), + '843502085052710912': ('eliso', MATCH_MOBILITHEK), + '951517095896416256': ('bnetza_api', MATCH_OTHER_SOURCE_LIKELY), + '842113170303512576': ('bnetza_excel', MATCH_OTHER_SOURCE_LIKELY), + # Bundled data of MobiData BW, which is the output of OCPDB itself + '-2035239451501556619': ('', MATCH_OWN_PUBLICATION), + '980084361378099200': ('', MATCH_OWN_PUBLICATION), + '981124859836461056': ('', MATCH_OWN_PUBLICATION), +} + +# Offers in the charging category which are no charging data sources, so we don't subscribe to them +IGNORE_REASON_BY_DATA_OFFER_ID: dict[str, str] = { + '935580278881759232': 'fuel station', + '937348385824550912': 'fuel station', + '753618942489935872': 'fuel station', + '834870693179846656': 'fuel station', + '968150797115953152': 'test data', + '981218494322012160': 'test data', + '981223821767557120': 'test data', + '1006136734156312576': 'test data', + '755033335157133312': 'statistical analysis', + '1033716358314917888': 'organisation data only', + '980084361378099200': 'own publication', + '981124859836461056': 'own publication', +} + +# Subscriptions in these states don't need a new subscription +OPEN_SUBSCRIPTION_STATUSES = ('ACTIVE', 'REQUESTED') + +FIELDS = [ + 'publicationId', + 'title', + 'publisher', + 'dataModel', + 'license', + 'mobilithekBrokering', + 'region', + 'subscriptionStatus', + 'subscriptionId', + 'converter', + 'converterMatch', + 'ignoreReason', + 'url', +] + + +def short_uri(uri: str | None) -> str: + """ + Strips vocabulary URIs like https://w3id.org/mdp/schema/license#NO_LICENSE down to NO_LICENSE + """ + if not uri: + return '' + return uri.rsplit('#', 1)[-1] + + +def get_row(data_offer: dict, subscriptions: list[dict]) -> dict: + publication_id = data_offer['publicationId'] + converter, converter_match = CONVERTER_BY_DATA_OFFER_ID.get(publication_id, ('', '')) + data_models = { + short_uri(content_data.get('dataModel')) + for content_data in data_offer.get('contentData') or [] + if content_data.get('dataModel') + } + return { + 'publicationId': publication_id, + 'title': data_offer['title'].strip(), + 'publisher': data_offer['agents']['publisher']['name'], + 'dataModel': '; '.join(sorted(data_models)), + 'license': short_uri(data_offer.get('rights')), + # Brokered offers are delivered by the Mobilithek itself and can be pulled with the machine certificate + 'mobilithekBrokering': data_offer.get('mdpBrokering'), + 'region': '; '.join(nuts['name'] for nuts in (data_offer.get('spatialCoverage') or {}).get('nuts') or []), + 'subscriptionStatus': '; '.join(subscription['contractStatus'] for subscription in subscriptions), + 'subscriptionId': '; '.join(subscription['id'] for subscription in subscriptions), + 'converter': converter, + 'converterMatch': converter_match, + 'ignoreReason': IGNORE_REASON_BY_DATA_OFFER_ID.get(publication_id, ''), + 'url': f'https://mobilithek.info/offers/{publication_id}', + } + + +def is_missing_subscription(row: dict) -> bool: + return ( + row['mobilithekBrokering'] is True + and not row['ignoreReason'] + and not any(status in OPEN_SUBSCRIPTION_STATUSES for status in row['subscriptionStatus'].split('; ')) + ) + + +def list_charging_offers(output_path: Path): + client = MobilithekClient() + data_offers = client.search_data_offers([CHARGING_CATEGORY]) + known_data_offer_ids = {data_offer['publicationId'] for data_offer in data_offers} + data_offers += [ + client.get_data_offer_by_id(data_offer_id) + for data_offer_id in EXTRA_DATA_OFFER_IDS + if data_offer_id not in known_data_offer_ids + ] + data_offers.sort(key=lambda data_offer: (data_offer['agents']['publisher']['name'], data_offer['title'])) + + subscriptions_by_data_offer_id: dict[str, list[dict]] = defaultdict(list) + for subscription in client.get_subscriptions(): + subscriptions_by_data_offer_id[subscription['dataOfferId']].append(subscription) + + rows = [ + get_row(data_offer, subscriptions_by_data_offer_id[data_offer['publicationId']]) for data_offer in data_offers + ] + + output_path.parent.mkdir(parents=True, exist_ok=True) + with output_path.open('w', newline='', encoding='utf-8') as output_file: + writer = csv.DictWriter(output_file, fieldnames=FIELDS) + writer.writeheader() + writer.writerows(rows) + + print(f'{len(rows)} charging data offers written to {output_path}') + overview = Counter( + ( + 'converter' if row['converter'] or row['converterMatch'] else 'no converter', + row['subscriptionStatus'] or 'no subscription', + ) + for row in rows + ) + for (converter, subscription_status), count in sorted(overview.items()): + print(f' {converter}, {subscription_status}: {count}') + + missing_rows = [row for row in rows if is_missing_subscription(row)] + print(f'{len(missing_rows)} brokered charging data offers without active or requested subscription:') + for row in missing_rows: + print(f' {row["publicationId"]} {row["publisher"]}: {row["title"]}') + if missing_rows: + print(f'subscribe with: subscribe.py {" ".join(row["publicationId"] for row in missing_rows)}') + + +if __name__ == '__main__': + list_charging_offers(Path(sys.argv[1]) if len(sys.argv) > 1 else DEFAULT_OUTPUT_PATH) diff --git a/dev/mobilithek_helper/list_subscriptions.py b/dev/mobilithek_helper/list_subscriptions.py new file mode 100644 index 00000000..f72448c1 --- /dev/null +++ b/dev/mobilithek_helper/list_subscriptions.py @@ -0,0 +1,125 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Writes all subscriptions of the Mobilithek account as CSV, including their status (REQUESTED, ACTIVE, INACTIVE) and +metadata of the subscribed data offer like licence, category and data model. For restricted or withdrawn data offers, +only parts of the metadata are available, see column metadataComplete. + +Usage: python dev/mobilithek_helper/list_subscriptions.py [] +Without output file, the CSV is written to temp/mobilithek_subscriptions.csv. +""" + +import csv +import sys +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +from mobilithek_client import MobilithekClient + +DEFAULT_OUTPUT_PATH = Path(__file__).parents[2] / 'temp' / 'mobilithek_subscriptions.csv' + +SUBSCRIPTION_FIELDS = [ + 'id', + 'dataOfferId', + 'dataOfferTitle', + 'providerOrgName', + 'contractStatus', + 'providerApproved', + 'napApproved', + 'validFrom', + 'validTo', + 'deliveryMode', + 'currentAccessCount', + 'maxAccessCount', +] + +DATA_OFFER_FIELDS = [ + 'metadataComplete', + 'license', + 'standardLicense', + 'customLicense', + 'dataCategory', + 'dataCategoryDetail', + 'themes', + 'transportMode', + 'dataModel', + 'accrualPeriodicity', + 'spatialCoverage', + 'keywords', + 'dataOfferModified', + 'irregularSince', +] + + +def short_uri(uri: str | None) -> str: + """ + Strips vocabulary URIs like https://w3id.org/mdp/schema/license#NO_LICENSE down to NO_LICENSE + """ + if not uri: + return '' + return uri.rsplit('#', 1)[-1] if '#' in uri else uri.rsplit('/', 1)[-1] + + +def join_values(values: list | None) -> str: + return '; '.join(str(value) for value in values or [] if value) + + +def get_data_offer_row(data_offer: dict, metadata_complete: bool) -> dict: + custom_license = data_offer.get('customLicenseAsResource') or {} + return { + 'metadataComplete': metadata_complete, + 'license': short_uri(data_offer.get('rights')), + 'standardLicense': data_offer.get('standardLicense') or '', + 'customLicense': (custom_license.get('instance') or {}).get('fileName') or '', + 'dataCategory': short_uri(data_offer.get('dataCategory')), + 'dataCategoryDetail': join_values([short_uri(uri) for uri in data_offer.get('dataCategoryDetail') or []]), + 'themes': join_values([short_uri(uri) for uri in data_offer.get('themes') or []]), + 'transportMode': join_values([short_uri(uri) for uri in data_offer.get('transportMode') or []]), + 'dataModel': join_values([short_uri(item.get('dataModel')) for item in data_offer.get('contentData') or []]), + 'accrualPeriodicity': short_uri(data_offer.get('accrualPeriodicity')), + 'spatialCoverage': join_values( + [nuts.get('name') for nuts in (data_offer.get('spatialCoverage') or {}).get('nuts') or []], + ), + 'keywords': join_values(data_offer.get('keyword')), + 'dataOfferModified': data_offer.get('modified') or '', + 'irregularSince': data_offer.get('irregularSince') or '', + } + + +def list_subscriptions(output_path: Path): + client = MobilithekClient() + subscriptions = client.get_subscriptions() + subscriptions.sort(key=lambda subscription: (subscription['providerOrgName'] or '', subscription['dataOfferTitle'])) + + # One request per subscription, so we parallelize a bit + with ThreadPoolExecutor(max_workers=8) as executor: + data_offers = executor.map(client.get_data_offer, subscriptions) + rows = [ + {**subscription, **get_data_offer_row(data_offer, metadata_complete)} + for subscription, (data_offer, metadata_complete) in zip(subscriptions, data_offers, strict=True) + ] + + with output_path.open('w', newline='', encoding='utf-8') as output_file: + writer = csv.DictWriter(output_file, fieldnames=SUBSCRIPTION_FIELDS + DATA_OFFER_FIELDS, extrasaction='ignore') + writer.writeheader() + writer.writerows(rows) + + print(f'{len(rows)} subscriptions written to {output_path}') + + +if __name__ == '__main__': + list_subscriptions(Path(sys.argv[1]) if len(sys.argv) > 1 else DEFAULT_OUTPUT_PATH) diff --git a/dev/mobilithek_helper/mobilithek_client.py b/dev/mobilithek_helper/mobilithek_client.py new file mode 100644 index 00000000..6917b504 --- /dev/null +++ b/dev/mobilithek_helper/mobilithek_client.py @@ -0,0 +1,124 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +import os +from pathlib import Path + +import requests +from dotenv import load_dotenv + +TOKEN_URL = 'https://mobilithek.info/auth/realms/MDP/protocol/openid-connect/token' +CONTRACTS_URL = 'https://mobilithek.info/mdp-api/mdp-msa-contracts/v1/contracts' +OFFERS_URL = 'https://mobilithek.info/mdp-api/mdp-msa-metadata/v2/offers' + + +class MobilithekClient: + """ + Client for the (undocumented) API behind the Mobilithek web GUI. Managing subscriptions needs a user login, the + machine certificate is not sufficient. Credentials are read from MOBILITHEK_USER and MOBILITHEK_PASSWORD in .env. + """ + + def __init__(self): + load_dotenv(Path(__file__).parents[2] / '.env') + response = requests.post( + TOKEN_URL, + data={ + 'grant_type': 'password', + 'client_id': 'Platform', + 'username': os.environ['MOBILITHEK_USER'], + 'password': os.environ['MOBILITHEK_PASSWORD'], + }, + timeout=30, + ) + response.raise_for_status() + + self.session = requests.Session() + self.session.headers['Authorization'] = f'Bearer {response.json()["access_token"]}' + + def get_subscriptions(self) -> list[dict]: + subscriptions: list[dict] = [] + page = 0 + while True: + response = self.session.post( + f'{CONTRACTS_URL}/search', + json={}, + params={'page': page, 'size': 100}, + timeout=30, + ) + response.raise_for_status() + data = response.json() + subscriptions += data['content'] + if data['last']: + return subscriptions + page += 1 + + def search_data_offers(self, categories: list[str]) -> list[dict]: + """ + Returns all data offers in the given data categories, e.g. data_categories#FILLING_AND_CHARGING_STATIONS. + Logged in, this includes restricted offers which are not visible in the public search. + """ + data_offers: list[dict] = [] + page = 0 + while True: + # Without explicit sorting, the order is not stable between pages, so offers get duplicated or skipped + response = self.session.post( + f'{OFFERS_URL}/search', + json={'categories': categories}, + params={'page': page, 'size': 1000, 'sort': 'created,desc'}, + timeout=60, + ) + response.raise_for_status() + data = response.json()['dataOffers'] + data_offers += data['content'] + if data['last']: + return data_offers + page += 1 + + def get_data_offer_by_id(self, data_offer_id: str) -> dict: + response = self.session.get(f'{OFFERS_URL}/{data_offer_id}', timeout=30) + response.raise_for_status() + return response.json() + + def get_data_offer(self, subscription: dict) -> tuple[dict, bool]: + """ + Returns the data offer of a subscription and whether its metadata is complete. Restricted or withdrawn offers + are just available via the subscription, which only contains parts of the metadata, e.g. the licence. + """ + response = self.session.get(f'{OFFERS_URL}/{subscription["dataOfferId"]}', timeout=30) + if response.ok: + return response.json(), True + if response.status_code not in (403, 404): + response.raise_for_status() + + response = self.session.get(f'{CONTRACTS_URL}/{subscription["id"]}/with-offer', timeout=30) + response.raise_for_status() + return response.json()['dataOffer'], False + + def subscribe(self, data_offer_id: str) -> str: + response = self.session.post( + CONTRACTS_URL, + json={'dataOfferId': data_offer_id, 'relevantToMDVPBefG': False}, + timeout=30, + ) + response.raise_for_status() + # The API returns the new subscription id as plain body + return response.text + + def unsubscribe(self, subscription_id: str): + response = self.session.delete(f'{CONTRACTS_URL}/{subscription_id}', timeout=30) + response.raise_for_status() diff --git a/dev/mobilithek_helper/send_reminder.py b/dev/mobilithek_helper/send_reminder.py new file mode 100644 index 00000000..1fca4ca4 --- /dev/null +++ b/dev/mobilithek_helper/send_reminder.py @@ -0,0 +1,170 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Sends a German plain text reminder to the contact person of a data provider, asking to approve our pending +subscriptions of the given charging data offers. All data offers have to belong to the same provider and have a +REQUESTED subscription. + +The Mobilithek does not expose contact email addresses of other organisations, so the recipient has to be given +explicitly. SMTP credentials are read from .env: SMTP_HOST, SMTP_PORT (465 uses SSL, everything else STARTTLS), +SMTP_USER, SMTP_PASSWORD and SMTP_FROM. + +Usage: python dev/mobilithek_helper/send_reminder.py [--dry-run] --to [--to ...] [--cc ...] + [ ...] +""" + +import argparse +import os +import smtplib +import ssl +import sys +import textwrap +from datetime import datetime +from email.message import EmailMessage +from email.utils import formatdate, make_msgid + +from mobilithek_client import MobilithekClient + +SUBJECT = 'Mobilithek: Bitte um Freigabe unserer Abonnement-Anfrage' + + +def format_request_date(subscription: dict) -> str: + if not subscription.get('createdAt'): + return 'Datum unbekannt' + return datetime.fromisoformat(subscription['createdAt'].replace('Z', '+00:00')).strftime('%d.%m.%Y') + + +def build_body(provider_name: str, subscriptions: list[dict]) -> str: + plural = len(subscriptions) > 1 + offer_lines = '\n'.join( + f'- {subscription["dataOfferTitle"].strip()} (angefragt am {format_request_date(subscription)})\n' + f' https://mobilithek.info/offers/{subscription["dataOfferId"]}' + for subscription in subscriptions + ) + sender_name = f'{subscriptions[0]["subscriberUserFirstName"]} {subscriptions[0]["subscriberUserLastName"]}'.strip() + + paragraphs = [ + 'Guten Tag,', + ( + 'im Auftrag der NVBW – Nahverkehrsgesellschaft Baden-Württemberg mbH betreiben wir die ' + 'Ladeinfrastruktur-Datenbank von MobiData BW, der Mobilitätsdatenplattform des Landes Baden-Württemberg. ' + 'Dort führen wir die Daten öffentlich zugänglicher Ladepunkte aus vielen Quellen zusammen und stellen sie ' + 'als Open Data für Apps, Routenplaner und Reiseinformation bereit:' + ), + 'https://mobidata-bw.de', + ( + f'Gerne würden wir prüfen, ob sich auch die Daten von {provider_name} für eine Aufnahme eignen. Dazu ' + f'haben wir über die Mobilithek {"folgende Datenangebote" if plural else "folgendes Datenangebot"} ' + 'abonniert:' + ), + offer_lines, + ( + f'{"Diese Anfragen sind" if plural else "Diese Anfrage ist"} bisher noch nicht freigegeben. Wir würden uns ' + f'sehr freuen, wenn Sie sie in der Mobilithek unter „Organisation“ → „Abonnements“ ' + 'bestätigen könnten:' + ), + 'https://mobilithek.info/organisation/subscriptions', + ( + 'Zum Hintergrund: Nach Art. 20 der Verordnung (EU) 2023/1804 über den Aufbau der Infrastruktur für ' + 'alternative Kraftstoffe (AFIR) müssen Betreiber öffentlich zugänglicher Ladepunkte statische und dynamische ' + 'Daten zu ihren Ladepunkten kostenlos und diskriminierungsfrei über den nationalen Zugangspunkt ' + 'bereitstellen. In Deutschland ist das die Mobilithek. Solange ein Abonnement nicht freigegeben ist, sind die ' + 'Daten für Nutzer jedoch nicht abrufbar.' + ), + ( + 'Falls Sie Fragen haben oder die Anfrage an eine andere Stelle in Ihrem Haus gehen sollte, melden Sie sich ' + 'gerne.' + ), + f'Vielen Dank und viele Grüße\n{sender_name}', + ] + # The offer list is already formatted, everything else is wrapped for plain text email clients + return ( + '\n\n'.join( + paragraph if paragraph == offer_lines or '\n' in paragraph else textwrap.fill(paragraph, width=72) + for paragraph in paragraphs + ) + + '\n' + ) + + +def get_pending_subscriptions(client: MobilithekClient, data_offer_ids: list[str]) -> list[dict]: + subscriptions_by_data_offer_id = { + subscription['dataOfferId']: subscription + for subscription in client.get_subscriptions() + if subscription['contractStatus'] == 'REQUESTED' + } + missing = [data_offer_id for data_offer_id in data_offer_ids if data_offer_id not in subscriptions_by_data_offer_id] + if missing: + sys.exit(f'No REQUESTED subscription for data offer(s): {", ".join(missing)}') + + subscriptions = [subscriptions_by_data_offer_id[data_offer_id] for data_offer_id in data_offer_ids] + providers = {subscription['providerOrgName'] for subscription in subscriptions} + if len(providers) > 1: + sys.exit(f'Data offers belong to different providers: {", ".join(sorted(providers))}') + return subscriptions + + +def send(message: EmailMessage): + host = os.environ['SMTP_HOST'] + port = int(os.environ.get('SMTP_PORT', '587')) + context = ssl.create_default_context() + if port == 465: + smtp = smtplib.SMTP_SSL(host, port, context=context, timeout=30) + else: + smtp = smtplib.SMTP(host, port, timeout=30) + smtp.starttls(context=context) + with smtp: + smtp.login(os.environ['SMTP_USER'], os.environ['SMTP_PASSWORD']) + smtp.send_message(message) + + +def send_reminder(data_offer_ids: list[str], to: list[str], cc: list[str], dry_run: bool): + # MobilithekClient loads .env, so the SMTP variables are available afterwards + client = MobilithekClient() + subscriptions = get_pending_subscriptions(client, data_offer_ids) + + message = EmailMessage() + message['Subject'] = SUBJECT + message['From'] = os.environ.get('SMTP_FROM', '' if dry_run else os.environ['SMTP_FROM']) + message['To'] = ', '.join(to) + if cc: + message['Cc'] = ', '.join(cc) + message['Date'] = formatdate(localtime=True) + message['Message-ID'] = make_msgid() + message.set_content(build_body(subscriptions[0]['providerOrgName'], subscriptions), charset='utf-8') + + if dry_run: + print(render(message)) + return + + send(message) + print(f'Reminder for {len(subscriptions)} data offer(s) sent to {", ".join(to + cc)}') + + +def render(message: EmailMessage) -> str: + headers = '\n'.join(f'{key}: {value}' for key, value in message.items() if key in ('From', 'To', 'Cc', 'Subject')) + return f'{headers}\n\n{message.get_content()}' + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Sends a reminder to approve pending Mobilithek subscriptions.') + parser.add_argument('data_offer_ids', nargs='+', metavar='data offer id') + parser.add_argument('--to', action='append', required=True, help='recipient, can be given multiple times') + parser.add_argument('--cc', action='append', default=[], help='cc recipient, can be given multiple times') + parser.add_argument('--dry-run', action='store_true', help='print the email instead of sending it') + args = parser.parse_args() + send_reminder(args.data_offer_ids, args.to, args.cc, args.dry_run) diff --git a/dev/mobilithek_helper/subscribe.py b/dev/mobilithek_helper/subscribe.py new file mode 100644 index 00000000..228f83c3 --- /dev/null +++ b/dev/mobilithek_helper/subscribe.py @@ -0,0 +1,39 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Requests a subscription for each given Mobilithek data offer id. New subscriptions start as REQUESTED and have to be +approved by the data provider. + +Usage: python dev/mobilithek_helper/subscribe.py [ ...] +""" + +import sys + +from mobilithek_client import MobilithekClient + + +def subscribe(data_offer_ids: list[str]): + client = MobilithekClient() + for data_offer_id in data_offer_ids: + subscription_id = client.subscribe(data_offer_id) + print(f'data offer {data_offer_id}: subscription {subscription_id}') + + +if __name__ == '__main__': + if len(sys.argv) < 2: + sys.exit(f'Usage: {sys.argv[0]} [ ...]') + subscribe(sys.argv[1:]) diff --git a/dev/mobilithek_helper/unsubscribe.py b/dev/mobilithek_helper/unsubscribe.py new file mode 100644 index 00000000..a0e65d92 --- /dev/null +++ b/dev/mobilithek_helper/unsubscribe.py @@ -0,0 +1,38 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Deletes each given Mobilithek subscription. + +Usage: python dev/mobilithek_helper/unsubscribe.py [ ...] +""" + +import sys + +from mobilithek_client import MobilithekClient + + +def unsubscribe(subscription_ids: list[str]): + client = MobilithekClient() + for subscription_id in subscription_ids: + client.unsubscribe(subscription_id) + print(f'subscription {subscription_id}: deleted') + + +if __name__ == '__main__': + if len(sys.argv) < 2: + sys.exit(f'Usage: {sys.argv[0]} [ ...]') + unsubscribe(sys.argv[1:]) diff --git a/dev/mobilithek_helper/validate_datex2.py b/dev/mobilithek_helper/validate_datex2.py new file mode 100644 index 00000000..83432cc4 --- /dev/null +++ b/dev/mobilithek_helper/validate_datex2.py @@ -0,0 +1,304 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Validates DATEX II static and realtime data of a Mobilithek subscription against the validators and mappers of the +DATEX II v3.5 and v3.7 import services, without touching the database. This shows which import service base class +fits the data and why an import fails. The static data is validated as a whole, so a single invalid site lets the +whole import fail. Realtime EVSE references are matched against the EVSE ids of the static data. + +Each argument is either a local file or a Mobilithek subscription id, which is fetched with the machine certificate. +This script needs the webapp, so it has to run in the flask container: + +Usage: docker compose run --rm flask python dev/mobilithek_helper/validate_datex2.py [] [--json] +""" + +import json +import sys +from collections import Counter +from dataclasses import dataclass, field +from pathlib import Path + +import requests +from validataclass.exceptions import ValidationError +from validataclass.helpers import UnsetValue +from validataclass.validators import DataclassValidator + +sys.path.insert(0, str(Path(__file__).parents[2])) + +from webapp.services.import_services.datex2.v3_5.base_datex2_v3_5_import_service import ( # noqa: E402 + BaseDatex2V35ImportService, +) +from webapp.services.import_services.datex2.v3_5.datex2_v3_5_json_static_mapper import ( # noqa: E402 + Datex2V35JSONStaticMapper, +) +from webapp.services.import_services.datex2.v3_7.base_datex2_v3_7_import_service import ( # noqa: E402 + BaseDatex2V37ImportService, +) +from webapp.services.import_services.datex2.v3_7.datex2_v3_7_json_static_mapper import ( # noqa: E402 + Datex2V37JSONStaticMapper, +) +from webapp.shared.datex2.models import MessageContainerWrapperInput # noqa: E402 +from webapp.shared.datex2.v3_5_json_static.models.d_a_t_e_x_i_i3_d2_payload_input import ( # noqa: E402 + DATEXII3D2PayloadInput as V35StaticPayloadInput, +) +from webapp.shared.datex2.v3_5_json_static.models.energy_infrastructure_site_input import ( # noqa: E402 + EnergyInfrastructureSiteInput as V35SiteInput, +) +from webapp.shared.datex2.v3_7.static.d_a_t_e_x_i_i3_d2_payload_input import ( # noqa: E402 + DATEXII3D2PayloadInput as V37StaticPayloadInput, +) +from webapp.shared.datex2.v3_7.static.energy_infrastructure_site_input import ( # noqa: E402 + EnergyInfrastructureSiteInput as V37SiteInput, +) + +SUBSCRIPTION_URL = 'https://mobilithek.info:8443/mobilithek/api/v1.0/subscription' +CERT = ( + str(Path(__file__).parents[2] / 'files' / 'mobilithek-binary-butterfly.crt.pem'), + str(Path(__file__).parents[2] / 'files' / 'mobilithek-binary-butterfly.key.pem'), +) + +VERSIONS = { + '3.5': { + 'static_validator': DataclassValidator(V35StaticPayloadInput), + 'site_validator': DataclassValidator(V35SiteInput), + 'realtime_payloads': lambda data: get_v3_5_realtime_payloads(data), + 'mapper': Datex2V35JSONStaticMapper(), + }, + '3.7': { + 'static_validator': DataclassValidator(V37StaticPayloadInput), + 'site_validator': DataclassValidator(V37SiteInput), + 'realtime_payloads': BaseDatex2V37ImportService.get_realtime_payloads, + 'mapper': Datex2V37JSONStaticMapper(), + }, +} + + +@dataclass +class StaticResult: + valid: bool = False + site_count: int = 0 + invalid_site_count: int = 0 + location_count: int = 0 + unmapped_site_count: int = 0 + evse_count: int = 0 + errors: list[str] = field(default_factory=list) + evse_uids: set[str] = field(default_factory=set) + + +@dataclass +class RealtimeResult: + valid: bool = False + status_count: int = 0 + unmapped_status_count: int = 0 + matched_count: int = 0 + errors: list[str] = field(default_factory=list) + + +def load_data(source: str) -> dict | None: + """ + Loads a local file or fetches a Mobilithek subscription. Returns None for empty responses. + """ + if Path(source).is_file(): + raw = Path(source).read_bytes() + else: + response = requests.get(SUBSCRIPTION_URL, params={'subscriptionID': source}, cert=CERT, timeout=300) + response.raise_for_status() + raw = response.content + if not raw.strip(): + return None + return json.loads(raw) + + +def flatten_errors(error: dict, path: str = '') -> list[str]: + """ + Flattens nested validataclass errors to paths, with list indices replaced by [*] + """ + errors: list[str] = [] + for key, child in (error.get('field_errors') or {}).items(): + errors += flatten_errors(child, f'{path}.{key}') + for child in (error.get('item_errors') or {}).values(): + errors += flatten_errors(child, f'{path}[*]') + if not errors: + reason = error.get('reason') or error.get('code') + errors.append(f'{path or "."}: {reason}') + return errors + + +def summarize_errors(errors: list[str], limit: int = 8) -> list[str]: + return [f'{count}x {error}' for error, count in Counter(errors).most_common(limit)] + + +def get_sites(data: dict) -> list[dict]: + publication = (data.get('payload') or {}).get('aegiEnergyInfrastructureTablePublication') or {} + return [ + site + for table in publication.get('energyInfrastructureTable') or [] + for site in table.get('energyInfrastructureSite') or [] + ] + + +def validate_static(data: dict, version: str) -> StaticResult: + config = VERSIONS[version] + result = StaticResult() + # Same hack as in the import services to support an invalid MessageContainer + if 'messageContainer' in data: + data = {'payload': data['messageContainer']['payload'][0]} + + sites = get_sites(data) + result.site_count = len(sites) + site_errors: list[str] = [] + for site in sites: + try: + config['site_validator'].validate(site) + except ValidationError as e: + result.invalid_site_count += 1 + site_errors += flatten_errors(e.to_dict(), 'site') + + try: + datex_input = config['static_validator'].validate(data) + except ValidationError as e: + result.errors = summarize_errors(site_errors or flatten_errors(e.to_dict())) + return result + + publication = datex_input.payload.aegiEnergyInfrastructureTablePublication + if publication is UnsetValue or not publication.energyInfrastructureTable: + result.errors = ['missing aegiEnergyInfrastructureTablePublication'] + return result + if version == '3.7' and len(publication.energyInfrastructureTable) != 1: + result.errors = [ + f'{len(publication.energyInfrastructureTable)} energyInfrastructureTables, 3.7 needs exactly one' + ] + return result + + result.valid = True + for table in publication.energyInfrastructureTable: + for site in table.energyInfrastructureSite: + location_update = config['mapper'].map_energy_infrastructure_site_to_location('validation', site) + if location_update is None: + result.unmapped_site_count += 1 + continue + result.location_count += 1 + for charging_station in location_update.charging_pool: + for evse in charging_station.evses: + result.evse_count += 1 + result.evse_uids.add(evse.uid) + return result + + +def get_v3_5_realtime_payloads(data: dict) -> list: + """ + Same envelope handling as BaseDatex2V35ImportService.fetch_realtime_data + """ + datex_input = DataclassValidator(MessageContainerWrapperInput).validate( + BaseDatex2V35ImportService.wrap_realtime_payload(data), + ) + if not datex_input.messageContainer or not datex_input.messageContainer.payload: + raise ValidationError(reason='missing messageContainer payload') + return [datex_input.messageContainer.payload[0]] + + +def validate_realtime(data: dict, version: str, evse_uids: set[str]) -> RealtimeResult: + config = VERSIONS[version] + result = RealtimeResult() + try: + payloads = config['realtime_payloads'](data) + except ValidationError as e: + result.errors = summarize_errors(flatten_errors(e.to_dict())) + return result + + for payload in payloads: + if payload is UnsetValue or payload.aegiEnergyInfrastructureStatusPublication is UnsetValue: + result.errors = ['missing aegiEnergyInfrastructureStatusPublication'] + return result + + result.valid = True + for payload in payloads: + for site_status in payload.aegiEnergyInfrastructureStatusPublication.energyInfrastructureSiteStatus or []: + if site_status.energyInfrastructureStationStatus is UnsetValue: + continue + for station_status in site_status.energyInfrastructureStationStatus: + if station_status.refillPointStatus is UnsetValue: + continue + for refill_point_status_g in station_status.refillPointStatus: + result.status_count += 1 + evse_update = config['mapper'].map_energy_infrastructure_station_status(refill_point_status_g) + if evse_update is None: + result.unmapped_status_count += 1 + elif evse_update.evse_id in evse_uids: + result.matched_count += 1 + return result + + +def validate(static_source: str, realtime_source: str | None) -> dict: + report: dict = {} + static_data = load_data(static_source) + realtime_data = load_data(realtime_source) if realtime_source else None + if static_data is None: + report['static'] = 'empty response' + if realtime_source and realtime_data is None: + report['realtime'] = 'empty response' + + for version in VERSIONS: + version_report: dict = {} + evse_uids: set[str] = set() + if static_data is not None: + static_result = validate_static(static_data, version) + evse_uids = static_result.evse_uids + version_report['static'] = { + key: value for key, value in static_result.__dict__.items() if key != 'evse_uids' + } + if realtime_data is not None: + version_report['realtime'] = validate_realtime(realtime_data, version, evse_uids).__dict__ + report[version] = version_report + return report + + +def print_report(report: dict): + for key in ('static', 'realtime'): + if isinstance(report.get(key), str): + print(f'{key}: {report[key]}') + for version in VERSIONS: + static = report[version].get('static') + if static: + state = 'valid' if static['valid'] else 'INVALID' + print( + f'v{version} static {state}: {static["site_count"]} sites, {static["invalid_site_count"]} invalid, ' + f'{static["location_count"]} locations mapped, {static["unmapped_site_count"]} unmapped, ' + f'{static["evse_count"]} EVSEs', + ) + for error in static['errors']: + print(f' {error}') + realtime = report[version].get('realtime') + if realtime: + state = 'valid' if realtime['valid'] else 'INVALID' + print( + f'v{version} realtime {state}: {realtime["status_count"]} statuses, ' + f'{realtime["unmapped_status_count"]} unmapped, {realtime["matched_count"]} matching static EVSEs', + ) + for error in realtime['errors']: + print(f' {error}') + + +if __name__ == '__main__': + arguments = [argument for argument in sys.argv[1:] if argument != '--json'] + if not 1 <= len(arguments) <= 2: + sys.exit(f'Usage: {sys.argv[0]} [] [--json]') + validation_report = validate(arguments[0], arguments[1] if len(arguments) > 1 else None) + if '--json' in sys.argv: + print(json.dumps(validation_report)) + else: + print_report(validation_report) diff --git a/pyproject.toml b/pyproject.toml index 701747f4..379a2209 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -166,6 +166,14 @@ line-length = 120 # Don't require __init__.py files "INP", ] +"dev/*" = [ + # Don't require __init__.py files + "INP", + # Allow print in helper scripts + "T201", + # Token URLs are no passwords + "S105", +] [tool.ruff.lint.flake8-quotes] inline-quotes = "single" diff --git a/tests/integration/public_api/ocpi/connector_tariff_api_test.py b/tests/integration/public_api/ocpi/connector_tariff_api_test.py index f9790def..f0a8b29d 100644 --- a/tests/integration/public_api/ocpi/connector_tariff_api_test.py +++ b/tests/integration/public_api/ocpi/connector_tariff_api_test.py @@ -36,7 +36,9 @@ def test_connector_outputs_tariff_uids_of_its_evse( db: SQLAlchemy, public_test_client: OpenApiFlaskClient, ) -> None: - """Sources publish tariffs per EVSE, so the connectors of an EVSE output its tariff uids.""" + """ + Sources publish tariffs per EVSE, so the connectors of an EVSE output its tariff uids. + """ tariff = get_tariff_1() db.session.add(tariff) db.session.flush() @@ -56,7 +58,9 @@ def test_connector_outputs_all_tariff_uids_of_its_evse( db: SQLAlchemy, public_test_client: OpenApiFlaskClient, ) -> None: - """An EVSE offered under several tariffs passes all of their uids to its connectors.""" + """ + An EVSE offered under several tariffs passes all of their uids to its connectors. + """ first_tariff = get_tariff_1() second_tariff = get_tariff_2() db.session.add_all([first_tariff, second_tariff]) @@ -106,7 +110,9 @@ def test_connector_tariff_uids_take_precedence_over_the_evse_ones( db: SQLAlchemy, public_test_client: OpenApiFlaskClient, ) -> None: - """A connector carrying its own tariffs does not inherit the ones of its EVSE.""" + """ + A connector carrying its own tariffs does not inherit the ones of its EVSE. + """ evse_tariff = get_tariff_1() connector_tariff = get_tariff_2() db.session.add_all([evse_tariff, connector_tariff]) @@ -128,7 +134,9 @@ def test_connector_without_tariffs_omits_tariff_ids( db: SQLAlchemy, public_test_client: OpenApiFlaskClient, ) -> None: - """tariff_ids is optional in OCPI, so a connector without any tariff leaves it out.""" + """ + tariff_ids is optional in OCPI, so a connector without any tariff leaves it out. + """ db.session.add(get_location_1(evses=[get_full_evse_1()], operator=get_business_1())) db.session.commit() @@ -142,7 +150,9 @@ def test_evse_endpoint_outputs_connector_tariff_uids( db: SQLAlchemy, public_test_client: OpenApiFlaskClient, ) -> None: - """The connectors nested into an EVSE response carry the tariff uids too.""" + """ + The connectors nested into an EVSE response carry the tariff uids too. + """ tariff = get_tariff_1() db.session.add(tariff) db.session.flush() diff --git a/tests/integration/public_api/ocpi/v2_2/tariff_api_test.py b/tests/integration/public_api/ocpi/v2_2/tariff_api_test.py index 2a41d35f..571c3403 100644 --- a/tests/integration/public_api/ocpi/v2_2/tariff_api_test.py +++ b/tests/integration/public_api/ocpi/v2_2/tariff_api_test.py @@ -145,7 +145,9 @@ def test_get_ocpi_22_tariff_merges_tariff_data( db: SQLAlchemy, public_test_client: OpenApiFlaskClient, ) -> None: - """Two associations pointing to the same tariff produce two OCPI 2.2 tariffs with same pricing but different types.""" + """ + Two associations pointing to the same tariff produce two OCPI 2.2 tariffs with same pricing but different types. + """ tariff = get_tariff_1() tariff.elements = [ TariffElement( @@ -178,7 +180,9 @@ def test_get_ocpi_22_tariffs_does_not_load_evses_and_connectors( db: SQLAlchemy, public_test_client: OpenApiFlaskClient, ) -> None: - """OCPI 2.2 tariffs do not render the EVSEs and connectors of a tariff association, so they are not loaded.""" + """ + OCPI 2.2 tariffs do not render the EVSEs and connectors of a tariff association, so they are not loaded. + """ tariff = get_tariff_1() db.session.add(tariff) db.session.flush() diff --git a/tests/integration/public_api/ocpi/v3_0/tariff_association_api_test.py b/tests/integration/public_api/ocpi/v3_0/tariff_association_api_test.py index 5a0c4eef..2ae138d0 100644 --- a/tests/integration/public_api/ocpi/v3_0/tariff_association_api_test.py +++ b/tests/integration/public_api/ocpi/v3_0/tariff_association_api_test.py @@ -130,7 +130,9 @@ def test_get_ocpi_30_tariff_associations_loads_just_evse_ids( db: SQLAlchemy, public_test_client: OpenApiFlaskClient, ) -> None: - """Tariff associations render just the ids of their EVSEs, so the EVSEs are not loaded completely.""" + """ + Tariff associations render just the ids of their EVSEs, so the EVSEs are not loaded completely. + """ tariff = get_tariff_1() db.session.add(tariff) db.session.flush() diff --git a/tests/integration/repositories/location_repository_test.py b/tests/integration/repositories/location_repository_test.py index 8b3d6824..22d9127e 100644 --- a/tests/integration/repositories/location_repository_test.py +++ b/tests/integration/repositories/location_repository_test.py @@ -26,7 +26,9 @@ def test_fetch_location_by_uid_respects_source(db: SQLAlchemy) -> None: - """Different sources can use the same location uid, so the source has to be part of the lookup.""" + """ + Different sources can use the same location uid, so the source has to be part of the lookup. + """ db.session.add_all([ get_location_1(source=SOURCE_UID_1, name='Location of source 1'), get_location_1(source=SOURCE_UID_2, name='Location of source 2'), diff --git a/tests/integration/server_rest_api/bnetza/bnetza_import_api_test.py b/tests/integration/server_rest_api/bnetza/bnetza_import_api_test.py index 87b352c9..b4dc21ff 100644 --- a/tests/integration/server_rest_api/bnetza/bnetza_import_api_test.py +++ b/tests/integration/server_rest_api/bnetza/bnetza_import_api_test.py @@ -29,14 +29,18 @@ @pytest.fixture def stub_celery_delay(): - """Stub out CeleryHelper.delay so the bnetza import task is not actually enqueued.""" + """ + Stub out CeleryHelper.delay so the bnetza import task is not actually enqueued. + """ with patch.object(dependencies.get_celery_helper(), 'delay') as mock_delay: yield mock_delay @pytest.fixture def isolated_bnetza_dir(tmp_path: Path): - """Redirect BNETZA_IMPORT_DIR to a per-test temp directory to avoid polluting `data/`.""" + """ + Redirect BNETZA_IMPORT_DIR to a per-test temp directory to avoid polluting `data/`. + """ config = dependencies.get_config_helper().get_config() original = config.get('BNETZA_IMPORT_DIR') config['BNETZA_IMPORT_DIR'] = str(tmp_path) diff --git a/tests/integration/server_rest_api/datex2/datex2_import_api_test.py b/tests/integration/server_rest_api/datex2/datex2_import_api_test.py index f2b5ae84..da69507d 100644 --- a/tests/integration/server_rest_api/datex2/datex2_import_api_test.py +++ b/tests/integration/server_rest_api/datex2/datex2_import_api_test.py @@ -60,7 +60,9 @@ def _import_static_data(requests_mock: Mocker) -> None: def _fetch_realtime_data_updated_at(db: SQLAlchemy) -> datetime | None: - """Return the source's ``realtime_data_updated_at`` as an aware UTC datetime.""" + """ + Return the source's ``realtime_data_updated_at`` as an aware UTC datetime. + """ db.session.expire_all() source = db.session.query(Source).filter(Source.uid == SOURCE_UID).first() if source is None or source.realtime_data_updated_at is None: @@ -74,7 +76,9 @@ def _fetch_realtime_data_updated_at(db: SQLAlchemy) -> datetime | None: @pytest.fixture def isolated_datex2_dir(tmp_path: Path): - """Redirect DATEX2_IMPORT_DIR to a per-test temp directory.""" + """ + Redirect DATEX2_IMPORT_DIR to a per-test temp directory. + """ config = dependencies.get_config_helper().get_config() original = config.get('DATEX2_IMPORT_DIR') config['DATEX2_IMPORT_DIR'] = str(tmp_path) @@ -100,7 +104,9 @@ def run_task_synchronously(task, *args, **kwargs): @pytest.fixture def stubbed_celery_delay(): - """Replace ``CeleryHelper.delay`` with a no-op mock for tests that don't need the task to run.""" + """ + Replace ``CeleryHelper.delay`` with a no-op mock for tests that don't need the task to run. + """ with patch.object(dependencies.get_celery_helper(), 'delay') as mock_delay: yield mock_delay @@ -430,7 +436,9 @@ def test_push_realtime_v35_delta_push_updates_realtime_data_updated_at( isolated_datex2_dir: Path, stubbed_celery_delay, ) -> None: - """A synchronously applied deltaPush must update ``realtime_data_updated_at`` as well.""" + """ + A synchronously applied deltaPush must update ``realtime_data_updated_at`` as well. + """ _import_static_data(requests_mock) realtime_data = _load_test_data('datex2_enbw_realtime_reduced.json') diff --git a/tests/integration/server_rest_api/giroe/giroe_api_test.py b/tests/integration/server_rest_api/giroe/giroe_api_test.py index 79783d32..4cb723ba 100644 --- a/tests/integration/server_rest_api/giroe/giroe_api_test.py +++ b/tests/integration/server_rest_api/giroe/giroe_api_test.py @@ -60,7 +60,9 @@ def _giroe_connector_uid(connector_id: int) -> str: def _valid_location_payload(location_id: int, *, public: bool = True) -> dict: - """Build a minimal-but-complete giroe Location payload accepted by LocationInput.""" + """ + Build a minimal-but-complete giroe Location payload accepted by LocationInput. + """ now = datetime.now(timezone.utc).isoformat() return { 'id': location_id, diff --git a/tests/integration/server_rest_api/openapi_docs_test.py b/tests/integration/server_rest_api/openapi_docs_test.py index fe246fe5..91d45cfd 100644 --- a/tests/integration/server_rest_api/openapi_docs_test.py +++ b/tests/integration/server_rest_api/openapi_docs_test.py @@ -56,7 +56,9 @@ def test_server_openapi_json_renders(test_client: OpenApiFlaskClient) -> None: def test_server_openapi_html_renders(test_client: OpenApiFlaskClient) -> None: - """The Stoplight-Elements HTML wrapper page must load without errors.""" + """ + The Stoplight-Elements HTML wrapper page must load without errors. + """ response = test_client.get('/documentation/server.html') assert response.status_code == HTTPStatus.OK diff --git a/tests/integration/services/import_services/base_import_service_test.py b/tests/integration/services/import_services/base_import_service_test.py index 5d3a2701..c1b959c8 100644 --- a/tests/integration/services/import_services/base_import_service_test.py +++ b/tests/integration/services/import_services/base_import_service_test.py @@ -498,7 +498,9 @@ def test_save_location_updates_replaces_per_evse_associations_with_a_shared_one( def _create_location_update_with_two_evses(*, tariff_association_uids: list[str]) -> LocationUpdate: - """One location with two EVSEs, each pointing at the tariff association of the given uid.""" + """ + One location with two EVSEs, each pointing at the tariff association of the given uid. + """ tariff_update = TariffUpdate( uid='tariff1', source='test_source_uid', @@ -631,7 +633,9 @@ def _create_location_update_with_tariff( def test_save_location_updates_deletes_tariffs_with_new_uid(db, testing_import_service: BaseImportService): - """A tariff which the source does not send anymore is deleted instead of staying behind unreferenced.""" + """ + A tariff which the source does not send anymore is deleted instead of staying behind unreferenced. + """ testing_import_service.save_location_updates([_create_location_update_with_tariff(tariff_uid='tariff1')]) testing_import_service.save_location_updates([_create_location_update_with_tariff(tariff_uid='tariff2')]) @@ -647,7 +651,9 @@ def test_save_location_updates_deletes_tariff_associations_of_kept_tariff( db, testing_import_service: BaseImportService, ): - """The tariff survives its renamed association, but the old association does not.""" + """ + The tariff survives its renamed association, but the old association does not. + """ testing_import_service.save_location_updates([_create_location_update_with_tariff(tariff_association_uid='ta1')]) testing_import_service.save_location_updates([_create_location_update_with_tariff(tariff_association_uid='ta2')]) @@ -656,7 +662,9 @@ def test_save_location_updates_deletes_tariff_associations_of_kept_tariff( def test_save_location_updates_deletes_tariffs_of_deleted_locations(db, testing_import_service: BaseImportService): - """Tariffs do not belong to a location, so a vanished location has to take its tariffs with it.""" + """ + Tariffs do not belong to a location, so a vanished location has to take its tariffs with it. + """ testing_import_service.save_location_updates([ _create_location_update_with_tariff(location_uid='loc1', tariff_association_uid='ta1', tariff_uid='tariff1'), _create_location_update_with_tariff(location_uid='loc2', tariff_association_uid='ta2', tariff_uid='tariff2'), @@ -674,7 +682,9 @@ def test_save_location_updates_deletes_tariffs_of_deleted_locations(db, testing_ def test_save_location_updates_keeps_tariffs_of_other_sources(db, testing_import_service: BaseImportService): - """The cleanup is limited to the source which is imported.""" + """ + The cleanup is limited to the source which is imported. + """ other_source_tariff = Tariff( uid='tariff1', source='other_source_uid', diff --git a/tests/integration/services/import_services/datex2/data/datex2_giroe_realtime_reduced.json b/tests/integration/services/import_services/datex2/data/datex2_giroe_realtime_reduced.json new file mode 100644 index 00000000..77e9a429 --- /dev/null +++ b/tests/integration/services/import_services/datex2/data/datex2_giroe_realtime_reduced.json @@ -0,0 +1,201 @@ +{ + "messageContainer": { + "exchangeInformation": { + "dynamicInformation": { + "exchangeStatus": { + "value": "online" + }, + "messageGenerationTimestamp": "2026-10-28T20:00:00+00:00" + }, + "exchangeContext": { + "codedExchangeProtocol": { + "value": "snapshotPush" + }, + "exchangeSpecificationVersion": "3.7", + "supplierOrCisRequester": { + "name": "GLS Mobility GmbH" + } + } + }, + "payload": [ + { + "aegiEnergyInfrastructureStatusPublication": { + "energyInfrastructureSiteStatus": [ + { + "energyInfrastructureStationStatus": [ + { + "lastUpdated": "2026-10-28T20:00:00+00:00", + "reference": { + "idG": "1", + "targetClass": "FacilityObject", + "versionG": "1" + }, + "refillPointStatus": [ + { + "aegiElectricChargingPointStatus": { + "lastUpdated": "2026-10-28T20:00:00+00:00", + "reference": { + "idG": "1", + "targetClass": "FacilityObject", + "versionG": "1" + }, + "status": { + "value": "available" + } + } + }, + { + "aegiElectricChargingPointStatus": { + "lastUpdated": "2026-10-28T20:00:00+00:00", + "reference": { + "idG": "2", + "targetClass": "FacilityObject", + "versionG": "1" + }, + "status": { + "value": "charging" + } + } + } + ] + }, + { + "lastUpdated": "2026-10-28T20:00:00+00:00", + "reference": { + "idG": "2", + "targetClass": "FacilityObject", + "versionG": "1" + }, + "refillPointStatus": [ + { + "aegiElectricChargingPointStatus": { + "lastUpdated": "2026-10-28T20:00:00+00:00", + "reference": { + "idG": "3", + "targetClass": "FacilityObject", + "versionG": "1" + }, + "status": { + "value": "charging" + } + } + }, + { + "aegiElectricChargingPointStatus": { + "lastUpdated": "2026-10-28T20:00:00+00:00", + "reference": { + "idG": "4", + "targetClass": "FacilityObject", + "versionG": "1" + }, + "status": { + "value": "available" + } + } + } + ] + } + ], + "lastUpdated": "2026-10-28T20:00:00+00:00", + "reference": { + "idG": "1", + "targetClass": "FacilityObject", + "versionG": "1" + } + }, + { + "energyInfrastructureStationStatus": [ + { + "lastUpdated": "2026-10-28T20:00:00+00:00", + "reference": { + "idG": "3", + "targetClass": "FacilityObject", + "versionG": "1" + }, + "refillPointStatus": [ + { + "aegiElectricChargingPointStatus": { + "lastUpdated": "2026-10-28T20:00:00+00:00", + "reference": { + "idG": "5", + "targetClass": "FacilityObject", + "versionG": "1" + }, + "status": { + "value": "available" + } + } + }, + { + "aegiElectricChargingPointStatus": { + "lastUpdated": "2026-10-28T20:00:00+00:00", + "reference": { + "idG": "6", + "targetClass": "FacilityObject", + "versionG": "1" + }, + "status": { + "value": "charging" + } + } + } + ] + }, + { + "lastUpdated": "2026-10-28T20:00:00+00:00", + "reference": { + "idG": "4", + "targetClass": "FacilityObject", + "versionG": "1" + }, + "refillPointStatus": [ + { + "aegiElectricChargingPointStatus": { + "lastUpdated": "2026-10-28T20:00:00+00:00", + "reference": { + "idG": "7", + "targetClass": "FacilityObject", + "versionG": "1" + }, + "status": { + "value": "charging" + } + } + }, + { + "aegiElectricChargingPointStatus": { + "lastUpdated": "2026-10-28T20:00:00+00:00", + "reference": { + "idG": "8", + "targetClass": "FacilityObject", + "versionG": "1" + }, + "status": { + "value": "available" + } + } + } + ] + } + ], + "lastUpdated": "2026-10-28T20:00:00+00:00", + "reference": { + "idG": "2", + "targetClass": "FacilityObject", + "versionG": "1" + } + } + ], + "lang": "de", + "publicationCreator": { + "country": "DE", + "nationalIdentifier": "DE-GLS" + }, + "publicationTime": "2026-10-28T20:00:00+00:00" + }, + "modelBaseVersionG": "3", + "versionG": "1" + } + ] + } +} diff --git a/tests/integration/services/import_services/datex2/enbw_datex2_import_service_test.py b/tests/integration/services/import_services/datex2/enbw_datex2_import_service_test.py index 69b53655..ab3bf7da 100644 --- a/tests/integration/services/import_services/datex2/enbw_datex2_import_service_test.py +++ b/tests/integration/services/import_services/datex2/enbw_datex2_import_service_test.py @@ -362,3 +362,38 @@ def test_enbw_datex2_realtime_loops_until_last_modified_stabilizes( assert source.realtime_data_updated_at.replace(tzinfo=timezone.utc) == datetime( 2026, 4, 19, 12, 0, 0, tzinfo=timezone.utc ) + + +def test_enbw_datex2_realtime_not_modified_ends_pull( + db: SQLAlchemy, + requests_mock: Mocker, +) -> None: + """ + Mobilithek answers with 304 Not Modified once there is no newer data. This ends the pull loop: the statuses + received so far have to be saved and the source stays ACTIVE. + """ + _mock_static(requests_mock) + requests_mock.get( + REALTIME_SUBSCRIPTION_URL, + [ + { + 'status_code': 200, + 'text': _load_test_data('datex2_enbw_realtime_reduced.json'), + 'headers': {'Content-Type': 'application/json', 'Last-Modified': 'Sat, 19 Apr 2026 10:00:00 GMT'}, + }, + {'status_code': 304, 'text': ''}, + ], + ) + + enbw_service: EnBWDatex2ImportService = dependencies.get_import_services().importer_by_uid['datex2_enbw'] + enbw_service.fetch_static_data() + enbw_service.fetch_realtime_data() + + db.session.expire_all() + + source = enbw_service.get_source() + assert source.realtime_status == SourceStatus.ACTIVE + assert source.realtime_data_updated_at.replace(tzinfo=timezone.utc) == parsedate_to_datetime( + 'Sat, 19 Apr 2026 10:00:00 GMT', + ) + assert db.session.query(Evse).filter(Evse.uid == 'DE*EBW*E914082*2').first().status == EvseStatus.AVAILABLE diff --git a/tests/integration/services/import_services/ocpi/eaaze_pbw/eaaze_pbw_service_test.py b/tests/integration/services/import_services/ocpi/eaaze_pbw/eaaze_pbw_service_test.py index da86ec4c..ebddd714 100644 --- a/tests/integration/services/import_services/ocpi/eaaze_pbw/eaaze_pbw_service_test.py +++ b/tests/integration/services/import_services/ocpi/eaaze_pbw/eaaze_pbw_service_test.py @@ -34,7 +34,9 @@ def test_eaaze_pbw_sends_x_request_id(db: SQLAlchemy, requests_mock: Mocker) -> None: - """Every request to eaaze_pbw must carry an X-Request-ID header containing a random uuid4.""" + """ + Every request to eaaze_pbw must carry an X-Request-ID header containing a random uuid4. + """ eaaze_pbw_service: EaazePbwImportService = dependencies.get_import_services().importer_by_uid['eaaze_pbw'] requests_mock.get(SOURCE_URL, status_code=200, json=EMPTY_OCPI_RESPONSE) @@ -50,7 +52,9 @@ def test_eaaze_pbw_sends_x_request_id(db: SQLAlchemy, requests_mock: Mocker) -> def test_eaaze_pbw_x_request_id_is_unique_per_request(db: SQLAlchemy, requests_mock: Mocker) -> None: - """The X-Request-ID must be regenerated for each request, not reused.""" + """ + The X-Request-ID must be regenerated for each request, not reused. + """ eaaze_pbw_service: EaazePbwImportService = dependencies.get_import_services().importer_by_uid['eaaze_pbw'] requests_mock.get(SOURCE_URL, status_code=200, json=EMPTY_OCPI_RESPONSE) diff --git a/tests/unit/common/unset_parameter_test.py b/tests/unit/common/unset_parameter_test.py index 35349aa5..d249171c 100644 --- a/tests/unit/common/unset_parameter_test.py +++ b/tests/unit/common/unset_parameter_test.py @@ -22,11 +22,15 @@ class UnsetParameterTest: - """Tests for the UnsetParameter sentinel object and its type UnsetParameterType.""" + """ + Tests for the UnsetParameter sentinel object and its type UnsetParameterType. + """ @staticmethod def test_unset_parameter(): - """Test UnsetParameter and its magic methods.""" + """ + Test UnsetParameter and its magic methods. + """ assert type(UnsetParameter) is UnsetParameterType assert repr(UnsetParameter) == 'UnsetParameter' assert str(UnsetParameter) == '' @@ -34,7 +38,9 @@ def test_unset_parameter(): @staticmethod def test_unset_parameter_unique(): - """Test that UnsetParameter is a unique sentinel object, i.e. all UnsetParameter values are the same.""" + """ + Test that UnsetParameter is a unique sentinel object, i.e. all UnsetParameter values are the same. + """ unset_parameter1 = UnsetParameter unset_parameter2 = copy(unset_parameter1) unset_parameter3 = UnsetParameterType() diff --git a/tests/unit/common/validation/multi_dataclass_validator_test.py b/tests/unit/common/validation/multi_dataclass_validator_test.py index dcc1094f..2d7211fa 100644 --- a/tests/unit/common/validation/multi_dataclass_validator_test.py +++ b/tests/unit/common/validation/multi_dataclass_validator_test.py @@ -45,7 +45,9 @@ class _MixedShape(ValidataclassMixin): class MultiValidataclassValidatorTest: - """Tests for MultiValidataclassValidator and MultiValidataclassValidationError.""" + """ + Tests for MultiValidataclassValidator and MultiValidataclassValidationError. + """ @staticmethod def test_requires_at_least_one_dataclass(): diff --git a/tests/unit/common/validation/multi_dict_validator_test.py b/tests/unit/common/validation/multi_dict_validator_test.py index 32759071..0dcd3c35 100644 --- a/tests/unit/common/validation/multi_dict_validator_test.py +++ b/tests/unit/common/validation/multi_dict_validator_test.py @@ -41,7 +41,9 @@ def _mixed_shape_validator() -> DictValidator: class MultiDictValidatorTest: - """Tests for MultiDictValidator and MultiDictValidationError.""" + """ + Tests for MultiDictValidator and MultiDictValidationError. + """ @staticmethod def test_requires_at_least_one_validator(): diff --git a/tests/unit/models/capability_bits_test.py b/tests/unit/models/capability_bits_test.py index fed89a35..8eba1d8b 100644 --- a/tests/unit/models/capability_bits_test.py +++ b/tests/unit/models/capability_bits_test.py @@ -49,17 +49,23 @@ class CapabilityBitsTest: @staticmethod def test_bit_values_are_unchanged(): - """Every capability must keep the bit value it was stored with.""" + """ + Every capability must keep the bit value it was stored with. + """ assert {item.name: bit for bit, item in CAPABILITY_BITS} == EXPECTED_BITS @staticmethod def test_every_capability_has_a_bit(): - """A capability missing from the table would silently never be stored or returned.""" + """ + A capability missing from the table would silently never be stored or returned. + """ assert {item for _bit, item in CAPABILITY_BITS} == set(Capability) @staticmethod def test_bits_are_unique_and_single(): - """Each entry must be a distinct power of two, otherwise capabilities alias each other.""" + """ + Each entry must be a distinct power of two, otherwise capabilities alias each other. + """ bits = [bit for bit, _item in CAPABILITY_BITS] assert len(set(bits)) == len(bits) @@ -69,12 +75,16 @@ def test_bits_are_unique_and_single(): @staticmethod def test_bit_by_member_matches_the_table(): - """The lookup used by the setter must agree with the table used by the getter.""" + """ + The lookup used by the setter must agree with the table used by the getter. + """ assert CAPABILITY_BIT_BY_MEMBER == {item: bit for bit, item in CAPABILITY_BITS} @staticmethod def test_table_is_ordered_by_bit_value(): - """The getter returns capabilities in table order, which callers expect to be ascending by bit.""" + """ + The getter returns capabilities in table order, which callers expect to be ascending by bit. + """ bits = [bit for bit, _item in CAPABILITY_BITS] assert bits == sorted(bits) diff --git a/tests/unit/models/parking_restriction_bits_test.py b/tests/unit/models/parking_restriction_bits_test.py index 5127ebc2..ecf17c04 100644 --- a/tests/unit/models/parking_restriction_bits_test.py +++ b/tests/unit/models/parking_restriction_bits_test.py @@ -43,17 +43,23 @@ class ParkingRestrictionBitsTest: @staticmethod def test_bit_values_are_unchanged(): - """Every parking restriction must keep the bit value it was stored with.""" + """ + Every parking restriction must keep the bit value it was stored with. + """ assert {item.name: bit for bit, item in PARKING_RESTRICTION_BITS} == EXPECTED_BITS @staticmethod def test_every_parking_restriction_has_a_bit(): - """A restriction missing from the table would silently never be stored or returned.""" + """ + A restriction missing from the table would silently never be stored or returned. + """ assert {item for _bit, item in PARKING_RESTRICTION_BITS} == set(ParkingRestriction) @staticmethod def test_bits_are_unique_and_single(): - """Each entry must be a distinct power of two, otherwise restrictions alias each other.""" + """ + Each entry must be a distinct power of two, otherwise restrictions alias each other. + """ bits = [bit for bit, _item in PARKING_RESTRICTION_BITS] assert len(set(bits)) == len(bits) @@ -63,12 +69,16 @@ def test_bits_are_unique_and_single(): @staticmethod def test_bit_by_member_matches_the_table(): - """The lookup used by the setter must agree with the table used by the getter.""" + """ + The lookup used by the setter must agree with the table used by the getter. + """ assert PARKING_RESTRICTION_BIT_BY_MEMBER == {item: bit for bit, item in PARKING_RESTRICTION_BITS} @staticmethod def test_table_is_ordered_by_bit_value(): - """The getter returns restrictions in table order, which callers expect to be ascending by bit.""" + """ + The getter returns restrictions in table order, which callers expect to be ascending by bit. + """ bits = [bit for bit, _item in PARKING_RESTRICTION_BITS] assert bits == sorted(bits) diff --git a/tests/unit/services/import_services/datex2/datex2_v3_5_input_test.py b/tests/unit/services/import_services/datex2/datex2_v3_5_input_test.py new file mode 100644 index 00000000..48982287 --- /dev/null +++ b/tests/unit/services/import_services/datex2/datex2_v3_5_input_test.py @@ -0,0 +1,81 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +import json +from pathlib import Path + +from validataclass.validators import DataclassValidator + +from webapp.services.import_services.datex2.v3_5.base_datex2_v3_5_import_service import BaseDatex2V35ImportService +from webapp.services.import_services.datex2.v3_5.datex2_v3_5_json_static_mapper import Datex2V35JSONStaticMapper +from webapp.shared.datex2.models import MessageContainerWrapperInput, ProtocolTypeEnum +from webapp.shared.datex2.v3_5_json_static.models.location_reference_g_input import LocationReferenceGInput + +REALTIME_DATA_PATH = Path( + Path(__file__).parents[4], + 'integration', + 'services', + 'import_services', + 'datex2', + 'data', + 'datex2_enbw_realtime_reduced.json', +) + + +def _load_realtime_data() -> dict: + with REALTIME_DATA_PATH.open() as realtime_file: + return json.load(realtime_file) + + +def test_location_reference_without_facility_location() -> None: + location_reference = DataclassValidator(LocationReferenceGInput).validate({ + 'locPointLocation': {'pointByCoordinates': {'pointCoordinates': {'latitude': 47.67, 'longitude': 9.14}}}, + }) + + assert Datex2V35JSONStaticMapper._get_facility_location_by_location_reference(location_reference) is None + + +def test_message_container_with_empty_extensions() -> None: + data = _load_realtime_data() + data['messageContainer']['exchangeInformation']['dynamicInformation']['exDynamicInformationExtensionG'] = {} + + message_container = DataclassValidator(MessageContainerWrapperInput).validate(data) + + assert message_container.messageContainer.exchangeInformation.dynamicInformation is not None + + +def test_wrap_realtime_payload_wraps_d2_payload() -> None: + payload = _load_realtime_data()['messageContainer']['payload'][0] + + data = BaseDatex2V35ImportService.wrap_realtime_payload({'payload': payload}) + message_container = DataclassValidator(MessageContainerWrapperInput).validate(data) + + assert len(message_container.messageContainer.payload) == 1 + exchange_context = message_container.messageContainer.exchangeInformation.exchangeContext + assert exchange_context.codedExchangeProtocol.value is ProtocolTypeEnum.SNAPSHOT_PULL + + +def test_wrap_realtime_payload_keeps_other_documents() -> None: + message_container_data = _load_realtime_data() + payload = message_container_data['messageContainer']['payload'] + # A payload list beside exchange information is neither a D2Payload nor a MessageContainer document + invalid_data = {'payload': payload, 'exchangeInformation': message_container_data['messageContainer']} + + assert BaseDatex2V35ImportService.wrap_realtime_payload(message_container_data) is message_container_data + assert BaseDatex2V35ImportService.wrap_realtime_payload(invalid_data) is invalid_data + assert BaseDatex2V35ImportService.wrap_realtime_payload({'payload': payload}) == {'payload': payload} diff --git a/tests/unit/services/import_services/datex2/datex2_v3_7_realtime_input_test.py b/tests/unit/services/import_services/datex2/datex2_v3_7_realtime_input_test.py new file mode 100644 index 00000000..c8ae2009 --- /dev/null +++ b/tests/unit/services/import_services/datex2/datex2_v3_7_realtime_input_test.py @@ -0,0 +1,81 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +import json +from pathlib import Path + +import pytest +from validataclass.exceptions import ValidationError + +from webapp.services.import_services.datex2.v3_7.base_datex2_v3_7_import_service import BaseDatex2V37ImportService + +REALTIME_DATA_PATH = Path( + Path(__file__).parents[4], + 'integration', + 'services', + 'import_services', + 'datex2', + 'data', + 'datex2_giroe_realtime_reduced.json', +) + + +def _load_realtime_data() -> dict: + with REALTIME_DATA_PATH.open() as realtime_file: + return json.load(realtime_file) + + +def _count_refill_point_statuses(payloads: list) -> int: + return sum( + len(station_status.refillPointStatus) + for payload in payloads + for site_status in payload.aegiEnergyInfrastructureStatusPublication.energyInfrastructureSiteStatus + for station_status in site_status.energyInfrastructureStationStatus + ) + + +def test_get_realtime_payloads_from_message_container() -> None: + payloads = BaseDatex2V37ImportService.get_realtime_payloads(_load_realtime_data()) + + assert len(payloads) == 1 + assert _count_refill_point_statuses(payloads) == 8 + + +def test_get_realtime_payloads_from_d2_payload() -> None: + payload = _load_realtime_data()['messageContainer']['payload'][0] + + payloads = BaseDatex2V37ImportService.get_realtime_payloads({'payload': payload}) + + assert len(payloads) == 1 + assert _count_refill_point_statuses(payloads) == 8 + + +def test_get_realtime_payloads_rejects_message_container_without_wrapper() -> None: + # Payloads and exchange information outside a messageContainer field are no valid DATEX II document + data = _load_realtime_data()['messageContainer'] + + with pytest.raises(ValidationError): + BaseDatex2V37ImportService.get_realtime_payloads(data) + + +def test_get_realtime_payloads_rejects_unknown_message_container_fields() -> None: + data = _load_realtime_data() + data['messageContainer']['unknownField'] = True + + with pytest.raises(ValidationError): + BaseDatex2V37ImportService.get_realtime_payloads(data) diff --git a/tests/unit/services/import_services/datex2/tariff_grouping_mixin_test.py b/tests/unit/services/import_services/datex2/tariff_grouping_mixin_test.py index d8678ec4..df0b4400 100644 --- a/tests/unit/services/import_services/datex2/tariff_grouping_mixin_test.py +++ b/tests/unit/services/import_services/datex2/tariff_grouping_mixin_test.py @@ -40,7 +40,9 @@ class TariffGroupingService(TariffGroupingMixin): - """Minimal host for the mixin: it only needs a source_info to name the source in its log line.""" + """ + Minimal host for the mixin: it only needs a source_info to name the source in its log line. + """ source_info = SourceInfo( uid=SOURCE_UID, @@ -118,7 +120,9 @@ def _build_location_update(*tariff_updates: TariffUpdate) -> LocationUpdate: def _build_location_update_from_associations( *tariff_association_updates_per_evse: list[TariffAssociationUpdate], ) -> LocationUpdate: - """Wrap each list of tariff associations into its own EVSE.""" + """ + Wrap each list of tariff associations into its own EVSE. + """ evse_updates: list[EvseUpdate] = [] for index, tariff_association_updates in enumerate(tariff_association_updates_per_evse): evse_updates.append( @@ -147,7 +151,9 @@ def _build_location_update_from_associations( def _grouped_tariff_association_updates(*location_updates: LocationUpdate) -> list[TariffAssociationUpdate]: - """The tariff associations which survived the grouping, in the order the EVSEs hold them.""" + """ + The tariff associations which survived the grouping, in the order the EVSEs hold them. + """ return [ tariff_association_update for location_update in location_updates @@ -158,7 +164,9 @@ def _grouped_tariff_association_updates(*location_updates: LocationUpdate) -> li def _grouped_tariff_updates(*location_updates: LocationUpdate) -> list[TariffUpdate]: - """The tariffs which survived the grouping, in the order their associations appear.""" + """ + The tariffs which survived the grouping, in the order their associations appear. + """ return [ tariff_association_update.tariff for tariff_association_update in _grouped_tariff_association_updates(*location_updates) @@ -166,7 +174,9 @@ def _grouped_tariff_updates(*location_updates: LocationUpdate) -> list[TariffUpd def test_group_identical_tariffs_collapses_identical_fees() -> None: - """Tariffs with identical fees collapse into one tariff, no matter which uid they started with.""" + """ + Tariffs with identical fees collapse into one tariff, no matter which uid they started with. + """ tariff_updates = [_build_tariff_update(uid='first'), _build_tariff_update(uid='second')] location_update = _build_location_update(*tariff_updates) @@ -220,7 +230,9 @@ def test_group_identical_tariffs_keeps_differing_audiences_apart() -> None: def test_group_identical_tariffs_uses_newest_last_updated() -> None: - """A group is stamped with the newest last_updated of its members.""" + """ + A group is stamped with the newest last_updated of its members. + """ oldest = datetime(2026, 1, 1, 10, 0, tzinfo=timezone.utc) newest = datetime(2026, 5, 17, 8, 30, tzinfo=timezone.utc) middle = datetime(2026, 3, 2, 12, 0, tzinfo=timezone.utc) @@ -246,7 +258,9 @@ def test_group_identical_tariffs_uses_newest_last_updated() -> None: def test_group_identical_tariffs_uses_oldest_start_date_time() -> None: - """The grouped association starts when the first EVSE of its group started to charge these fees.""" + """ + The grouped association starts when the first EVSE of its group started to charge these fees. + """ oldest = datetime(2026, 1, 1, 10, 0, tzinfo=timezone.utc) newest = datetime(2026, 5, 17, 8, 30, tzinfo=timezone.utc) location_update = _build_location_update_from_associations( @@ -279,7 +293,9 @@ def test_group_identical_tariffs_keeps_timestamps_none_without_timestamps() -> N def test_group_identical_tariffs_keeps_differing_fees_apart() -> None: - """Any difference in the fees keeps the tariffs, and with them their associations, separate.""" + """ + Any difference in the fees keeps the tariffs, and with them their associations, separate. + """ reference = _build_tariff_update(uid='reference') tariff_updates = [ reference, @@ -302,7 +318,9 @@ def test_group_identical_tariffs_keeps_differing_fees_apart() -> None: def test_group_identical_tariffs_groups_across_locations() -> None: - """Grouping spans the whole import, not just a single location.""" + """ + Grouping spans the whole import, not just a single location. + """ first_location_update = _build_location_update(_build_tariff_update(uid='first')) second_location_update = _build_location_update(_build_tariff_update(uid='second')) diff --git a/tests/unit/services/import_services/eliso/eliso_validators_test.py b/tests/unit/services/import_services/eliso/eliso_validators_test.py new file mode 100644 index 00000000..8ac25c8e --- /dev/null +++ b/tests/unit/services/import_services/eliso/eliso_validators_test.py @@ -0,0 +1,111 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from datetime import datetime, timezone +from decimal import Decimal + +import pytest +from validataclass.exceptions import ValidationError +from validataclass.validators import DataclassValidator + +from webapp.models.connector import ConnectorFormat, ConnectorType, PowerType +from webapp.models.evse import EvseStatus +from webapp.services.import_services.eliso.eliso_validators import ElisoEvseStatusInput, ElisoLocationInput + + +def _get_location_dict(**kwargs) -> dict: + return { + 'city': 'Haßfurt', + 'address': 'Am Wasserwerk 2', + 'postalCode': '97437', + 'country_iso_3166_alpha_2': 'DE', + 'coordinates': {'latitude': 50.030589789349, 'longitude': 10.520250965406}, + 'operator_name': 'eliso GmbH', + 'hotline_number': '+49 711 40042100', + 'opening_time': '24/7', + 'mobilithek_last_updated_dts': '2026-10-28T04:04:51.23388+00:00', + 'evses': [ + { + 'evseId': 'DE*ELI*E5207046', + 'connectors': [{'maxPower': 135, 'powerType': 'DC', 'type_of_connector': 'Combo2/CCS (DC)'}], + }, + { + 'evseId': 'DE*ELI*E5207045', + 'connectors': [{'maxPower': 22, 'powerType': 'AC_3_PHASE', 'type_of_connector': 'Type 2 (AC)'}], + }, + ], + **kwargs, + } + + +def test_location_to_location_update() -> None: + location_input = DataclassValidator(ElisoLocationInput).validate(_get_location_dict()) + + location_update = location_input.to_location_update('eliso') + + assert location_update.uid == 'DE*ELI*E5207045' + assert location_update.country == 'DEU' + assert location_update.lat == Decimal('50.0305898') + assert location_update.twentyfourseven is True + assert location_update.operator.name == 'eliso GmbH' + evses = location_update.charging_pool[0].evses + assert [evse.evse_id for evse in evses] == ['DE*ELI*E5207046', 'DE*ELI*E5207045'] + assert evses[0].connectors[0].standard == ConnectorType.IEC_62196_T2_COMBO + assert evses[0].connectors[0].format == ConnectorFormat.CABLE + assert evses[0].connectors[0].power_type == PowerType.DC + assert evses[0].connectors[0].max_electric_power == 135000 + assert evses[1].connectors[0].standard == ConnectorType.IEC_62196_T2 + assert evses[1].connectors[0].format == ConnectorFormat.SOCKET + + +def test_location_with_unknown_connector_type() -> None: + location_dict = _get_location_dict() + location_dict['evses'][0]['connectors'][0]['type_of_connector'] = 'CHAdeMO' + + with pytest.raises(ValidationError): + DataclassValidator(ElisoLocationInput).validate(location_dict) + + +@pytest.mark.parametrize( + ('operational_status', 'availability_status', 'evse_status'), + [ + ('Operational', 'Not in use', EvseStatus.AVAILABLE), + ('Operational', 'In use', EvseStatus.CHARGING), + ('Operational', None, EvseStatus.UNKNOWN), + ('Non-operational', None, EvseStatus.INOPERATIVE), + ], +) +def test_evse_status_to_evse_realtime_update( + operational_status: str, + availability_status: str | None, + evse_status: EvseStatus, +) -> None: + evse_status_dict = { + 'evseId': 'DE*ELI*E3584570', + 'adhoc_price': 0.49, + 'operational_status': operational_status, + 'mobilithek_last_updated_dts': '2026-10-28T19:53:17.994069+00:00', + } + if availability_status is not None: + evse_status_dict['availability_status'] = availability_status + + evse_update = DataclassValidator(ElisoEvseStatusInput).validate(evse_status_dict).to_evse_realtime_update() + + assert evse_update.evse_id == 'DE*ELI*E3584570' + assert evse_update.status == evse_status + assert evse_update.status_last_updated == datetime(2026, 10, 28, 19, 53, 17, 994069, tzinfo=timezone.utc) diff --git a/webapp/public_api/ocpi/ocpi_redirect.py b/webapp/public_api/ocpi/ocpi_redirect.py index d029f7b7..307291d4 100644 --- a/webapp/public_api/ocpi/ocpi_redirect.py +++ b/webapp/public_api/ocpi/ocpi_redirect.py @@ -20,8 +20,10 @@ class OcpiPathRewriteMiddleware: - """WSGI middleware that rewrites /api/ocpi/ paths to /api/public/ocpi/ so the existing - OCPI endpoints are transparently available under both prefixes.""" + """ + WSGI middleware that rewrites /api/ocpi/ paths to /api/public/ocpi/ so the existing + OCPI endpoints are transparently available under both prefixes. + """ def __init__(self, app: Callable): self.app = app diff --git a/webapp/public_api/ocpi/v2_2/locations/rest_api.py b/webapp/public_api/ocpi/v2_2/locations/rest_api.py index ccecd6c5..c434dcbd 100644 --- a/webapp/public_api/ocpi/v2_2/locations/rest_api.py +++ b/webapp/public_api/ocpi/v2_2/locations/rest_api.py @@ -69,7 +69,9 @@ def __init__(self): class OcpiLegacyLocationBlueprint(BaseBlueprint): - """Backward compatibility: /api/ocpi/2.2/location""" + """ + Backward compatibility: /api/ocpi/2.2/location + """ location_handler: LocationHandler diff --git a/webapp/services/import_services/datex2/__init__.py b/webapp/services/import_services/datex2/__init__.py index 5c232005..c061c72b 100644 --- a/webapp/services/import_services/datex2/__init__.py +++ b/webapp/services/import_services/datex2/__init__.py @@ -17,27 +17,44 @@ """ from .v3_5 import ( + AhBeteiligungDatex2ImportService, AmpecoDatex2ImportService, + AudiDatex2ImportService, ChargecloudDatex2ImportService, + EaazeDatex2ImportService, EClearingNetDatex2ImportService, EcoMovementDatex2ImportService, EluMobilityDatex2ImportService, EnBWDatex2ImportService, + EnerandoDatex2ImportService, EnioDatex2ImportService, ERoundDatex2ImportService, + EulektroDatex2ImportService, + EvPriceDatex2ImportService, + EweDatex2ImportService, + FlaviaDatex2ImportService, + GpJouleDatex2ImportService, GridAndCoDatex2ImportService, InergiaDatex2ImportService, + InselwerkeDatex2ImportService, + KulRudnickDatex2ImportService, LadebusinessDatex2ImportService, + LadesonneDatex2ImportService, LichtblickDatex2ImportService, MidorionDatex2ImportService, MontaDatex2ImportService, MsuDatex2ImportService, PumpDatex2ImportService, QwelloDatex2ImportService, + RMBeteiligungDatex2ImportService, + RoadDatex2ImportService, SmatricsDatex2ImportService, + SonnenstromSaarpfalzDatex2ImportService, TaubertDatex2ImportService, TeslaDatex2ImportService, + UlmDatex2ImportService, VaylensDatex2ImportService, + VirtaDatex2ImportService, VolkswagenDatex2ImportService, WirelaneDatex2ImportService, ) diff --git a/webapp/services/import_services/datex2/v3_5/__init__.py b/webapp/services/import_services/datex2/v3_5/__init__.py index 22595673..89d51909 100644 --- a/webapp/services/import_services/datex2/v3_5/__init__.py +++ b/webapp/services/import_services/datex2/v3_5/__init__.py @@ -16,28 +16,45 @@ along with this program. If not, see . """ +from .ah_beteiligung_datex2_import_service import AhBeteiligungDatex2ImportService from .ampeco_datex2_import_service import AmpecoDatex2ImportService +from .audi_datex2_import_service import AudiDatex2ImportService from .base_datex2_v3_5_import_service import BaseDatex2V35ImportService, RealtimeResult from .chargecloud_datex2_import_service import ChargecloudDatex2ImportService from .datex2_v3_5_json_static_mapper import Datex2V35JSONStaticMapper from .e_clearing_net_datex2_import_service import EClearingNetDatex2ImportService +from .eaaze_datex2_import_service import EaazeDatex2ImportService from .ecomovement_datex2_import_service import EcoMovementDatex2ImportService from .elu_mobility_datex2_import_service import EluMobilityDatex2ImportService from .enbw_datex2_import_service import EnBWDatex2ImportService +from .enerando_datex2_import_service import EnerandoDatex2ImportService from .enio_datex2_import_service import EnioDatex2ImportService from .eround_datex2_import_service import ERoundDatex2ImportService +from .eulektro_datex2_import_service import EulektroDatex2ImportService +from .ev_price_datex2_import_service import EvPriceDatex2ImportService +from .ewe_datex2_import_service import EweDatex2ImportService +from .flavia_datex2_import_service import FlaviaDatex2ImportService +from .gp_joule_datex2_import_service import GpJouleDatex2ImportService from .gridandco_datex2_import_service import GridAndCoDatex2ImportService from .inergia_datex2_import_service import InergiaDatex2ImportService +from .inselwerke_datex2_import_service import InselwerkeDatex2ImportService +from .kul_rudnick_datex2_import_service import KulRudnickDatex2ImportService from .ladebusiness_datex2_import_service import LadebusinessDatex2ImportService +from .ladesonne_datex2_import_service import LadesonneDatex2ImportService from .lichtblick_datex2_import_service import LichtblickDatex2ImportService from .midorion_datex2_import_service import MidorionDatex2ImportService from .monta_datex2_import_service import MontaDatex2ImportService from .msu_datex2_import_service import MsuDatex2ImportService from .pump_datex2_import_service import PumpDatex2ImportService from .qwello_datex2_import_service import QwelloDatex2ImportService +from .r_m_beteiligung_datex2_import_service import RMBeteiligungDatex2ImportService +from .road_datex2_import_service import RoadDatex2ImportService from .smatrics_datex2_import_service import SmatricsDatex2ImportService +from .sonnenstrom_saarpfalz_datex2_import_service import SonnenstromSaarpfalzDatex2ImportService from .taubert_datex2_import_service import TaubertDatex2ImportService from .tesla_datex2_import_service import TeslaDatex2ImportService +from .ulm_datex2_import_service import UlmDatex2ImportService from .vaylens_datex2_import_service import VaylensDatex2ImportService +from .virta_datex2_import_service import VirtaDatex2ImportService from .volkswagen_datex2_import_service import VolkswagenDatex2ImportService from .wirelane_datex2_import_service import WirelaneDatex2ImportService diff --git a/webapp/services/import_services/datex2/v3_5/ah_beteiligung_datex2_import_service.py b/webapp/services/import_services/datex2/v3_5/ah_beteiligung_datex2_import_service.py new file mode 100644 index 00000000..56b1c976 --- /dev/null +++ b/webapp/services/import_services/datex2/v3_5/ah_beteiligung_datex2_import_service.py @@ -0,0 +1,31 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from webapp.services.import_services.datex2.v3_5.base_datex2_v3_5_import_service import BaseDatex2V35ImportService +from webapp.services.import_services.models import SourceInfo + + +class AhBeteiligungDatex2ImportService(BaseDatex2V35ImportService): + source_info = SourceInfo( + uid='datex2_ah_beteiligung', + name='AH Beteiligungsgesellschaft Datex II', + public_url='https://mobilithek.info/offers/993495185148837888', + source_url='https://mobilithek.info/offers/993495185148837888', + attribution_contributor='AH Beteiligungsgesellschaft mbH', + has_realtime_data=True, + ) diff --git a/webapp/services/import_services/datex2/v3_5/audi_datex2_import_service.py b/webapp/services/import_services/datex2/v3_5/audi_datex2_import_service.py new file mode 100644 index 00000000..69fccaf3 --- /dev/null +++ b/webapp/services/import_services/datex2/v3_5/audi_datex2_import_service.py @@ -0,0 +1,32 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from webapp.services.import_services.datex2.v3_5.base_datex2_v3_5_import_service import BaseDatex2V35ImportService +from webapp.services.import_services.models import SourceInfo + + +class AudiDatex2ImportService(BaseDatex2V35ImportService): + source_info = SourceInfo( + uid='datex2_audi', + name='Audi charging hub Datex II', + public_url='https://mobilithek.info/offers/998571342932365312', + source_url='https://mobilithek.info/offers/998571342932365312', + attribution_contributor='Audi AG', + attribution_license='CC-0', + has_realtime_data=True, + ) diff --git a/webapp/services/import_services/datex2/v3_5/base_datex2_v3_5_import_service.py b/webapp/services/import_services/datex2/v3_5/base_datex2_v3_5_import_service.py index a97bd760..3f764971 100644 --- a/webapp/services/import_services/datex2/v3_5/base_datex2_v3_5_import_service.py +++ b/webapp/services/import_services/datex2/v3_5/base_datex2_v3_5_import_service.py @@ -156,6 +156,10 @@ def fetch_realtime_data(self): try: data, last_modified = self.request_data(subscription_id, if_modified_since) except RemoteException as e: + # Mobilithek answers with 304 Not Modified if there is no newer data + if e.http_status == 304: + last_modified = if_modified_since + break logger.error( f'Datex2 realtime request failed: {e.to_dict()}. Text: {e.data}', extra={'attributes': {'type': LogMessageType.IMPORT_SOURCE}}, @@ -172,7 +176,7 @@ def fetch_realtime_data(self): return try: - message_container = self.validate_realtime_data(data) + message_container = self.validate_realtime_data(self.wrap_realtime_payload(data)) self.add_realtime_data(message_container, result) except ImportException as e: logger.error( @@ -261,7 +265,9 @@ def mark_realtime_push_received( @staticmethod def get_publication_time(message_container: MessageContainerWrapperInput) -> datetime | None: - """Return the payload's ``publicationTime``, or ``None`` if the payload does not carry one.""" + """ + Return the payload's ``publicationTime``, or ``None`` if the payload does not carry one. + """ if not message_container.messageContainer.payload: return None @@ -303,6 +309,32 @@ def store_realtime_data(self, message_container: MessageContainerWrapperInput) - extra={'attributes': {'type': LogMessageType.IMPORT_LOCATION}}, ) + @staticmethod + def wrap_realtime_payload(data: dict) -> dict: + """ + Realtime data can also be a plain D2Payload document instead of a MessageContainer. As the import works with + MessageContainers, the payload gets wrapped in one with a generic snapshotPull exchange information. + """ + if set(data.keys()) != {'payload'} or not isinstance(data['payload'], dict): + return data + + return { + 'messageContainer': { + 'payload': [data['payload']], + 'exchangeInformation': { + 'exchangeContext': { + 'codedExchangeProtocol': {'value': ProtocolTypeEnum.SNAPSHOT_PULL.value}, + 'exchangeSpecificationVersion': '3.5', + 'supplierOrCisRequester': {}, + }, + 'dynamicInformation': { + 'exchangeStatus': {'value': 'online'}, + 'messageGenerationTimestamp': datetime.now(tz=timezone.utc).isoformat(), + }, + }, + }, + } + def validate_realtime_data(self, data: dict) -> MessageContainerWrapperInput: """ Validate the raw DATEX II v3.5 realtime envelope and return the parsed message container. @@ -339,7 +371,9 @@ def validate_realtime_data(self, data: dict) -> MessageContainerWrapperInput: @staticmethod def count_station_statuses(message_container: MessageContainerWrapperInput) -> int: - """Return the number of ``energyInfrastructureStationStatus`` entries carried by the payload.""" + """ + Return the number of ``energyInfrastructureStationStatus`` entries carried by the payload. + """ payload = message_container.messageContainer.payload[0] publication = payload.aegiEnergyInfrastructureStatusPublication if publication is UnsetValue or publication.energyInfrastructureSiteStatus is UnsetValue: diff --git a/webapp/services/import_services/datex2/v3_5/eaaze_datex2_import_service.py b/webapp/services/import_services/datex2/v3_5/eaaze_datex2_import_service.py new file mode 100644 index 00000000..49201810 --- /dev/null +++ b/webapp/services/import_services/datex2/v3_5/eaaze_datex2_import_service.py @@ -0,0 +1,32 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from webapp.services.import_services.datex2.v3_5.base_datex2_v3_5_import_service import BaseDatex2V35ImportService +from webapp.services.import_services.models import SourceInfo + + +class EaazeDatex2ImportService(BaseDatex2V35ImportService): + source_info = SourceInfo( + uid='datex2_eaaze', + name='EAAZE Datex II', + public_url='https://mobilithek.info/offers/980426161955500032', + source_url='https://mobilithek.info/offers/980426161955500032', + attribution_contributor='EAAZE GmbH', + attribution_license='CC-0', + has_realtime_data=True, + ) diff --git a/webapp/services/import_services/datex2/v3_5/enerando_datex2_import_service.py b/webapp/services/import_services/datex2/v3_5/enerando_datex2_import_service.py new file mode 100644 index 00000000..88076fc8 --- /dev/null +++ b/webapp/services/import_services/datex2/v3_5/enerando_datex2_import_service.py @@ -0,0 +1,31 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from webapp.services.import_services.datex2.v3_5.base_datex2_v3_5_import_service import BaseDatex2V35ImportService +from webapp.services.import_services.models import SourceInfo + + +class EnerandoDatex2ImportService(BaseDatex2V35ImportService): + source_info = SourceInfo( + uid='datex2_enerando', + name='ENERANDO Datex II', + public_url='https://mobilithek.info/offers/1037036344672452608', + source_url='https://mobilithek.info/offers/1037036344672452608', + attribution_contributor='ENERANDO Technologies GmbH', + has_realtime_data=True, + ) diff --git a/webapp/services/import_services/datex2/v3_5/eulektro_datex2_import_service.py b/webapp/services/import_services/datex2/v3_5/eulektro_datex2_import_service.py new file mode 100644 index 00000000..fbd9a8ff --- /dev/null +++ b/webapp/services/import_services/datex2/v3_5/eulektro_datex2_import_service.py @@ -0,0 +1,31 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from webapp.services.import_services.datex2.v3_5.base_datex2_v3_5_import_service import BaseDatex2V35ImportService +from webapp.services.import_services.models import SourceInfo + + +class EulektroDatex2ImportService(BaseDatex2V35ImportService): + source_info = SourceInfo( + uid='datex2_eulektro', + name='Eulektro Datex II', + public_url='https://mobilithek.info/offers/973938793916375040', + source_url='https://mobilithek.info/offers/973938793916375040', + attribution_contributor='Eulektro GmbH', + has_realtime_data=True, + ) diff --git a/webapp/services/import_services/datex2/v3_5/ev_price_datex2_import_service.py b/webapp/services/import_services/datex2/v3_5/ev_price_datex2_import_service.py new file mode 100644 index 00000000..1f12be34 --- /dev/null +++ b/webapp/services/import_services/datex2/v3_5/ev_price_datex2_import_service.py @@ -0,0 +1,32 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from webapp.services.import_services.datex2.v3_5.base_datex2_v3_5_import_service import BaseDatex2V35ImportService +from webapp.services.import_services.models import SourceInfo + + +class EvPriceDatex2ImportService(BaseDatex2V35ImportService): + source_info = SourceInfo( + uid='datex2_ev_price', + name='EV Price Datex II', + public_url='https://mobilithek.info/offers/989311807176560640', + source_url='https://mobilithek.info/offers/989311807176560640', + attribution_contributor='EW Pricing GmbH', + attribution_license='CC-0', + has_realtime_data=True, + ) diff --git a/webapp/services/import_services/datex2/v3_5/ewe_datex2_import_service.py b/webapp/services/import_services/datex2/v3_5/ewe_datex2_import_service.py new file mode 100644 index 00000000..39d18d2f --- /dev/null +++ b/webapp/services/import_services/datex2/v3_5/ewe_datex2_import_service.py @@ -0,0 +1,32 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from webapp.services.import_services.datex2.v3_5.base_datex2_v3_5_import_service import BaseDatex2V35ImportService +from webapp.services.import_services.models import SourceInfo + + +class EweDatex2ImportService(BaseDatex2V35ImportService): + source_info = SourceInfo( + uid='datex2_ewe', + name='EWE Datex II', + public_url='https://mobilithek.info/offers/1024264572575293440', + source_url='https://mobilithek.info/offers/1024264572575293440', + attribution_contributor='EWE', + attribution_license='CC-0', + has_realtime_data=True, + ) diff --git a/webapp/services/import_services/datex2/v3_5/flavia_datex2_import_service.py b/webapp/services/import_services/datex2/v3_5/flavia_datex2_import_service.py new file mode 100644 index 00000000..267a5d3b --- /dev/null +++ b/webapp/services/import_services/datex2/v3_5/flavia_datex2_import_service.py @@ -0,0 +1,31 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from webapp.services.import_services.datex2.v3_5.base_datex2_v3_5_import_service import BaseDatex2V35ImportService +from webapp.services.import_services.models import SourceInfo + + +class FlaviaDatex2ImportService(BaseDatex2V35ImportService): + source_info = SourceInfo( + uid='datex2_flavia', + name='Flavia Datex II', + public_url='https://mobilithek.info/offers/1004450433724211200', + source_url='https://mobilithek.info/offers/1004450433724211200', + attribution_contributor='Grid & Co. GmbH', + has_realtime_data=True, + ) diff --git a/webapp/services/import_services/datex2/v3_5/gp_joule_datex2_import_service.py b/webapp/services/import_services/datex2/v3_5/gp_joule_datex2_import_service.py new file mode 100644 index 00000000..dbb0f21d --- /dev/null +++ b/webapp/services/import_services/datex2/v3_5/gp_joule_datex2_import_service.py @@ -0,0 +1,32 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from webapp.services.import_services.datex2.v3_5.base_datex2_v3_5_import_service import BaseDatex2V35ImportService +from webapp.services.import_services.models import SourceInfo + + +class GpJouleDatex2ImportService(BaseDatex2V35ImportService): + source_info = SourceInfo( + uid='datex2_gp_joule', + name='GP JOULE Connect Datex II', + public_url='https://mobilithek.info/offers/997111469996658688', + source_url='https://mobilithek.info/offers/997111469996658688', + attribution_contributor='GP JOULE Connect GmbH', + attribution_license='CC-0', + has_realtime_data=True, + ) diff --git a/webapp/services/import_services/datex2/v3_5/inselwerke_datex2_import_service.py b/webapp/services/import_services/datex2/v3_5/inselwerke_datex2_import_service.py new file mode 100644 index 00000000..52b75c53 --- /dev/null +++ b/webapp/services/import_services/datex2/v3_5/inselwerke_datex2_import_service.py @@ -0,0 +1,31 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from webapp.services.import_services.datex2.v3_5.base_datex2_v3_5_import_service import BaseDatex2V35ImportService +from webapp.services.import_services.models import SourceInfo + + +class InselwerkeDatex2ImportService(BaseDatex2V35ImportService): + source_info = SourceInfo( + uid='datex2_inselwerke', + name='Inselwerke Datex II', + public_url='https://mobilithek.info/offers/981180413145477120', + source_url='https://mobilithek.info/offers/981180413145477120', + attribution_contributor='Inselwerke eG', + has_realtime_data=True, + ) diff --git a/webapp/services/import_services/datex2/v3_5/kul_rudnick_datex2_import_service.py b/webapp/services/import_services/datex2/v3_5/kul_rudnick_datex2_import_service.py new file mode 100644 index 00000000..0bb68989 --- /dev/null +++ b/webapp/services/import_services/datex2/v3_5/kul_rudnick_datex2_import_service.py @@ -0,0 +1,32 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from webapp.services.import_services.datex2.v3_5.base_datex2_v3_5_import_service import BaseDatex2V35ImportService +from webapp.services.import_services.models import SourceInfo + + +class KulRudnickDatex2ImportService(BaseDatex2V35ImportService): + source_info = SourceInfo( + uid='datex2_kul_rudnick', + name='K.u.L. Rudnick Datex II', + public_url='https://mobilithek.info/offers/983747474988376064', + source_url='https://mobilithek.info/offers/983747474988376064', + attribution_contributor='K.u.L. Rudnick Bau und Bauträger GmbH', + attribution_license='CC-0', + has_realtime_data=False, + ) diff --git a/webapp/services/import_services/datex2/v3_5/ladesonne_datex2_import_service.py b/webapp/services/import_services/datex2/v3_5/ladesonne_datex2_import_service.py new file mode 100644 index 00000000..75beed30 --- /dev/null +++ b/webapp/services/import_services/datex2/v3_5/ladesonne_datex2_import_service.py @@ -0,0 +1,31 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from webapp.services.import_services.datex2.v3_5.base_datex2_v3_5_import_service import BaseDatex2V35ImportService +from webapp.services.import_services.models import SourceInfo + + +class LadesonneDatex2ImportService(BaseDatex2V35ImportService): + source_info = SourceInfo( + uid='datex2_ladesonne', + name='Ladesonne Datex II', + public_url='https://mobilithek.info/offers/988133177339846656', + source_url='https://mobilithek.info/offers/988133177339846656', + attribution_contributor='Ladesonne GmbH & Co. KG', + has_realtime_data=True, + ) diff --git a/webapp/services/import_services/datex2/v3_5/r_m_beteiligung_datex2_import_service.py b/webapp/services/import_services/datex2/v3_5/r_m_beteiligung_datex2_import_service.py new file mode 100644 index 00000000..8c5baec6 --- /dev/null +++ b/webapp/services/import_services/datex2/v3_5/r_m_beteiligung_datex2_import_service.py @@ -0,0 +1,31 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from webapp.services.import_services.datex2.v3_5.base_datex2_v3_5_import_service import BaseDatex2V35ImportService +from webapp.services.import_services.models import SourceInfo + + +class RMBeteiligungDatex2ImportService(BaseDatex2V35ImportService): + source_info = SourceInfo( + uid='datex2_r_m_beteiligung', + name='R & M Beteiligungsgesellschaft Datex II', + public_url='https://mobilithek.info/offers/993499364131065856', + source_url='https://mobilithek.info/offers/993499364131065856', + attribution_contributor='R & M Beteiligungsgesellschaft mbH', + has_realtime_data=True, + ) diff --git a/webapp/services/import_services/datex2/v3_5/road_datex2_import_service.py b/webapp/services/import_services/datex2/v3_5/road_datex2_import_service.py new file mode 100644 index 00000000..f9f5a4c3 --- /dev/null +++ b/webapp/services/import_services/datex2/v3_5/road_datex2_import_service.py @@ -0,0 +1,31 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from webapp.services.import_services.datex2.v3_5.base_datex2_v3_5_import_service import BaseDatex2V35ImportService +from webapp.services.import_services.models import SourceInfo + + +class RoadDatex2ImportService(BaseDatex2V35ImportService): + source_info = SourceInfo( + uid='datex2_road', + name='Road Datex II', + public_url='https://mobilithek.info/offers/1021366453630017536', + source_url='https://mobilithek.info/offers/1021366453630017536', + attribution_contributor='Road B.V.', + has_realtime_data=True, + ) diff --git a/webapp/services/import_services/datex2/v3_5/sonnenstrom_saarpfalz_datex2_import_service.py b/webapp/services/import_services/datex2/v3_5/sonnenstrom_saarpfalz_datex2_import_service.py new file mode 100644 index 00000000..95599677 --- /dev/null +++ b/webapp/services/import_services/datex2/v3_5/sonnenstrom_saarpfalz_datex2_import_service.py @@ -0,0 +1,31 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from webapp.services.import_services.datex2.v3_5.base_datex2_v3_5_import_service import BaseDatex2V35ImportService +from webapp.services.import_services.models import SourceInfo + + +class SonnenstromSaarpfalzDatex2ImportService(BaseDatex2V35ImportService): + source_info = SourceInfo( + uid='datex2_sonnenstrom_saarpfalz', + name='Sonnenstrom Saarpfalz Datex II', + public_url='https://mobilithek.info/offers/979427073235566592', + source_url='https://mobilithek.info/offers/979427073235566592', + attribution_contributor='Sonnenstrom Saarpfalz OHG', + has_realtime_data=True, + ) diff --git a/webapp/services/import_services/datex2/v3_5/ulm_datex2_import_service.py b/webapp/services/import_services/datex2/v3_5/ulm_datex2_import_service.py new file mode 100644 index 00000000..85adf054 --- /dev/null +++ b/webapp/services/import_services/datex2/v3_5/ulm_datex2_import_service.py @@ -0,0 +1,32 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from webapp.services.import_services.datex2.v3_5.base_datex2_v3_5_import_service import BaseDatex2V35ImportService +from webapp.services.import_services.models import SourceInfo + + +class UlmDatex2ImportService(BaseDatex2V35ImportService): + source_info = SourceInfo( + uid='datex2_ulm', + name='Ulm Datex II', + public_url='https://mobilithek.info/offers/854410608351543296', + source_url='https://mobilithek.info/offers/854410608351543296', + attribution_contributor='Smartlab Innovationsgesellschaft mbH', + attribution_license='CC-0', + has_realtime_data=True, + ) diff --git a/webapp/services/import_services/datex2/v3_5/virta_datex2_import_service.py b/webapp/services/import_services/datex2/v3_5/virta_datex2_import_service.py new file mode 100644 index 00000000..b19ed38f --- /dev/null +++ b/webapp/services/import_services/datex2/v3_5/virta_datex2_import_service.py @@ -0,0 +1,32 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from webapp.services.import_services.datex2.v3_5.base_datex2_v3_5_import_service import BaseDatex2V35ImportService +from webapp.services.import_services.models import SourceInfo + + +class VirtaDatex2ImportService(BaseDatex2V35ImportService): + source_info = SourceInfo( + uid='datex2_virta', + name='Virta Datex II', + public_url='https://mobilithek.info/offers/1042468553977937920', + source_url='https://mobilithek.info/offers/1042468553977937920', + attribution_contributor='Virta International GmbH', + attribution_license='CC-0', + has_realtime_data=True, + ) diff --git a/webapp/services/import_services/datex2/v3_5/volkswagen_datex2_import_service.py b/webapp/services/import_services/datex2/v3_5/volkswagen_datex2_import_service.py index cbd03843..0523a1df 100644 --- a/webapp/services/import_services/datex2/v3_5/volkswagen_datex2_import_service.py +++ b/webapp/services/import_services/datex2/v3_5/volkswagen_datex2_import_service.py @@ -24,8 +24,8 @@ class VolkswagenDatex2ImportService(BaseDatex2V35ImportService): source_info = SourceInfo( uid='datex2_volkswagen', name='Volkswagen Charging Group Datex II', - public_url='https://mobilithek.info/offers/976221024898781184', - source_url='https://mobilithek.info/offers/976221024898781184', + public_url='https://mobilithek.info/offers/983008874583728128', + source_url='https://mobilithek.info/offers/983008874583728128', attribution_contributor='Volkswagen Charging Group', attribution_license='CC-0', has_realtime_data=True, diff --git a/webapp/services/import_services/datex2/v3_7/base_datex2_v3_7_import_service.py b/webapp/services/import_services/datex2/v3_7/base_datex2_v3_7_import_service.py index adcb778c..710ae850 100644 --- a/webapp/services/import_services/datex2/v3_7/base_datex2_v3_7_import_service.py +++ b/webapp/services/import_services/datex2/v3_7/base_datex2_v3_7_import_service.py @@ -34,9 +34,13 @@ from webapp.services.import_services.datex2.tariff_grouping_mixin import TariffGroupingMixin from webapp.services.import_services.exceptions import ImportException from webapp.services.import_services.models import EvseRealtimeUpdate, LocationUpdate +from webapp.shared.datex2.models import MessageContainerWrapperV37RealtimeInput from webapp.shared.datex2.v3_7.realtime.d_a_t_e_x_i_i3_d2_payload_input import ( DATEXII3D2PayloadInput as DATEXII3D2RealtimePayloadInput, ) +from webapp.shared.datex2.v3_7.realtime.payload_publication_g_input import ( + PayloadPublicationGInput as PayloadPublicationGRealtimeInput, +) from webapp.shared.datex2.v3_7.static.d_a_t_e_x_i_i3_d2_payload_input import ( DATEXII3D2PayloadInput as DATEXII3D2StaticPayloadInput, ) @@ -56,6 +60,7 @@ class RealtimeResult: class BaseDatex2V37ImportService(BaseImportService, TariffGroupingMixin, ABC): v3_7_json_static_datex_validator = DataclassValidator(DATEXII3D2StaticPayloadInput) v3_7_json_realtime_datex_validator = DataclassValidator(DATEXII3D2RealtimePayloadInput) + v3_7_json_realtime_message_container_validator = DataclassValidator(MessageContainerWrapperV37RealtimeInput) v3_7_json_static_datex_mapper = Datex2V37JSONStaticMapper() def fetch_static_data(self): @@ -151,6 +156,10 @@ def fetch_realtime_data(self): try: data, last_modified = self.request_data(subscription_id, if_modified_since) except RemoteException as e: + # Mobilithek answers with 304 Not Modified if there is no newer data + if e.http_status == 304: + last_modified = if_modified_since + break logger.error( f'Datex2 realtime request failed: {e.to_dict()}. Text: {e.data}', extra={'attributes': {'type': LogMessageType.IMPORT_SOURCE}}, @@ -201,45 +210,61 @@ def fetch_realtime_data(self): extra={'attributes': {'type': LogMessageType.IMPORT_LOCATION}}, ) - def add_realtime_data(self, data: dict, result: RealtimeResult) -> DATEXII3D2RealtimePayloadInput: + @classmethod + def get_realtime_payloads(cls, data: dict) -> list[PayloadPublicationGRealtimeInput]: + """ + Realtime data is either a MessageContainer document, as in the Mobilithek AFIR profile, or a D2Payload document. + + Raises ValidationError if the data matches neither of them. + """ + if 'messageContainer' in data: + message_container_input: MessageContainerWrapperV37RealtimeInput = ( + cls.v3_7_json_realtime_message_container_validator.validate(data) + ) + return message_container_input.messageContainer.payload + + datex_input: DATEXII3D2RealtimePayloadInput = cls.v3_7_json_realtime_datex_validator.validate(data) + if datex_input.payload is UnsetValue: + return [] + return [datex_input.payload] + + def add_realtime_data(self, data: dict, result: RealtimeResult) -> None: try: - datex_input: DATEXII3D2RealtimePayloadInput = self.v3_7_json_realtime_datex_validator.validate(data) + payloads = self.get_realtime_payloads(data) except ValidationError as e: raise ImportException( - message=f'missing payload in {self.source_info.uid} realtime data', + message=f'invalid messageContainer or payload in {self.source_info.uid} realtime data', ) from e - if datex_input.payload is UnsetValue: + if not payloads and 'messageContainer' not in data: raise ImportException( message=f'missing payload in {self.source_info.uid} realtime data', ) - payload = datex_input.payload - if ( - payload.aegiEnergyInfrastructureStatusPublication is UnsetValue - or payload.aegiEnergyInfrastructureStatusPublication.energyInfrastructureSiteStatus is UnsetValue - ): - raise ImportException( - message=f'missing aegiEnergyInfrastructureStatusPublication in {self.source_info.uid} realtime data', - ) + for payload in payloads: + if ( + payload.aegiEnergyInfrastructureStatusPublication is UnsetValue + or payload.aegiEnergyInfrastructureStatusPublication.energyInfrastructureSiteStatus is UnsetValue + ): + raise ImportException( + message=f'missing aegiEnergyInfrastructureStatusPublication in {self.source_info.uid} realtime data', + ) - for site_status in payload.aegiEnergyInfrastructureStatusPublication.energyInfrastructureSiteStatus: - if site_status.energyInfrastructureStationStatus is UnsetValue: - continue - for station_status in site_status.energyInfrastructureStationStatus: - if station_status.refillPointStatus is UnsetValue: + for site_status in payload.aegiEnergyInfrastructureStatusPublication.energyInfrastructureSiteStatus: + if site_status.energyInfrastructureStationStatus is UnsetValue: continue - for refill_point_status_g in station_status.refillPointStatus: - evse_update = self.v3_7_json_static_datex_mapper.map_energy_infrastructure_station_status( - refill_point_status_g, - ) - if evse_update is None: - result.realtime_error_count += 1 + for station_status in site_status.energyInfrastructureStationStatus: + if station_status.refillPointStatus is UnsetValue: continue - result.evse_updates_by_evse[evse_update.evse_id] = evse_update - result.realtime_success_count += 1 - - return datex_input + for refill_point_status_g in station_status.refillPointStatus: + evse_update = self.v3_7_json_static_datex_mapper.map_energy_infrastructure_station_status( + refill_point_status_g, + ) + if evse_update is None: + result.realtime_error_count += 1 + continue + result.evse_updates_by_evse[evse_update.evse_id] = evse_update + result.realtime_success_count += 1 def request_data( self, diff --git a/webapp/services/import_services/eliso/__init__.py b/webapp/services/import_services/eliso/__init__.py new file mode 100644 index 00000000..6c7e0051 --- /dev/null +++ b/webapp/services/import_services/eliso/__init__.py @@ -0,0 +1,19 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from .eliso_service import ElisoImportService diff --git a/webapp/services/import_services/eliso/eliso_service.py b/webapp/services/import_services/eliso/eliso_service.py new file mode 100644 index 00000000..20dabd03 --- /dev/null +++ b/webapp/services/import_services/eliso/eliso_service.py @@ -0,0 +1,158 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +import logging +from datetime import datetime, timezone + +from validataclass.exceptions import ValidationError +from validataclass.validators import AnythingValidator, DataclassValidator, ListValidator + +from webapp.common.error_handling.exceptions import RemoteException +from webapp.common.logging.models import LogMessageType +from webapp.common.rest.exceptions import IncompleteConfigException +from webapp.models import SourceStatus +from webapp.services.import_services.base_import_service import BaseImportService +from webapp.services.import_services.models import EvseRealtimeUpdate, LocationUpdate, SourceInfo + +from .eliso_validators import ElisoEvseStatusInput, ElisoLocationInput + +logger = logging.getLogger(__name__) + + +class ElisoImportService(BaseImportService): + """ + eliso publishes its AFIR data at the Mobilithek in a custom JSON format instead of DATEX II. Static data is a list + of locations, realtime data is a snapshot of all EVSE statuses, with one entry per EVSE and price. + """ + + source_info = SourceInfo( + uid='eliso', + name='eliso', + public_url='https://mobilithek.info/offers/843477276990078976', + source_url='https://mobilithek.info/offers/843477276990078976', + attribution_contributor='eliso GmbH', + has_realtime_data=True, + ) + + list_validator = ListValidator(AnythingValidator(allowed_types=[dict])) + location_validator = DataclassValidator(ElisoLocationInput) + evse_status_validator = DataclassValidator(ElisoEvseStatusInput) + + def fetch_static_data(self): + source = self.get_source() + static_data_updated_at = datetime.now(tz=timezone.utc) + + try: + location_dicts = self.list_validator.validate(self.request_data('static_subscription_id')) + except (RemoteException, ValidationError) as e: + logger.error( + f'{self.source_info.uid} static data has error: {e.to_dict()}', + extra={'attributes': {'type': LogMessageType.IMPORT_SOURCE}}, + ) + self.update_source(source, static_status=SourceStatus.FAILED) + return + + location_updates: list[LocationUpdate] = [] + error_count = 0 + for location_dict in location_dicts: + try: + location_input: ElisoLocationInput = self.location_validator.validate(location_dict) + except ValidationError as e: + logger.warning( + f'{self.source_info.uid} location has error: {e.to_dict()}', + extra={'attributes': {'type': LogMessageType.IMPORT_SOURCE}}, + ) + error_count += 1 + continue + location_updates.append(location_input.to_location_update(self.source_info.uid)) + + self.save_location_updates(location_updates) + + self.update_source( + source=source, + static_status=SourceStatus.ACTIVE, + static_error_count=error_count, + static_data_updated_at=static_data_updated_at, + ) + logger.info( + f'Successfully updated {self.source_info.uid} static data with {len(location_updates)} valid ' + f'locations and {error_count} failed locations.', + extra={'attributes': {'type': LogMessageType.IMPORT_LOCATION}}, + ) + + def fetch_realtime_data(self): + source = self.get_source() + if source.static_status != SourceStatus.ACTIVE: + return + + realtime_data_updated_at = datetime.now(tz=timezone.utc) + + try: + data = self.request_data('realtime_subscription_id') + if not isinstance(data, dict): + raise ValidationError(reason='realtime data is no object') + evse_status_dicts = self.list_validator.validate(data.get('evses')) + except (RemoteException, ValidationError) as e: + logger.error( + f'{self.source_info.uid} realtime data has error: {e.to_dict()}', + extra={'attributes': {'type': LogMessageType.IMPORT_SOURCE}}, + ) + self.update_source(source, realtime_status=SourceStatus.FAILED) + return + + # There is one entry per EVSE and price, all with the same status + evse_updates_by_uid: dict[str, EvseRealtimeUpdate] = {} + error_count = 0 + for evse_status_dict in evse_status_dicts: + try: + evse_status_input: ElisoEvseStatusInput = self.evse_status_validator.validate(evse_status_dict) + except ValidationError as e: + logger.warning( + f'{self.source_info.uid} EVSE status has error: {e.to_dict()}', + extra={'attributes': {'type': LogMessageType.IMPORT_SOURCE}}, + ) + error_count += 1 + continue + evse_updates_by_uid[evse_status_input.evseId] = evse_status_input.to_evse_realtime_update() + + self.save_evse_updates(list(evse_updates_by_uid.values())) + + self.update_source( + source=source, + realtime_status=SourceStatus.ACTIVE, + realtime_error_count=error_count, + realtime_data_updated_at=realtime_data_updated_at, + ) + logger.info( + f'Successfully updated {self.source_info.uid} realtime data with {len(evse_updates_by_uid)} valid EVSEs ' + f'and {error_count} failed EVSEs.', + extra={'attributes': {'type': LogMessageType.IMPORT_LOCATION}}, + ) + + def request_data(self, subscription_config_key: str) -> dict | list: + subscription_id: int | None = self.config.get(subscription_config_key) + if subscription_id is None: + raise IncompleteConfigException(message=f'Missing {subscription_config_key} in config') + + return self.json_request( + url=f'https://mobilithek.info:8443/mobilithek/api/v1.0/subscription?subscriptionID={subscription_id}', + cert=( + self.config.get('mobilithek_cert_path'), + self.config.get('mobilithek_key_path'), + ), + ) diff --git a/webapp/services/import_services/eliso/eliso_validators.py b/webapp/services/import_services/eliso/eliso_validators.py new file mode 100644 index 00000000..36bf75ee --- /dev/null +++ b/webapp/services/import_services/eliso/eliso_validators.py @@ -0,0 +1,176 @@ +""" +Open ChargePoint DataBase OCPDB +Copyright (C) 2026 binary butterfly GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from datetime import datetime +from decimal import Decimal +from enum import Enum + +from pycountry import countries +from validataclass.dataclasses import Default, validataclass +from validataclass.validators import ( + DataclassValidator, + DateTimeValidator, + EnumValidator, + FloatToDecimalValidator, + IntegerValidator, + ListValidator, + StringValidator, +) + +from webapp.models.connector import ConnectorFormat, ConnectorType, PowerType +from webapp.models.evse import EvseStatus +from webapp.services.import_services.models import ( + BusinessUpdate, + ChargingStationUpdate, + ConnectorUpdate, + EvseRealtimeUpdate, + EvseUpdate, + LocationUpdate, +) + + +class ElisoConnectorType(Enum): + CCS = 'Combo2/CCS (DC)' + TYPE_2 = 'Type 2 (AC)' + + def to_connector_type(self) -> ConnectorType: + return { + self.CCS: ConnectorType.IEC_62196_T2_COMBO, + self.TYPE_2: ConnectorType.IEC_62196_T2, + }.get(self) + + def to_connector_format(self) -> ConnectorFormat: + return { + self.CCS: ConnectorFormat.CABLE, + self.TYPE_2: ConnectorFormat.SOCKET, + }.get(self) + + +class ElisoOperationalStatus(Enum): + OPERATIONAL = 'Operational' + NON_OPERATIONAL = 'Non-operational' + + +class ElisoAvailabilityStatus(Enum): + IN_USE = 'In use' + NOT_IN_USE = 'Not in use' + + +@validataclass +class ElisoConnectorInput: + maxPower: int = IntegerValidator(min_value=0) # in kW + powerType: PowerType = EnumValidator(PowerType) + type_of_connector: ElisoConnectorType = EnumValidator(ElisoConnectorType) + + +@validataclass +class ElisoEvseInput: + evseId: str = StringValidator(min_length=1, max_length=48) + connectors: list[ElisoConnectorInput] = ListValidator(DataclassValidator(ElisoConnectorInput), min_length=1) + + +@validataclass +class ElisoCoordinatesInput: + latitude: Decimal = FloatToDecimalValidator(min_value=-90, max_value=90, allow_integers=True) + longitude: Decimal = FloatToDecimalValidator(min_value=-180, max_value=180, allow_integers=True) + + +@validataclass +class ElisoLocationInput: + address: str = StringValidator(min_length=1) + postalCode: str = StringValidator(min_length=1) + city: str = StringValidator(min_length=1) + country_iso_3166_alpha_2: str = StringValidator(min_length=2, max_length=2) + coordinates: ElisoCoordinatesInput = DataclassValidator(ElisoCoordinatesInput) + operator_name: str = StringValidator(min_length=1) + hotline_number: str | None = StringValidator(), Default(None) + opening_time: str | None = StringValidator(), Default(None) + mobilithek_last_updated_dts: datetime = DateTimeValidator() + evses: list[ElisoEvseInput] = ListValidator(DataclassValidator(ElisoEvseInput), min_length=1) + + def to_location_update(self, source_uid: str) -> LocationUpdate: + # eliso does not provide location ids, so we use the lowest EVSE id as stable identifier + uid = min(evse.evseId for evse in self.evses) + return LocationUpdate( + uid=uid, + source=source_uid, + time_zone='Europe/Berlin', + name=self.address, + address=self.address, + postal_code=self.postalCode, + city=self.city, + country=countries.get(alpha_2=self.country_iso_3166_alpha_2).alpha_3, + lat=self.coordinates.latitude, + lon=self.coordinates.longitude, + operator=BusinessUpdate(name=self.operator_name), + help_phone=self.hotline_number, + twentyfourseven=True if self.opening_time == '24/7' else None, + last_updated=self.mobilithek_last_updated_dts, + charging_pool=[ + ChargingStationUpdate( + uid=uid, + last_updated=self.mobilithek_last_updated_dts, + evses=[ + EvseUpdate( + uid=evse.evseId, + evse_id=evse.evseId, + status=EvseStatus.UNKNOWN, + last_updated=self.mobilithek_last_updated_dts, + connectors=[ + ConnectorUpdate( + uid=str(index), + standard=connector.type_of_connector.to_connector_type(), + format=connector.type_of_connector.to_connector_format(), + power_type=connector.powerType, + max_electric_power=connector.maxPower * 1000, + last_updated=self.mobilithek_last_updated_dts, + ) + for index, connector in enumerate(evse.connectors, start=1) + ], + ) + for evse in self.evses + ], + ), + ], + ) + + +@validataclass +class ElisoEvseStatusInput: + evseId: str = StringValidator(min_length=1, max_length=48) + operational_status: ElisoOperationalStatus = EnumValidator(ElisoOperationalStatus) + availability_status: ElisoAvailabilityStatus | None = EnumValidator(ElisoAvailabilityStatus), Default(None) + mobilithek_last_updated_dts: datetime = DateTimeValidator() + + def to_evse_realtime_update(self) -> EvseRealtimeUpdate: + if self.operational_status == ElisoOperationalStatus.NON_OPERATIONAL: + status = EvseStatus.INOPERATIVE + elif self.availability_status == ElisoAvailabilityStatus.IN_USE: + status = EvseStatus.CHARGING + elif self.availability_status == ElisoAvailabilityStatus.NOT_IN_USE: + status = EvseStatus.AVAILABLE + else: + status = EvseStatus.UNKNOWN + + return EvseRealtimeUpdate( + uid=self.evseId, + evse_id=self.evseId, + status=status, + last_updated=self.mobilithek_last_updated_dts, + status_last_updated=self.mobilithek_last_updated_dts, + ) diff --git a/webapp/services/import_services/import_services.py b/webapp/services/import_services/import_services.py index 43c58a92..d7694412 100644 --- a/webapp/services/import_services/import_services.py +++ b/webapp/services/import_services/import_services.py @@ -35,31 +35,49 @@ from webapp.services.import_services.base_import_service import BaseImportService from webapp.services.import_services.bnetza import BnetzaApiImportService, BnetzaExcelImportService from webapp.services.import_services.datex2 import ( + AhBeteiligungDatex2ImportService, AmpecoDatex2ImportService, + AudiDatex2ImportService, ChargecloudDatex2ImportService, + EaazeDatex2ImportService, EClearingNetDatex2ImportService, EcoMovementDatex2ImportService, EluMobilityDatex2ImportService, EnBWDatex2ImportService, + EnerandoDatex2ImportService, EnioDatex2ImportService, ERoundDatex2ImportService, + EulektroDatex2ImportService, + EvPriceDatex2ImportService, + EweDatex2ImportService, + FlaviaDatex2ImportService, GiroEDatex2ImportService, + GpJouleDatex2ImportService, GridAndCoDatex2ImportService, InergiaDatex2ImportService, + InselwerkeDatex2ImportService, + KulRudnickDatex2ImportService, LadebusinessDatex2ImportService, + LadesonneDatex2ImportService, LichtblickDatex2ImportService, MidorionDatex2ImportService, MontaDatex2ImportService, MsuDatex2ImportService, PumpDatex2ImportService, QwelloDatex2ImportService, + RMBeteiligungDatex2ImportService, + RoadDatex2ImportService, SmatricsDatex2ImportService, + SonnenstromSaarpfalzDatex2ImportService, TaubertDatex2ImportService, TeslaDatex2ImportService, + UlmDatex2ImportService, VaylensDatex2ImportService, + VirtaDatex2ImportService, VolkswagenDatex2ImportService, WirelaneDatex2ImportService, ) +from webapp.services.import_services.eliso import ElisoImportService from webapp.services.import_services.giroe import GiroeImportService from webapp.services.import_services.goldbeck_ipcm import HeilbronnNeckarbogenImportService from webapp.services.import_services.lichtblick import LichtblickImportService @@ -82,25 +100,38 @@ class ImportServices(BaseService): importer_by_uid: dict[str, BaseImportService] = {} importer_classes: list[type[BaseImportService]] = [ + AhBeteiligungDatex2ImportService, AlbwerkOchpImportService, AmpecoDatex2ImportService, + AudiDatex2ImportService, BnetzaApiImportService, BnetzaExcelImportService, ChargecloudDatex2ImportService, + EaazeDatex2ImportService, EaazePbwImportService, EClearingNetDatex2ImportService, EcoMovementDatex2ImportService, + ElisoImportService, EluMobilityDatex2ImportService, EnBWDatex2ImportService, + EnerandoDatex2ImportService, EnioDatex2ImportService, ERoundDatex2ImportService, + EulektroDatex2ImportService, + EvPriceDatex2ImportService, + EweDatex2ImportService, + FlaviaDatex2ImportService, GiroEDatex2ImportService, GiroeImportService, + GpJouleDatex2ImportService, GridAndCoDatex2ImportService, HeilbronnNeckarbogenImportService, InergiaDatex2ImportService, + InselwerkeDatex2ImportService, + KulRudnickDatex2ImportService, LadebusinessDatex2ImportService, LadenetzOchpImportService, + LadesonneDatex2ImportService, LichtblickDatex2ImportService, LichtblickImportService, LudwigsburgImportService, @@ -111,13 +142,18 @@ class ImportServices(BaseService): PforzheimImportService, PumpDatex2ImportService, QwelloDatex2ImportService, + RMBeteiligungDatex2ImportService, + RoadDatex2ImportService, SmatricsDatex2ImportService, + SonnenstromSaarpfalzDatex2ImportService, StadtnaviImportService, SWStuttgartImportService, TaubertDatex2ImportService, TeslaDatex2ImportService, TuebingenImportService, + UlmDatex2ImportService, VaylensDatex2ImportService, + VirtaDatex2ImportService, VolkswagenDatex2ImportService, WirelaneDatex2ImportService, ] diff --git a/webapp/shared/datex2/models.py b/webapp/shared/datex2/models.py index 15f7780c..4caaaa6d 100644 --- a/webapp/shared/datex2/models.py +++ b/webapp/shared/datex2/models.py @@ -21,7 +21,6 @@ from validataclass.dataclasses import Default, ValidataclassMixin, validataclass from validataclass.validators import ( - AnythingValidator, DataclassValidator, DateTimeValidator, EnumValidator, @@ -110,7 +109,7 @@ class DynamicInformationInput(ValidataclassMixin): exchangeStatus: ExchangeStatusEnumGInput = DataclassValidator(ExchangeStatusEnumGInput) messageGenerationTimestamp: datetime = DateTimeValidator() exDynamicInformationExtensionG: ExtensionTypeGInput | None = ( - Noneable(AnythingValidator(allowed_types=[ExtensionTypeGInput])), + Noneable(DataclassValidator(ExtensionTypeGInput)), Default(None), ) @@ -163,3 +162,30 @@ class MessageContainerWrapperInput(ValidataclassMixin): Noneable(DataclassValidator(ExchangeInformationInput)), Default(None), ) + + +@validataclass(reject_unknown_fields=True) +class MessageContainerV37RealtimeInput(ValidataclassMixin): + payload: list[PayloadPublicationGV37RealtimeInput] = ( + ListValidator(DataclassValidator(PayloadPublicationGV37RealtimeInput)), + Default([]), + ) + exchangeInformation: ExchangeInformationInput = DataclassValidator(ExchangeInformationInput) + conMessageContainerExtensionG: ExtensionTypeGInput | None = ( + Noneable(DataclassValidator(ExtensionTypeGInput)), + Default(None), + ) + + +@validataclass(reject_unknown_fields=True) +class MessageContainerWrapperV37RealtimeInput(ValidataclassMixin): + """ + DATEX II v3.7 realtime data as MessageContainer document, as used in the Mobilithek AFIR profile. Unlike + MessageContainerWrapperInput, unknown fields are rejected, so payloads outside the messageContainer are invalid. + """ + + messageContainer: MessageContainerV37RealtimeInput = DataclassValidator(MessageContainerV37RealtimeInput) + exchangeInformation: ExchangeInformationInput | None = ( + Noneable(DataclassValidator(ExchangeInformationInput)), + Default(None), + )