diff --git a/statvar_imports/eurostat/Population_on_1_January_by_age_group_and_sex/Population_on_1_January_by_age_group_and_sex_metadata.csv b/statvar_imports/eurostat/Population_on_1_January_by_age_group_and_sex/Population_on_1_January_by_age_group_and_sex_metadata.csv new file mode 100644 index 0000000000..de770117da --- /dev/null +++ b/statvar_imports/eurostat/Population_on_1_January_by_age_group_and_sex/Population_on_1_January_by_age_group_and_sex_metadata.csv @@ -0,0 +1,3 @@ +parameter,value +header_rows,1 +#input_rows,10000 diff --git a/statvar_imports/eurostat/Population_on_1_January_by_age_group_and_sex/Population_on_1_January_by_age_group_and_sex_pvmap.csv b/statvar_imports/eurostat/Population_on_1_January_by_age_group_and_sex/Population_on_1_January_by_age_group_and_sex_pvmap.csv new file mode 100644 index 0000000000..4c38345832 --- /dev/null +++ b/statvar_imports/eurostat/Population_on_1_January_by_age_group_and_sex/Population_on_1_January_by_age_group_and_sex_pvmap.csv @@ -0,0 +1,55 @@ +key,property1,value1,property2,value2,property3,value3 + +# Global properties from DATAFLOW +"DATAFLOW:ESTAT:DEMO_PJANGROUP(1.0)",populationType,dcs:Person,measuredProperty,dcs:count,statType,dcs:measuredValue + +# --- AUTOMATED CLEANUP: Ignored Columns --- +LAST UPDATE,#ignore,"",,,, +OBS_FLAG,#ignore,"",,,, +CONF_STATUS,#ignore,"",,,, + +# Frequency +freq:A,observationPeriod,P1Y,,,, + +# Unit +unit:NR,unit,dcs:Person,,,, + +# Sex +sex:T,gender,"",,,, +sex:M,gender,dcs:Male,,,, +sex:F,gender,dcs:Female,,,, + +# --- Broad Age Group Mappings --- (dcs:Years0To4 corrected to dcs:YearsUpto4) +age:TOTAL,age,"",,,, +age:UNK,age,dcs:USC_AgeNotStated,,, +age:Y_LT5,age,dcs:YearsUpto4,,,, +age:Y_GE75,age,dcs:Years75Onwards,,,, +age:Y_GE80,age,dcs:Years80Onwards,,,, +age:Y_GE85,age,dcs:Years85Onwards,,,, + +# Explicit Ranges (Replaces the Regex) +age:Y5-9,age,dcs:Years5To9,,,, +age:Y10-14,age,dcs:Years10To14,,,, +age:Y15-19,age,dcs:Years15To19,,,, +age:Y20-24,age,dcs:Years20To24,,,, +age:Y25-29,age,dcs:Years25To29,,,, +age:Y30-34,age,dcs:Years30To34,,,, +age:Y35-39,age,dcs:Years35To39,,,, +age:Y40-44,age,dcs:Years40To44,,,, +age:Y45-49,age,dcs:Years45To49,,,, +age:Y50-54,age,dcs:Years50To54,,,, +age:Y55-59,age,dcs:Years55To59,,,, +age:Y60-64,age,dcs:Years60To64,,,, +age:Y65-69,age,dcs:Years65To69,,,, +age:Y70-74,age,dcs:Years70To74,,,, +age:Y75-79,age,dcs:Years75To79,,,, +age:Y80-84,age,dcs:Years80To84,,,, + +# --- Geopolitical Entity Translation (Bypasses offline API call failures) --- +geo,observationAbout,{Data},,,, + +# Time +TIME_PERIOD,observationDate,{Data},,,, + +# Value +OBS_VALUE,value,{Number},,,, \ No newline at end of file diff --git a/statvar_imports/eurostat/Population_on_1_January_by_age_group_and_sex/README.md b/statvar_imports/eurostat/Population_on_1_January_by_age_group_and_sex/README.md new file mode 100644 index 0000000000..b83f9f89de --- /dev/null +++ b/statvar_imports/eurostat/Population_on_1_January_by_age_group_and_sex/README.md @@ -0,0 +1,51 @@ +# Eurostat Population By Age Group And Sex On January 1 Import + +## Overview + +This dataset contains annual population stock data broken down by age groups and sex at the national level, sourced from Eurostat. The data tracks demographic distributions on January 1st across various European countries to support long-term social, economic, and institutional planning. + +type of place: Country +years: Historical data to present (1960-2025) +place_resolution: Resolved to DCIDs (e.g., dcid:country/FRA, dcid:country/DEU) + +## Data Source +**Source URL:** +https://ec.europa.eu/eurostat/databrowser/view/demo_pjangroup/default/table + +**Provenance Description:** +This dataset is produced and harmonized by Eurostat using demographic data provided by national statistical institutes across European countries. It breaks down the annual population stock on January 1st by specific age groups and sex to support EU socioeconomic policy and long-term planning. + +### Script Details: +- **Download**: Uses `curl` to fetch the latest SDMX-CSV data from Eurostat's dissemination API. +- **Processing**: Uses `stat_var_processor.py` to map raw data to Data Commons StatVarObservations using the PV map and metadata configuration. + +## Processing +To process the Eurostat Population By Age Group And Sex On January 1 data and generate statistical variables, use the following commands from your current import data directory: + +# Download input file + +```bash +mkdir -p source_files +curl -L --retry 3 "https://ec.europa.eu/eurostat/api/dissemination/sdmx/2.1/data/DEMO_PJANGROUP/?format=SDMX-CSV&compressed=false" -o ./source_files/Population_on_1_January_by_age_group_and_sex_data_input.csv + +## For Test Data Run + +python3 tools/statvar_importer/stat_var_processor.py \ + "--input_data=./testdata/Population_on_1_January_by_age_group_and_sex_data_input.csv"\ + "--pv_map=./Population_on_1_January_by_age_group_and_sex_pvmap.csv" \ + "--output_path=./testdata/Population_on_1_January_by_age_group_and_sex_output" \ + "--config_file=./Population_on_1_January_by_age_group_and_sex_metadata.csv" \ + "--existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf" + +## For Main data run + +python3 ../../../tools/statvar_importer/stat_var_processor.py \ + "--input_data=./source_files/*.csv" \ + "--pv_map=./Population_on_1_January_by_age_group_and_sex_pvmap.csv" \ + "--config_file=./Population_on_1_January_by_age_group_and_sex_metadata.csv" \ + "--generate_statvar_name=True" \ + "--skip_constant_csv_columns=False" \ + "--output_columns=observationDate,observationAbout,variableMeasured,value,observationPeriod,unit" \ + "--output_path=./Population_on_1_January_by_age_group_and_sex_output" \ + "--places_resolved_csv=./places_resolved_runtime.csv" \ + "--existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf" \ No newline at end of file diff --git a/statvar_imports/eurostat/Population_on_1_January_by_age_group_and_sex/manifest.json b/statvar_imports/eurostat/Population_on_1_January_by_age_group_and_sex/manifest.json new file mode 100644 index 0000000000..41f6c702a3 --- /dev/null +++ b/statvar_imports/eurostat/Population_on_1_January_by_age_group_and_sex/manifest.json @@ -0,0 +1,34 @@ +{ + "import_specifications": [ + { + "import_name": "EuroStat_Population_On_1_January_By_Age_Group_And_Sex", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://ec.europa.eu/eurostat/databrowser/view/demo_pjangroup/default/table", + "provenance_description": "Annual European population stock on January 1st by age group and sex, harmonized by Eurostat.", + "scripts": [ + "run.sh" + ], + "import_inputs": [ + { + "template_mcf": "Population_On_1_January_By_Age_Group_And_Sex_output.tmcf", + "cleaned_csv": "Population_On_1_January_By_Age_Group_And_Sex_output.csv" + } + ], + "source_files": [ + "source_files/*.csv" + ], + "cron_schedule": "5 1 1,15 * *", + "resource_limits": {"cpu": 4, "memory": 8, "disk":100}, + "config_override": { + "invoke_import_validation": true, + "invoke_import_tool": true, + "invoke_differ_tool": true, + "skip_input_upload": false, + "skip_gcs_upload": false, + "cleanup_gcs_volume_mount": false + } + } + ] +} \ No newline at end of file diff --git a/statvar_imports/eurostat/Population_on_1_January_by_age_group_and_sex/places_resolved_runtime.csv b/statvar_imports/eurostat/Population_on_1_January_by_age_group_and_sex/places_resolved_runtime.csv new file mode 100644 index 0000000000..270ff41a19 --- /dev/null +++ b/statvar_imports/eurostat/Population_on_1_January_by_age_group_and_sex/places_resolved_runtime.csv @@ -0,0 +1,264 @@ +place_name,dcid +AD,dcid:country/AND +AE,dcid:country/ARE +AF,dcid:country/AFG +AG,dcid:country/ATG +AI,dcid:country/AIA +AL,dcid:country/ALB +AM,dcid:country/ARM +AN,dcid:country/ANT +AO,dcid:country/AGO +AQ,dcid:country/ATA +AR,dcid:country/ARG +AS,dcid:country/ASM +AT,dcid:country/AUT +AU,dcid:country/AUS +AW,dcid:country/ABW +AZ,dcid:country/AZE +BA,dcid:country/BIH +BB,dcid:country/BRB +BD,dcid:country/BGD +BE,dcid:country/BEL +BF,dcid:country/BFA +BG,dcid:country/BGR +BH,dcid:country/BHR +BI,dcid:country/BDI +BJ,dcid:country/BEN +BL,dcid:country/BLM +BM,dcid:country/BMU +BN,dcid:country/BRN +BO,dcid:country/BOL +BR,dcid:country/BRA +BS,dcid:country/BHS +BT,dcid:country/BTN +BV,dcid:country/BVT +BW,dcid:country/BWA +BY,dcid:country/BLR +BZ,dcid:country/BLZ +CA,dcid:country/CAN +CC,dcid:country/CCK +CD,dcid:country/COD +CF,dcid:country/CAF +CG,dcid:country/COG +CH,dcid:country/CHE +CI,dcid:country/CIV +CK,dcid:country/COK +CL,dcid:country/CHL +CM,dcid:country/CMR +CN,dcid:country/CHN +CO,dcid:country/COL +CR,dcid:country/CRI +CU,dcid:country/CUB +CV,dcid:country/CPV +CX,dcid:country/CXR +CY,dcid:country/CYP +CZ,dcid:country/CZE +DE,dcid:country/DEU +DJ,dcid:country/DJI +DK,dcid:country/DNK +DM,dcid:country/DMA +DO,dcid:country/DOM +DZ,dcid:country/DZA +EC,dcid:country/ECU +EE,dcid:country/EST +EG,dcid:country/EGY +EH,dcid:country/ESH +EL,dcid:country/GRC +ER,dcid:country/ERI +ES,dcid:country/ESP +ET,dcid:country/ETH +FI,dcid:country/FIN +FJ,dcid:country/FJI +FK,dcid:country/FLK +FM,dcid:country/FSM +FO,dcid:country/FRO +FR,dcid:country/FRA +FX,dcid:country/FXX +GA,dcid:country/GAB +UK,dcid:country/GBR +GD,dcid:country/GRD +GE,dcid:country/GEO +GF,dcid:country/GUF +GG,dcid:country/GGY +GH,dcid:country/GHA +GI,dcid:country/GIB +GL,dcid:country/GRL +GM,dcid:country/GMB +GN,dcid:country/GIN +GQ,dcid:country/GNQ +GS,dcid:country/SGS +GT,dcid:country/GTM +GU,dcid:country/GUM +GW,dcid:country/GNB +GY,dcid:country/GUY +HK,dcid:country/HKG +HM,dcid:country/HMD +HN,dcid:country/HND +HR,dcid:country/HRV +HT,dcid:country/HTI +HU,dcid:country/HUN +ID,dcid:country/IDN +IE,dcid:country/IRL +IL,dcid:country/ISR +IM,dcid:country/IMN +IN,dcid:country/IND +IO,dcid:country/IOT +IQ,dcid:country/IRQ +IR,dcid:country/IRN +IS,dcid:country/ISL +IT,dcid:country/ITA +JE,dcid:country/JEY +JM,dcid:country/JAM +JO,dcid:country/JOR +JP,dcid:country/JPN +KE,dcid:country/KEN +KG,dcid:country/KGZ +KH,dcid:country/KHM +KI,dcid:country/KIR +KM,dcid:country/COM +KN,dcid:country/KNA +KP,dcid:country/PRK +KR,dcid:country/KOR +KW,dcid:country/KWT +KY,dcid:country/CYM +KZ,dcid:country/KAZ +LA,dcid:country/LAO +LB,dcid:country/LBN +LC,dcid:country/LCA +LI,dcid:country/LIE +LK,dcid:country/LKA +LR,dcid:country/LBR +LS,dcid:country/LSO +LT,dcid:country/LTU +LU,dcid:country/LUX +LV,dcid:country/LVA +LY,dcid:country/LBY +MA,dcid:country/MAR +MC,dcid:country/MCO +MD,dcid:country/MDA +ME,dcid:country/MNE +MF,dcid:country/MAF +MG,dcid:country/MDG +MH,dcid:country/MHL +MK,dcid:country/MKD +ML,dcid:country/MLI +MM,dcid:country/MMR +MN,dcid:country/MNG +MO,dcid:country/MAC +MP,dcid:country/MNP +MQ,dcid:country/MTQ +MR,dcid:country/MRT +MS,dcid:country/MSR +MT,dcid:country/MLT +MU,dcid:country/MUS +MV,dcid:country/MDV +MW,dcid:country/MWI +MX,dcid:country/MEX +MY,dcid:country/MYS +MZ,dcid:country/MOZ +NA,dcid:country/NAM +NC,dcid:country/NCL +NE,dcid:country/NER +NF,dcid:country/NFK +NG,dcid:country/NGA +NI,dcid:country/NIC +NL,dcid:country/NLD +NO,dcid:country/NOR +NP,dcid:country/NPL +NR,dcid:country/NRU +NU,dcid:country/NIU +NZ,dcid:country/NZL +OM,dcid:country/OMN +PA,dcid:country/PAN +PE,dcid:country/PER +PF,dcid:country/PYF +PG,dcid:country/PNG +PH,dcid:country/PHL +PK,dcid:country/PAK +PL,dcid:country/POL +PM,dcid:country/SPM +PN,dcid:country/PCN +PR,dcid:country/PRI +PS,dcid:country/PSE +PT,dcid:country/PRT +PW,dcid:country/PLW +PY,dcid:country/PRY +QA,dcid:country/QAT +RE,dcid:country/REU +RO,dcid:country/ROU +RS,dcid:country/SRB +RU,dcid:country/RUS +RW,dcid:country/RWA +SA,dcid:country/SAU +SB,dcid:country/SLB +SC,dcid:country/SYC +SD,dcid:country/SDN +SE,dcid:country/SWE +SG,dcid:country/SGP +SH,dcid:country/SHN +SI,dcid:country/SVN +SJ,dcid:country/SJM +SK,dcid:country/SVK +SL,dcid:country/SLE +SM,dcid:country/SMR +SN,dcid:country/SEN +SO,dcid:country/SOM +SR,dcid:country/SUR +ST,dcid:country/STP +SV,dcid:country/SLV +SX,dcid:country/SXM +SY,dcid:country/SYR +SZ,dcid:country/SWZ +TC,dcid:country/TCA +TD,dcid:country/TCD +TF,dcid:country/ATF +TG,dcid:country/TGO +TH,dcid:country/THA +TJ,dcid:country/TJK +TK,dcid:country/TKL +TL,dcid:country/TLS +TM,dcid:country/TKM +TN,dcid:country/TUN +TO,dcid:country/TON +TR,dcid:country/TUR +TT,dcid:country/TTO +TV,dcid:country/TUV +TW,dcid:country/TWN +TZ,dcid:country/TZA +UA,dcid:country/UKR +UG,dcid:country/UGA +UM,dcid:country/UMI +US,dcid:country/USA +UY,dcid:country/URY +UZ,dcid:country/UZB +VA,dcid:country/VAT +VC,dcid:country/VCT +VE,dcid:country/VEN +VG,dcid:country/VGB +VI,dcid:country/VIR +VN,dcid:country/VNM +VU,dcid:country/VUT +WF,dcid:country/WLF +WS,dcid:country/WSM +XK,dcid:country/XKS +YE,dcid:country/YEM +YT,dcid:country/MYT +ZA,dcid:country/ZAF +ZM,dcid:country/ZMB +ZW,dcid:country/ZWE +DE_TOT,dcid:nuts/DE_TOT +EA,dcid:nuts/EA +EA12,dcid:nuts/EA12 +EA18,dcid:nuts/EA18 +EA19,dcid:nuts/EA19 +EEA,dcid:nuts/EEA +EEA30_2007,dcid:nuts/EEA30_2007 +EEA31,dcid:nuts/EEA31 +EFTA,dcid:nuts/EFTA +EU,dcid:nuts/EU +EU27,dcid:nuts/EU27 +EU27_2007,dcid:nuts/EU27_2007 +EU27_2020,dcid:nuts/EU27_2020 +EA20,dcid:nuts/EA20 +EA21,dcid:nuts/EA21 +EU28,dcid:nuts/EU28 diff --git a/statvar_imports/eurostat/Population_on_1_January_by_age_group_and_sex/run.sh b/statvar_imports/eurostat/Population_on_1_January_by_age_group_and_sex/run.sh new file mode 100644 index 0000000000..082c5f1001 --- /dev/null +++ b/statvar_imports/eurostat/Population_on_1_January_by_age_group_and_sex/run.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# Step 1: Data Download +mkdir -p source_files +curl -L --retry 3 "https://ec.europa.eu/eurostat/api/dissemination/sdmx/2.1/data/DEMO_PJANGROUP/?format=SDMX-CSV&compressed=false" -o ./source_files/Population_on_1_January_by_age_group_and_sex_data_input.csv + +# Step 2: Data Processing +python3 ../../../tools/statvar_importer/stat_var_processor.py \ + "--input_data=./source_files/*.csv" \ + "--pv_map=./Population_on_1_January_by_age_group_and_sex_pvmap.csv" \ + "--config_file=./Population_on_1_January_by_age_group_and_sex_metadata.csv" \ + "--generate_statvar_name=True" \ + "--skip_constant_csv_columns=False" \ + "--output_columns=observationDate,observationAbout,variableMeasured,value,observationPeriod,unit" \ + "--output_path=./Population_on_1_January_by_age_group_and_sex_output" \ + "--places_resolved_csv=./places_resolved_runtime.csv" \ + "--existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf" diff --git a/statvar_imports/eurostat/Population_on_1_January_by_age_group_and_sex/test_data/Population_on_1_January_by_age_group_and_sex_data_input.csv b/statvar_imports/eurostat/Population_on_1_January_by_age_group_and_sex/test_data/Population_on_1_January_by_age_group_and_sex_data_input.csv new file mode 100644 index 0000000000..b8ffb59559 --- /dev/null +++ b/statvar_imports/eurostat/Population_on_1_January_by_age_group_and_sex/test_data/Population_on_1_January_by_age_group_and_sex_data_input.csv @@ -0,0 +1,100 @@ +DATAFLOW,LAST UPDATE,freq,unit,sex,age,geo,TIME_PERIOD,OBS_VALUE,OBS_FLAG,CONF_STATUS +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AD,1986,20898,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AD,1987,22148,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AD,1988,22814,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AD,1989,23813,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AD,1990,24171,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AD,1991,25542,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AD,1992,27572,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AD,1993,28864,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AD,1994,30406,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AD,1995,30228,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AD,1996,30200,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AD,1997,30627,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AD,1998,31025,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AD,1999,31384,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AD,2000,31538,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AD,2001,31576,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AD,2002,31915,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AD,2003,32354,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AD,2004,34738,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AD,2005,36790.0,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AD,2006,37575.0,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AD,2007,38705.0,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AD,2008,39622.0,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AD,2009,40303.0,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AD,2010,40296.0,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AD,2011,38252,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AD,2012,38252,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AD,2013,37408,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AD,2019,37388,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AL,2000,1526762,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AL,2001,1535822,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AL,2002,1532563,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AL,2003,1526180,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AL,2004,1520481,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AL,2005,1512745.0,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AL,2006,1503969.0,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AL,2007,1492439.0,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AL,2008,1480358.0,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AL,2009,1468935.0,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AL,2010,1459025.0,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AL,2011,1451691,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AL,2012,1444234,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AL,2013,1437193,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AL,2014,1430827,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AL,2015,1424597,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AL,2016,1417141,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AL,2017,1423050,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AL,2018,1431715,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AL,2019,1432833,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AL,2020,1425342,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AL,2021,1419759,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AL,2023,1394864,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AL,2025,1194597,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AM,2000,1681200,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AM,2001,1672200,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AM,2002,1670500,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AM,2003,1666700,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AM,2004,1665500,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AM,2005,1665200.0,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AM,2006,1665123.0,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AM,2007,1665554.0,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AM,2008,1667529.0,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AM,2009,1669711.0,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AM,2010,1673656.0,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AM,2011,1679066,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AM,2012,1684000,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AM,2015,1571450,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AM,2016,1569535,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AM,2017,1567380,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AM,2018,1564533,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AM,2019,1563538,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AM,2020,1562689,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AM,2021,1565144,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AM,2024,1577675,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AT,1960,3757167,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AT,1961,3773097,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AT,1962,3794130,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AT,1963,3814191,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AT,1964,3836415,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AT,1965,3857760,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AT,1966,3876559,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AT,1967,3899799,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AT,1968,3922359,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AT,1969,3932691,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AT,1970,3944298,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AT,1971,3953827,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AT,1972,3972070,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AT,1973,3990665,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AT,1974,4006377,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AT,1975,4003303,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AT,1976,3992886,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AT,1977,3992437,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AT,1978,3994255,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AT,1979,3985363,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AT,1980,3981359,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AT,1981,3983154,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AT,1982,3993808,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AT,1983,3981596,, +ESTAT:DEMO_PJANGROUP(1.0),08/05/26 11:00:00,A,NR,F,TOTAL,AT,1984,3976213,, \ No newline at end of file diff --git a/statvar_imports/eurostat/Population_on_1_January_by_age_group_and_sex/test_data/Population_on_1_January_by_age_group_and_sex_output.csv b/statvar_imports/eurostat/Population_on_1_January_by_age_group_and_sex/test_data/Population_on_1_January_by_age_group_and_sex_output.csv new file mode 100644 index 0000000000..1f3450141d --- /dev/null +++ b/statvar_imports/eurostat/Population_on_1_January_by_age_group_and_sex/test_data/Population_on_1_January_by_age_group_and_sex_output.csv @@ -0,0 +1,100 @@ +observationDate,observationAbout,variableMeasured,value,observationPeriod,unit +1986,dcid:country/AND,dcid:Count_Person_Female,20898,P1Y,dcs:Person +1987,dcid:country/AND,dcid:Count_Person_Female,22148,P1Y,dcs:Person +1988,dcid:country/AND,dcid:Count_Person_Female,22814,P1Y,dcs:Person +1989,dcid:country/AND,dcid:Count_Person_Female,23813,P1Y,dcs:Person +1990,dcid:country/AND,dcid:Count_Person_Female,24171,P1Y,dcs:Person +1991,dcid:country/AND,dcid:Count_Person_Female,25542,P1Y,dcs:Person +1992,dcid:country/AND,dcid:Count_Person_Female,27572,P1Y,dcs:Person +1993,dcid:country/AND,dcid:Count_Person_Female,28864,P1Y,dcs:Person +1994,dcid:country/AND,dcid:Count_Person_Female,30406,P1Y,dcs:Person +1995,dcid:country/AND,dcid:Count_Person_Female,30228,P1Y,dcs:Person +1996,dcid:country/AND,dcid:Count_Person_Female,30200,P1Y,dcs:Person +1997,dcid:country/AND,dcid:Count_Person_Female,30627,P1Y,dcs:Person +1998,dcid:country/AND,dcid:Count_Person_Female,31025,P1Y,dcs:Person +1999,dcid:country/AND,dcid:Count_Person_Female,31384,P1Y,dcs:Person +2000,dcid:country/AND,dcid:Count_Person_Female,31538,P1Y,dcs:Person +2001,dcid:country/AND,dcid:Count_Person_Female,31576,P1Y,dcs:Person +2002,dcid:country/AND,dcid:Count_Person_Female,31915,P1Y,dcs:Person +2003,dcid:country/AND,dcid:Count_Person_Female,32354,P1Y,dcs:Person +2004,dcid:country/AND,dcid:Count_Person_Female,34738,P1Y,dcs:Person +2005,dcid:country/AND,dcid:Count_Person_Female,36790,P1Y,dcs:Person +2006,dcid:country/AND,dcid:Count_Person_Female,37575,P1Y,dcs:Person +2007,dcid:country/AND,dcid:Count_Person_Female,38705,P1Y,dcs:Person +2008,dcid:country/AND,dcid:Count_Person_Female,39622,P1Y,dcs:Person +2009,dcid:country/AND,dcid:Count_Person_Female,40303,P1Y,dcs:Person +2010,dcid:country/AND,dcid:Count_Person_Female,40296,P1Y,dcs:Person +2011,dcid:country/AND,dcid:Count_Person_Female,38252,P1Y,dcs:Person +2012,dcid:country/AND,dcid:Count_Person_Female,38252,P1Y,dcs:Person +2013,dcid:country/AND,dcid:Count_Person_Female,37408,P1Y,dcs:Person +2019,dcid:country/AND,dcid:Count_Person_Female,37388,P1Y,dcs:Person +2000,dcid:country/ALB,dcid:Count_Person_Female,1526762,P1Y,dcs:Person +2001,dcid:country/ALB,dcid:Count_Person_Female,1535822,P1Y,dcs:Person +2002,dcid:country/ALB,dcid:Count_Person_Female,1532563,P1Y,dcs:Person +2003,dcid:country/ALB,dcid:Count_Person_Female,1526180,P1Y,dcs:Person +2004,dcid:country/ALB,dcid:Count_Person_Female,1520481,P1Y,dcs:Person +2005,dcid:country/ALB,dcid:Count_Person_Female,1512745,P1Y,dcs:Person +2006,dcid:country/ALB,dcid:Count_Person_Female,1503969,P1Y,dcs:Person +2007,dcid:country/ALB,dcid:Count_Person_Female,1492439,P1Y,dcs:Person +2008,dcid:country/ALB,dcid:Count_Person_Female,1480358,P1Y,dcs:Person +2009,dcid:country/ALB,dcid:Count_Person_Female,1468935,P1Y,dcs:Person +2010,dcid:country/ALB,dcid:Count_Person_Female,1459025,P1Y,dcs:Person +2011,dcid:country/ALB,dcid:Count_Person_Female,1451691,P1Y,dcs:Person +2012,dcid:country/ALB,dcid:Count_Person_Female,1444234,P1Y,dcs:Person +2013,dcid:country/ALB,dcid:Count_Person_Female,1437193,P1Y,dcs:Person +2014,dcid:country/ALB,dcid:Count_Person_Female,1430827,P1Y,dcs:Person +2015,dcid:country/ALB,dcid:Count_Person_Female,1424597,P1Y,dcs:Person +2016,dcid:country/ALB,dcid:Count_Person_Female,1417141,P1Y,dcs:Person +2017,dcid:country/ALB,dcid:Count_Person_Female,1423050,P1Y,dcs:Person +2018,dcid:country/ALB,dcid:Count_Person_Female,1431715,P1Y,dcs:Person +2019,dcid:country/ALB,dcid:Count_Person_Female,1432833,P1Y,dcs:Person +2020,dcid:country/ALB,dcid:Count_Person_Female,1425342,P1Y,dcs:Person +2021,dcid:country/ALB,dcid:Count_Person_Female,1419759,P1Y,dcs:Person +2023,dcid:country/ALB,dcid:Count_Person_Female,1394864,P1Y,dcs:Person +2025,dcid:country/ALB,dcid:Count_Person_Female,1194597,P1Y,dcs:Person +2000,dcid:country/ARM,dcid:Count_Person_Female,1681200,P1Y,dcs:Person +2001,dcid:country/ARM,dcid:Count_Person_Female,1672200,P1Y,dcs:Person +2002,dcid:country/ARM,dcid:Count_Person_Female,1670500,P1Y,dcs:Person +2003,dcid:country/ARM,dcid:Count_Person_Female,1666700,P1Y,dcs:Person +2004,dcid:country/ARM,dcid:Count_Person_Female,1665500,P1Y,dcs:Person +2005,dcid:country/ARM,dcid:Count_Person_Female,1665200,P1Y,dcs:Person +2006,dcid:country/ARM,dcid:Count_Person_Female,1665123,P1Y,dcs:Person +2007,dcid:country/ARM,dcid:Count_Person_Female,1665554,P1Y,dcs:Person +2008,dcid:country/ARM,dcid:Count_Person_Female,1667529,P1Y,dcs:Person +2009,dcid:country/ARM,dcid:Count_Person_Female,1669711,P1Y,dcs:Person +2010,dcid:country/ARM,dcid:Count_Person_Female,1673656,P1Y,dcs:Person +2011,dcid:country/ARM,dcid:Count_Person_Female,1679066,P1Y,dcs:Person +2012,dcid:country/ARM,dcid:Count_Person_Female,1684000,P1Y,dcs:Person +2015,dcid:country/ARM,dcid:Count_Person_Female,1571450,P1Y,dcs:Person +2016,dcid:country/ARM,dcid:Count_Person_Female,1569535,P1Y,dcs:Person +2017,dcid:country/ARM,dcid:Count_Person_Female,1567380,P1Y,dcs:Person +2018,dcid:country/ARM,dcid:Count_Person_Female,1564533,P1Y,dcs:Person +2019,dcid:country/ARM,dcid:Count_Person_Female,1563538,P1Y,dcs:Person +2020,dcid:country/ARM,dcid:Count_Person_Female,1562689,P1Y,dcs:Person +2021,dcid:country/ARM,dcid:Count_Person_Female,1565144,P1Y,dcs:Person +2024,dcid:country/ARM,dcid:Count_Person_Female,1577675,P1Y,dcs:Person +1960,dcid:country/AUT,dcid:Count_Person_Female,3757167,P1Y,dcs:Person +1961,dcid:country/AUT,dcid:Count_Person_Female,3773097,P1Y,dcs:Person +1962,dcid:country/AUT,dcid:Count_Person_Female,3794130,P1Y,dcs:Person +1963,dcid:country/AUT,dcid:Count_Person_Female,3814191,P1Y,dcs:Person +1964,dcid:country/AUT,dcid:Count_Person_Female,3836415,P1Y,dcs:Person +1965,dcid:country/AUT,dcid:Count_Person_Female,3857760,P1Y,dcs:Person +1966,dcid:country/AUT,dcid:Count_Person_Female,3876559,P1Y,dcs:Person +1967,dcid:country/AUT,dcid:Count_Person_Female,3899799,P1Y,dcs:Person +1968,dcid:country/AUT,dcid:Count_Person_Female,3922359,P1Y,dcs:Person +1969,dcid:country/AUT,dcid:Count_Person_Female,3932691,P1Y,dcs:Person +1970,dcid:country/AUT,dcid:Count_Person_Female,3944298,P1Y,dcs:Person +1971,dcid:country/AUT,dcid:Count_Person_Female,3953827,P1Y,dcs:Person +1972,dcid:country/AUT,dcid:Count_Person_Female,3972070,P1Y,dcs:Person +1973,dcid:country/AUT,dcid:Count_Person_Female,3990665,P1Y,dcs:Person +1974,dcid:country/AUT,dcid:Count_Person_Female,4006377,P1Y,dcs:Person +1975,dcid:country/AUT,dcid:Count_Person_Female,4003303,P1Y,dcs:Person +1976,dcid:country/AUT,dcid:Count_Person_Female,3992886,P1Y,dcs:Person +1977,dcid:country/AUT,dcid:Count_Person_Female,3992437,P1Y,dcs:Person +1978,dcid:country/AUT,dcid:Count_Person_Female,3994255,P1Y,dcs:Person +1979,dcid:country/AUT,dcid:Count_Person_Female,3985363,P1Y,dcs:Person +1980,dcid:country/AUT,dcid:Count_Person_Female,3981359,P1Y,dcs:Person +1981,dcid:country/AUT,dcid:Count_Person_Female,3983154,P1Y,dcs:Person +1982,dcid:country/AUT,dcid:Count_Person_Female,3993808,P1Y,dcs:Person +1983,dcid:country/AUT,dcid:Count_Person_Female,3981596,P1Y,dcs:Person +1984,dcid:country/AUT,dcid:Count_Person_Female,3976213,P1Y,dcs:Person \ No newline at end of file diff --git a/statvar_imports/eurostat/Population_on_1_January_by_age_group_and_sex/test_data/Population_on_1_January_by_age_group_and_sex_output.tmcf b/statvar_imports/eurostat/Population_on_1_January_by_age_group_and_sex/test_data/Population_on_1_January_by_age_group_and_sex_output.tmcf new file mode 100644 index 0000000000..39a28d1ba4 --- /dev/null +++ b/statvar_imports/eurostat/Population_on_1_January_by_age_group_and_sex/test_data/Population_on_1_January_by_age_group_and_sex_output.tmcf @@ -0,0 +1,8 @@ +Node: E:output->E0 +observationDate: C:output->observationDate +observationAbout: C:output->observationAbout +variableMeasured: C:output->variableMeasured +value: C:output->value +observationPeriod: C:output->observationPeriod +unit: C:output->unit +typeOf: dcs:StatVarObservation diff --git a/statvar_imports/eurostat/Usually_resident_population_on_1_January/README.md b/statvar_imports/eurostat/Usually_resident_population_on_1_January/README.md new file mode 100644 index 0000000000..608592e54f --- /dev/null +++ b/statvar_imports/eurostat/Usually_resident_population_on_1_January/README.md @@ -0,0 +1,47 @@ +# Eurostat Usually Resident Population On January 1 Import + +## Overview + +This dataset contains annual data reporting the usually resident population on January 1st at the national level, sourced from Eurostat. The data tracks the official total population counts across various European countries to support EU legislative, budgeting, and institutional voting procedures. + +type of place: Country +years: Historical data to present (2014-2025) +place_resolution: Resolved to DCIDs (e.g., dcid:country/FRA, dcid:country/DEU) + +## Data Source +**Source URL:** +https://ec.europa.eu/eurostat/databrowser/product/view/demo_urespop?lang=en + +**Provenance Description:** +This dataset is produced by **Eurostat** using harmonized population data provided by European national statistical institutes. It tracks the usually resident population across European countries on January 1st to support official EU policy, budgeting, and voting procedures. + +## Processing +To process the Eurostat Usually Resident Population On January 1 data and generate statistical variables, use the following commands from your current import data directory: + +# Download input file + +```bash +mkdir -p input_files +curl -L --retry 3 "https://ec.europa.eu/eurostat/api/dissemination/sdmx/2.1/data/DEMO_URESPOP/?format=SDMX-CSV&compressed=false" -o ./source_files/Usually_resident_population_on_1_January_input.csv + +## For Test Data Run + +python3 tools/statvar_importer/stat_var_processor.py \ + "--input_data=./testdata/Usually_resident_population_on_1_January_input.csv" \ + "--pv_map=./Usually_resident_population_on_1_January_pvmap.csv" \ + "--output_path=./testdata/Usually_resident_population_on_1_January_output" \ + "--config_file=./Usually_resident_population_on_1_January_metadata.csv" \ + "--existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf" + +## For Main data run + +python3 ../../../tools/statvar_importer/stat_var_processor.py \ + "--input_data=./source_files/*.csv" \ + "--pv_map=./Usually_resident_population_on_1_January_pvmap.csv" \ + "--config_file=./Usually_resident_population_on_1_January_metadata.csv" \ + "--generate_statvar_name=True" \ + "--skip_constant_csv_columns=False" \ + "--output_columns=observationDate,observationAbout,variableMeasured,value,observationPeriod,unit" \ + "--output_path=./Usually_resident_population_on_1_January_output" \ + "--places_resolved_csv=./places_resolved_runtime.csv" \ + "--existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf" \ No newline at end of file diff --git a/statvar_imports/eurostat/Usually_resident_population_on_1_January/Usually_resident_population_on_1_January_metadata.csv b/statvar_imports/eurostat/Usually_resident_population_on_1_January/Usually_resident_population_on_1_January_metadata.csv new file mode 100644 index 0000000000..09873ca101 --- /dev/null +++ b/statvar_imports/eurostat/Usually_resident_population_on_1_January/Usually_resident_population_on_1_January_metadata.csv @@ -0,0 +1,4 @@ +parameter,value +header_rows,1 +output_only_new_statvars,False +resolve_places,True diff --git a/statvar_imports/eurostat/Usually_resident_population_on_1_January/Usually_resident_population_on_1_January_pvmap.csv b/statvar_imports/eurostat/Usually_resident_population_on_1_January/Usually_resident_population_on_1_January_pvmap.csv new file mode 100644 index 0000000000..9528b2146a --- /dev/null +++ b/statvar_imports/eurostat/Usually_resident_population_on_1_January/Usually_resident_population_on_1_January_pvmap.csv @@ -0,0 +1,38 @@ +key,property1,value1,property2,value2,property3,value3,property4,value4 + +# Global Dataflow properties (Standard Data Commons Demographic Modeling) +"DATAFLOW:ESTAT:DEMO_URESPOP(1.0)",populationType,dcs:Person,measuredProperty,dcs:count,statType,dcs:measuredValue,, + +# --- AUTOMATED CLEANUP: Ignored Columns --- +LAST UPDATE,#ignore,"",,,,,, + +# Frequency dimension (P1Y is standard for annual resident population) +freq:A,observationPeriod,P1Y,,,,,, +freq:Q,observationPeriod,P3M,,,,,, +freq:M,observationPeriod,P1M,,,,,, +freq:W,observationPeriod,P1W,,,,,, +freq:D,observationPeriod,P1D,,,,,, + +# Unit dimension (dcs:Person is standard for demographic counts) +unit:NR,unit,dcs:Person,,,,,, + +# --- Geopolitical Entity Translation (Bypasses offline API call failures) --- +geo,observationAbout,{Data},,,,,, + +# Time and Value +TIME_PERIOD,observationDate,{Data},,,,,, +OBS_VALUE,value,{Number},,,,,, + +# Safely Ignoring Transient, Composite, and Confidentiality Flags to Prevent Ingestion Warnings +OBS_FLAG,#ignore,"",,,,,, +OBS_FLAG:e,#ignore,"",,,,,, +OBS_FLAG:b,#ignore,"",,,,,, +OBS_FLAG:p,#ignore,"",,,,,, +OBS_FLAG:be,#ignore,"",,,,,, +OBS_FLAG:M,MissingValue,"",,,,,, +OBS_FLAG:m,MissingValue,"",,,,,, + +CONF_STATUS,#ignore,"",,,,,, +CONF_STATUS:C,#ignore,"",,,,,, +CONF_STATUS:N,#ignore,"",,,,,, +CONF_STATUS:P,#ignore,"",,,,,, \ No newline at end of file diff --git a/statvar_imports/eurostat/Usually_resident_population_on_1_January/manifest.json b/statvar_imports/eurostat/Usually_resident_population_on_1_January/manifest.json new file mode 100644 index 0000000000..6447cc83f2 --- /dev/null +++ b/statvar_imports/eurostat/Usually_resident_population_on_1_January/manifest.json @@ -0,0 +1,34 @@ +{ + "import_specifications": [ + { + "import_name": "EuroStat_Usually_resident_population_on_1_January", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://ec.europa.eu/eurostat/databrowser/product/view/demo_urespop?lang=en", + "provenance_description": "Annual tracking of the usually resident population on January 1st across reporting European countries", + "scripts": [ + "run.sh" + ], + "import_inputs": [ + { + "template_mcf": "Usually_resident_population_on_1_January_output.tmcf", + "cleaned_csv": "Usually_resident_population_on_1_January_output.csv" + } + ], + "source_files": [ + "source_files/*.csv" + ], + "cron_schedule": "5 1 1,15 * *", + "resource_limits": {"cpu": 4, "memory": 8, "disk":100}, + "config_override": { + "invoke_import_validation": true, + "invoke_import_tool": true, + "invoke_differ_tool": true, + "skip_input_upload": false, + "skip_gcs_upload": false, + "cleanup_gcs_volume_mount": false + } + } + ] +} \ No newline at end of file diff --git a/statvar_imports/eurostat/Usually_resident_population_on_1_January/places_resolved_runtime.csv b/statvar_imports/eurostat/Usually_resident_population_on_1_January/places_resolved_runtime.csv new file mode 100644 index 0000000000..270ff41a19 --- /dev/null +++ b/statvar_imports/eurostat/Usually_resident_population_on_1_January/places_resolved_runtime.csv @@ -0,0 +1,264 @@ +place_name,dcid +AD,dcid:country/AND +AE,dcid:country/ARE +AF,dcid:country/AFG +AG,dcid:country/ATG +AI,dcid:country/AIA +AL,dcid:country/ALB +AM,dcid:country/ARM +AN,dcid:country/ANT +AO,dcid:country/AGO +AQ,dcid:country/ATA +AR,dcid:country/ARG +AS,dcid:country/ASM +AT,dcid:country/AUT +AU,dcid:country/AUS +AW,dcid:country/ABW +AZ,dcid:country/AZE +BA,dcid:country/BIH +BB,dcid:country/BRB +BD,dcid:country/BGD +BE,dcid:country/BEL +BF,dcid:country/BFA +BG,dcid:country/BGR +BH,dcid:country/BHR +BI,dcid:country/BDI +BJ,dcid:country/BEN +BL,dcid:country/BLM +BM,dcid:country/BMU +BN,dcid:country/BRN +BO,dcid:country/BOL +BR,dcid:country/BRA +BS,dcid:country/BHS +BT,dcid:country/BTN +BV,dcid:country/BVT +BW,dcid:country/BWA +BY,dcid:country/BLR +BZ,dcid:country/BLZ +CA,dcid:country/CAN +CC,dcid:country/CCK +CD,dcid:country/COD +CF,dcid:country/CAF +CG,dcid:country/COG +CH,dcid:country/CHE +CI,dcid:country/CIV +CK,dcid:country/COK +CL,dcid:country/CHL +CM,dcid:country/CMR +CN,dcid:country/CHN +CO,dcid:country/COL +CR,dcid:country/CRI +CU,dcid:country/CUB +CV,dcid:country/CPV +CX,dcid:country/CXR +CY,dcid:country/CYP +CZ,dcid:country/CZE +DE,dcid:country/DEU +DJ,dcid:country/DJI +DK,dcid:country/DNK +DM,dcid:country/DMA +DO,dcid:country/DOM +DZ,dcid:country/DZA +EC,dcid:country/ECU +EE,dcid:country/EST +EG,dcid:country/EGY +EH,dcid:country/ESH +EL,dcid:country/GRC +ER,dcid:country/ERI +ES,dcid:country/ESP +ET,dcid:country/ETH +FI,dcid:country/FIN +FJ,dcid:country/FJI +FK,dcid:country/FLK +FM,dcid:country/FSM +FO,dcid:country/FRO +FR,dcid:country/FRA +FX,dcid:country/FXX +GA,dcid:country/GAB +UK,dcid:country/GBR +GD,dcid:country/GRD +GE,dcid:country/GEO +GF,dcid:country/GUF +GG,dcid:country/GGY +GH,dcid:country/GHA +GI,dcid:country/GIB +GL,dcid:country/GRL +GM,dcid:country/GMB +GN,dcid:country/GIN +GQ,dcid:country/GNQ +GS,dcid:country/SGS +GT,dcid:country/GTM +GU,dcid:country/GUM +GW,dcid:country/GNB +GY,dcid:country/GUY +HK,dcid:country/HKG +HM,dcid:country/HMD +HN,dcid:country/HND +HR,dcid:country/HRV +HT,dcid:country/HTI +HU,dcid:country/HUN +ID,dcid:country/IDN +IE,dcid:country/IRL +IL,dcid:country/ISR +IM,dcid:country/IMN +IN,dcid:country/IND +IO,dcid:country/IOT +IQ,dcid:country/IRQ +IR,dcid:country/IRN +IS,dcid:country/ISL +IT,dcid:country/ITA +JE,dcid:country/JEY +JM,dcid:country/JAM +JO,dcid:country/JOR +JP,dcid:country/JPN +KE,dcid:country/KEN +KG,dcid:country/KGZ +KH,dcid:country/KHM +KI,dcid:country/KIR +KM,dcid:country/COM +KN,dcid:country/KNA +KP,dcid:country/PRK +KR,dcid:country/KOR +KW,dcid:country/KWT +KY,dcid:country/CYM +KZ,dcid:country/KAZ +LA,dcid:country/LAO +LB,dcid:country/LBN +LC,dcid:country/LCA +LI,dcid:country/LIE +LK,dcid:country/LKA +LR,dcid:country/LBR +LS,dcid:country/LSO +LT,dcid:country/LTU +LU,dcid:country/LUX +LV,dcid:country/LVA +LY,dcid:country/LBY +MA,dcid:country/MAR +MC,dcid:country/MCO +MD,dcid:country/MDA +ME,dcid:country/MNE +MF,dcid:country/MAF +MG,dcid:country/MDG +MH,dcid:country/MHL +MK,dcid:country/MKD +ML,dcid:country/MLI +MM,dcid:country/MMR +MN,dcid:country/MNG +MO,dcid:country/MAC +MP,dcid:country/MNP +MQ,dcid:country/MTQ +MR,dcid:country/MRT +MS,dcid:country/MSR +MT,dcid:country/MLT +MU,dcid:country/MUS +MV,dcid:country/MDV +MW,dcid:country/MWI +MX,dcid:country/MEX +MY,dcid:country/MYS +MZ,dcid:country/MOZ +NA,dcid:country/NAM +NC,dcid:country/NCL +NE,dcid:country/NER +NF,dcid:country/NFK +NG,dcid:country/NGA +NI,dcid:country/NIC +NL,dcid:country/NLD +NO,dcid:country/NOR +NP,dcid:country/NPL +NR,dcid:country/NRU +NU,dcid:country/NIU +NZ,dcid:country/NZL +OM,dcid:country/OMN +PA,dcid:country/PAN +PE,dcid:country/PER +PF,dcid:country/PYF +PG,dcid:country/PNG +PH,dcid:country/PHL +PK,dcid:country/PAK +PL,dcid:country/POL +PM,dcid:country/SPM +PN,dcid:country/PCN +PR,dcid:country/PRI +PS,dcid:country/PSE +PT,dcid:country/PRT +PW,dcid:country/PLW +PY,dcid:country/PRY +QA,dcid:country/QAT +RE,dcid:country/REU +RO,dcid:country/ROU +RS,dcid:country/SRB +RU,dcid:country/RUS +RW,dcid:country/RWA +SA,dcid:country/SAU +SB,dcid:country/SLB +SC,dcid:country/SYC +SD,dcid:country/SDN +SE,dcid:country/SWE +SG,dcid:country/SGP +SH,dcid:country/SHN +SI,dcid:country/SVN +SJ,dcid:country/SJM +SK,dcid:country/SVK +SL,dcid:country/SLE +SM,dcid:country/SMR +SN,dcid:country/SEN +SO,dcid:country/SOM +SR,dcid:country/SUR +ST,dcid:country/STP +SV,dcid:country/SLV +SX,dcid:country/SXM +SY,dcid:country/SYR +SZ,dcid:country/SWZ +TC,dcid:country/TCA +TD,dcid:country/TCD +TF,dcid:country/ATF +TG,dcid:country/TGO +TH,dcid:country/THA +TJ,dcid:country/TJK +TK,dcid:country/TKL +TL,dcid:country/TLS +TM,dcid:country/TKM +TN,dcid:country/TUN +TO,dcid:country/TON +TR,dcid:country/TUR +TT,dcid:country/TTO +TV,dcid:country/TUV +TW,dcid:country/TWN +TZ,dcid:country/TZA +UA,dcid:country/UKR +UG,dcid:country/UGA +UM,dcid:country/UMI +US,dcid:country/USA +UY,dcid:country/URY +UZ,dcid:country/UZB +VA,dcid:country/VAT +VC,dcid:country/VCT +VE,dcid:country/VEN +VG,dcid:country/VGB +VI,dcid:country/VIR +VN,dcid:country/VNM +VU,dcid:country/VUT +WF,dcid:country/WLF +WS,dcid:country/WSM +XK,dcid:country/XKS +YE,dcid:country/YEM +YT,dcid:country/MYT +ZA,dcid:country/ZAF +ZM,dcid:country/ZMB +ZW,dcid:country/ZWE +DE_TOT,dcid:nuts/DE_TOT +EA,dcid:nuts/EA +EA12,dcid:nuts/EA12 +EA18,dcid:nuts/EA18 +EA19,dcid:nuts/EA19 +EEA,dcid:nuts/EEA +EEA30_2007,dcid:nuts/EEA30_2007 +EEA31,dcid:nuts/EEA31 +EFTA,dcid:nuts/EFTA +EU,dcid:nuts/EU +EU27,dcid:nuts/EU27 +EU27_2007,dcid:nuts/EU27_2007 +EU27_2020,dcid:nuts/EU27_2020 +EA20,dcid:nuts/EA20 +EA21,dcid:nuts/EA21 +EU28,dcid:nuts/EU28 diff --git a/statvar_imports/eurostat/Usually_resident_population_on_1_January/run.sh b/statvar_imports/eurostat/Usually_resident_population_on_1_January/run.sh new file mode 100644 index 0000000000..6e067eb731 --- /dev/null +++ b/statvar_imports/eurostat/Usually_resident_population_on_1_January/run.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# Step 1: Data Download +mkdir -p source_files +curl -L --retry 3 "https://ec.europa.eu/eurostat/api/dissemination/sdmx/2.1/data/DEMO_URESPOP/?format=SDMX-CSV&compressed=false" -o ./source_files/Usually_resident_population_on_1_January_input.csv + +# Step 2: Data Processing +python3 ../../../tools/statvar_importer/stat_var_processor.py \ + "--input_data=./source_files/*.csv" \ + "--pv_map=./Usually_resident_population_on_1_January_pvmap.csv" \ + "--config_file=./Usually_resident_population_on_1_January_metadata.csv" \ + "--generate_statvar_name=True" \ + "--skip_constant_csv_columns=False" \ + "--output_columns=observationDate,observationAbout,variableMeasured,value,observationPeriod,unit" \ + "--output_path=./Usually_resident_population_on_1_January_output" \ + "--places_resolved_csv=./places_resolved_runtime.csv" \ + "--existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf" diff --git a/statvar_imports/eurostat/Usually_resident_population_on_1_January/test_data/Usually_resident_population_on_1_January_input.csv b/statvar_imports/eurostat/Usually_resident_population_on_1_January/test_data/Usually_resident_population_on_1_January_input.csv new file mode 100644 index 0000000000..45eb337956 --- /dev/null +++ b/statvar_imports/eurostat/Usually_resident_population_on_1_January/test_data/Usually_resident_population_on_1_January_input.csv @@ -0,0 +1,100 @@ +DATAFLOW,LAST UPDATE,freq,unit,geo,TIME_PERIOD,OBS_VALUE,OBS_FLAG,CONF_STATUS +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,AT,2014,8511000,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,AT,2015,8581500,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,AT,2016,8711500,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,AT,2017,8752500,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,AT,2018,8802000,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,AT,2019,8842000,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,AT,2020,8897000,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,AT,2021,8926000,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,AT,2022,8967500,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,AT,2023,9087000,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,AT,2024,9132500,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,AT,2025,9180000,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,BE,2014,11203992,, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,BE,2015,11258434,, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,BE,2016,11289853,b, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,BE,2017,11365834,, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,BE,2018,11413058,, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,BE,2019,11467923,, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,BE,2020,11549888,, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,BE,2021,11566041,, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,BE,2022,11631136,, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,BE,2023,11754004,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,BE,2024,11832049,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,BE,2025,11900123,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,BG,2014,7245677,, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,BG,2015,7202198,, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,BG,2016,7153784,, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,BG,2017,7101859,, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,BG,2018,7050034,, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,BG,2019,7000039,, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,BG,2020,6951482,, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,BG,2021,6916548,, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,BG,2022,6838937,, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,BG,2023,6498567,b, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,BG,2024,6445481,, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,BG,2025,6437360,, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,CY,2014,858000,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,CY,2015,847008,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,CY,2016,848319,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,CY,2017,854802,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,CY,2018,864236,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,CY,2019,875898,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,CY,2020,888005,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,CY,2021,896000,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,CY,2022,904700,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,CY,2023,920701,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,CY,2024,933505,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,CY,2025,982966,b, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,CZ,2014,10398697,, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,CZ,2015,10419743,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,CZ,2016,10445783,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,CZ,2017,10467628,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,CZ,2018,10493154,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,CZ,2019,10528984,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,CZ,2020,10557001,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,CZ,2021,10574153,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,CZ,2022,10545457,b, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,CZ,2023,10833385,be, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,CZ,2024,10916836,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,CZ,2025,10921096,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,DE_TOT,2014,80704691,, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,DE_TOT,2015,81089331,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,DE_TOT,2016,82064489,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,DE_TOT,2017,82437641,be, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,DE_TOT,2018,82719022,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,DE_TOT,2019,82940663,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,DE_TOT,2020,83100961,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,DE_TOT,2021,83120520,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,DE_TOT,2022,83203320,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,DE_TOT,2023,84311244,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,DE_TOT,2024,83399432,be, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,DE_TOT,2025,83502300,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,DK,2014,5621607,, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,DK,2015,5653357,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,DK,2016,5700917,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,DK,2017,5743947,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,DK,2018,5774877,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,DK,2019,5799763,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,DK,2020,5816443,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,DK,2021,5833883,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,DK,2022,5864667,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,DK,2023,5921952,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,DK,2024,5954471,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,DK,2025,5985790,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,EA19,2014,338270039,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,EA19,2015,339160633,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,EA19,2016,340472701,be, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,EA19,2017,341390052,be, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,EA19,2018,342177328,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,EA19,2019,342671916,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,EA19,2020,343526388,e, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,EA19,2021,343074378,be, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,EA19,2022,343600422,be, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,EA19,2023,346452616,be, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,EA19,2024,347062375,be, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,EA19,2025,348310251,be, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,EA20,2023,350303510,be, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,EA20,2024,350924342,be, +ESTAT:DEMO_URESPOP(1.0),08/10/25 23:00:00,A,NR,EA20,2025,352184601,be, \ No newline at end of file diff --git a/statvar_imports/eurostat/Usually_resident_population_on_1_January/test_data/Usually_resident_population_on_1_January_output.csv b/statvar_imports/eurostat/Usually_resident_population_on_1_January/test_data/Usually_resident_population_on_1_January_output.csv new file mode 100644 index 0000000000..2abed5bb10 --- /dev/null +++ b/statvar_imports/eurostat/Usually_resident_population_on_1_January/test_data/Usually_resident_population_on_1_January_output.csv @@ -0,0 +1,101 @@ +observationDate,observationAbout,variableMeasured,value,observationPeriod,measurementMethod,unit,scalingFactor +2014,country/AUT,dcid:Count_Person,8511000,P1Y,dcs:EstimatedValue,dcs:Number, +2015,country/AUT,dcid:Count_Person,8581500,P1Y,dcs:EstimatedValue,dcs:Number, +2016,country/AUT,dcid:Count_Person,8711500,P1Y,dcs:EstimatedValue,dcs:Number, +2017,country/AUT,dcid:Count_Person,8752500,P1Y,dcs:EstimatedValue,dcs:Number, +2018,country/AUT,dcid:Count_Person,8802000,P1Y,dcs:EstimatedValue,dcs:Number, +2019,country/AUT,dcid:Count_Person,8842000,P1Y,dcs:EstimatedValue,dcs:Number, +2020,country/AUT,dcid:Count_Person,8897000,P1Y,dcs:EstimatedValue,dcs:Number, +2021,country/AUT,dcid:Count_Person,8926000,P1Y,dcs:EstimatedValue,dcs:Number, +2022,country/AUT,dcid:Count_Person,8967500,P1Y,dcs:EstimatedValue,dcs:Number, +2024,country/AUT,dcid:Count_Person,9132500,P1Y,dcs:EstimatedValue,dcs:Number, +2014,country/BEL,dcid:Count_Person,11203992,P1Y,,dcs:Number, +2015,country/BEL,dcid:Count_Person,11258434,P1Y,,dcs:Number, +2016,country/BEL,dcid:Count_Person,11289853,P1Y,dcs:SeriesBreak,dcs:Number, +2017,country/BEL,dcid:Count_Person,11365834,P1Y,,dcs:Number, +2018,country/BEL,dcid:Count_Person,11413058,P1Y,,dcs:Number, +2020,country/BEL,dcid:Count_Person,11549888,P1Y,,dcs:Number, +2021,country/BEL,dcid:Count_Person,11566041,P1Y,,dcs:Number, +2023,country/BEL,dcid:Count_Person,11754004,P1Y,dcs:EstimatedValue,dcs:Number, +2014,country/BGR,dcid:Count_Person,7245677,P1Y,,dcs:Number, +2015,country/BGR,dcid:Count_Person,7202198,P1Y,,dcs:Number, +2016,country/BGR,dcid:Count_Person,7153784,P1Y,,dcs:Number, +2017,country/BGR,dcid:Count_Person,7101859,P1Y,,dcs:Number, +2018,country/BGR,dcid:Count_Person,7050034,P1Y,,dcs:Number, +2019,country/BGR,dcid:Count_Person,7000039,P1Y,,dcs:Number, +2020,country/BGR,dcid:Count_Person,6951482,P1Y,,dcs:Number, +2022,country/BGR,dcid:Count_Person,6838937,P1Y,,dcs:Number, +2023,country/BGR,dcid:Count_Person,6498567,P1Y,dcs:SeriesBreak,dcs:Number, +2024,country/BGR,dcid:Count_Person,6445481,P1Y,,dcs:Number, +2025,country/BGR,dcid:Count_Person,6437360,P1Y,,dcs:Number, +2014,country/CYP,dcid:Count_Person,858000,P1Y,dcs:EstimatedValue,dcs:Number, +2015,country/CYP,dcid:Count_Person,847008,P1Y,dcs:EstimatedValue,dcs:Number, +2016,country/CYP,dcid:Count_Person,848319,P1Y,dcs:EstimatedValue,dcs:Number, +2017,country/CYP,dcid:Count_Person,854802,P1Y,dcs:EstimatedValue,dcs:Number, +2018,country/CYP,dcid:Count_Person,864236,P1Y,dcs:EstimatedValue,dcs:Number, +2020,country/CYP,dcid:Count_Person,888005,P1Y,dcs:EstimatedValue,dcs:Number, +2022,country/CYP,dcid:Count_Person,904700,P1Y,dcs:EstimatedValue,dcs:Number, +2025,country/CYP,dcid:Count_Person,982966,P1Y,dcs:SeriesBreak,dcs:Number, +2014,country/CZE,dcid:Count_Person,10398697,P1Y,,dcs:Number, +2015,country/CZE,dcid:Count_Person,10419743,P1Y,dcs:EstimatedValue,dcs:Number, +2016,country/CZE,dcid:Count_Person,10445783,P1Y,dcs:EstimatedValue,dcs:Number, +2017,country/CZE,dcid:Count_Person,10467628,P1Y,dcs:EstimatedValue,dcs:Number, +2018,country/CZE,dcid:Count_Person,10493154,P1Y,dcs:EstimatedValue,dcs:Number, +2019,country/CZE,dcid:Count_Person,10528984,P1Y,dcs:EstimatedValue,dcs:Number, +2020,country/CZE,dcid:Count_Person,10557001,P1Y,dcs:EstimatedValue,dcs:Number, +2022,country/CZE,dcid:Count_Person,10545457,P1Y,dcs:SeriesBreak,dcs:Number, +2023,country/CZE,dcid:Count_Person,10833385,P1Y,,dcs:Number, +2024,country/CZE,dcid:Count_Person,10916836,P1Y,dcs:EstimatedValue,dcs:Number, +2025,country/CZE,dcid:Count_Person,10921096,P1Y,dcs:EstimatedValue,dcs:Number, +2014,country/DEU,dcid:Count_Person,80704691,P1Y,,dcs:Number, +2015,country/DEU,dcid:Count_Person,81089331,P1Y,dcs:EstimatedValue,dcs:Number, +2016,country/DEU,dcid:Count_Person,82064489,P1Y,dcs:EstimatedValue,dcs:Number, +2017,country/DEU,dcid:Count_Person,82437641,P1Y,,dcs:Number, +2018,country/DEU,dcid:Count_Person,82719022,P1Y,dcs:EstimatedValue,dcs:Number, +2020,country/DEU,dcid:Count_Person,83100961,P1Y,dcs:EstimatedValue,dcs:Number, +2021,country/DEU,dcid:Count_Person,83120520,P1Y,dcs:EstimatedValue,dcs:Number, +2022,country/DEU,dcid:Count_Person,83203320,P1Y,dcs:EstimatedValue,dcs:Number, +2024,country/DEU,dcid:Count_Person,83399432,P1Y,,dcs:Number, +2014,country/DNK,dcid:Count_Person,5621607,P1Y,,dcs:Number, +2015,country/DNK,dcid:Count_Person,5653357,P1Y,dcs:EstimatedValue,dcs:Number, +2016,country/DNK,dcid:Count_Person,5700917,P1Y,dcs:EstimatedValue,dcs:Number, +2017,country/DNK,dcid:Count_Person,5743947,P1Y,dcs:EstimatedValue,dcs:Number, +2018,country/DNK,dcid:Count_Person,5774877,P1Y,dcs:EstimatedValue,dcs:Number, +2019,country/DNK,dcid:Count_Person,5799763,P1Y,dcs:EstimatedValue,dcs:Number, +2021,country/DNK,dcid:Count_Person,5833883,P1Y,dcs:EstimatedValue,dcs:Number, +2024,country/DNK,dcid:Count_Person,5954471,P1Y,dcs:EstimatedValue,dcs:Number, +2025,country/DNK,dcid:Count_Person,5985790,P1Y,dcs:EstimatedValue,dcs:Number, +2014,dcid:EuroArea19,dcid:Count_Person,338270039,P1Y,dcs:EstimatedValue,dcs:Number, +2015,dcid:EuroArea19,dcid:Count_Person,339160633,P1Y,dcs:EstimatedValue,dcs:Number, +2016,dcid:EuroArea19,dcid:Count_Person,340472701,P1Y,,dcs:Number, +2017,dcid:EuroArea19,dcid:Count_Person,341390052,P1Y,,dcs:Number, +2018,dcid:EuroArea19,dcid:Count_Person,342177328,P1Y,dcs:EstimatedValue,dcs:Number, +2020,dcid:EuroArea19,dcid:Count_Person,343526388,P1Y,dcs:EstimatedValue,dcs:Number, +2022,dcid:EuroArea19,dcid:Count_Person,343600422,P1Y,,dcs:Number, +2024,dcid:EuroArea19,dcid:Count_Person,347062375,P1Y,,dcs:Number, +2023,dcid:EuroArea20,dcid:Count_Person,350303510,P1Y,,dcs:Number, +2024,dcid:EuroArea20,dcid:Count_Person,350924342,P1Y,,dcs:Number, +2016,country/EST,dcid:Count_Person,1315944,P1Y,dcs:SeriesBreak,dcs:Number, +2018,country/EST,dcid:Count_Person,1319133,P1Y,,dcs:Number, +2020,country/EST,dcid:Count_Person,1328976,P1Y,,dcs:Number, +2022,country/EST,dcid:Count_Person,1331796,P1Y,,dcs:Number, +2023,country/EST,dcid:Count_Person,1365884,P1Y,,dcs:Number, +2025,country/EST,dcid:Count_Person,1369995,P1Y,,dcs:Number, +2014,country/GRC,dcid:Count_Person,10992783,P1Y,dcs:EstimatedValue,dcs:Number, +2016,country/GRC,dcid:Count_Person,10793526,P1Y,dcs:EstimatedValue,dcs:Number, +2019,country/GRC,dcid:Count_Person,10722287,P1Y,dcs:EstimatedValue,dcs:Number, +2020,country/GRC,dcid:Count_Person,10709739,P1Y,dcs:EstimatedValue,dcs:Number, +2021,country/GRC,dcid:Count_Person,10682547,P1Y,dcs:EstimatedValue,dcs:Number, +2024,country/GRC,dcid:Count_Person,10401302,P1Y,dcs:EstimatedValue,dcs:Number, +2025,country/GRC,dcid:Count_Person,10409547,P1Y,dcs:EstimatedValue,dcs:Number, +2015,country/ESP,dcid:Count_Person,46439864,P1Y,dcs:EstimatedValue,dcs:Number, +2017,country/ESP,dcid:Count_Person,46528966,P1Y,dcs:EstimatedValue,dcs:Number, +2019,country/ESP,dcid:Count_Person,46934632,P1Y,dcs:EstimatedValue,dcs:Number, +2022,country/ESP,dcid:Count_Person,47432805,P1Y,dcs:EstimatedValue,dcs:Number, +2023,country/ESP,dcid:Count_Person,48063694,P1Y,dcs:SeriesBreak,dcs:Number, +2025,country/ESP,dcid:Count_Person,49093546,P1Y,dcs:EstimatedValue,dcs:Number, +2014,dcid:EuropeanUnion27,dcid:Count_Person,443265700,P1Y,dcs:EstimatedValue,dcs:Number, +2016,dcid:EuropeanUnion27,dcid:Count_Person,445519516,P1Y,,dcs:Number, +2017,dcid:EuropeanUnion27,dcid:Count_Person,446346533,P1Y,,dcs:Number, +2019,dcid:EuropeanUnion27,dcid:Count_Person,447470672,P1Y,dcs:EstimatedValue,dcs:Number, + diff --git a/statvar_imports/eurostat/Usually_resident_population_on_1_January/test_data/Usually_resident_population_on_1_January_output.tmcf b/statvar_imports/eurostat/Usually_resident_population_on_1_January/test_data/Usually_resident_population_on_1_January_output.tmcf new file mode 100644 index 0000000000..32e6da39bd --- /dev/null +++ b/statvar_imports/eurostat/Usually_resident_population_on_1_January/test_data/Usually_resident_population_on_1_January_output.tmcf @@ -0,0 +1,10 @@ +Node: E:output->E0 +observationDate: C:output->observationDate +observationAbout: C:output->observationAbout +variableMeasured: C:output->variableMeasured +value: C:output->value +observationPeriod: C:output->observationPeriod +measurementMethod: C:output->measurementMethod +unit: C:output->unit +scalingFactor: C:output->scalingFactor +typeOf: dcs:StatVarObservation