From 63a0d45d94a90e24e3f96ce0544f98c06b94863c Mon Sep 17 00:00:00 2001 From: Chandrashekar Harish Date: Fri, 13 Jun 2025 10:08:50 +0000 Subject: [PATCH 1/7] India Life expectancy autorefresh --- statvar_imports/india_ndap/ndap/README.md | 19 ++++++ statvar_imports/india_ndap/ndap/config.py | 2 + .../india_ndap/ndap/download_script.py | 61 +++++++++++++++++++ statvar_imports/india_ndap/ndap/manifest.json | 23 +++++++ statvar_imports/india_ndap/ndap/metadata.csv | 10 +++ .../india_ndap/ndap/places_resolved.csv | 24 ++++++++ statvar_imports/india_ndap/ndap/pv_map.csv | 29 +++++++++ 7 files changed, 168 insertions(+) create mode 100644 statvar_imports/india_ndap/ndap/README.md create mode 100644 statvar_imports/india_ndap/ndap/config.py create mode 100644 statvar_imports/india_ndap/ndap/download_script.py create mode 100644 statvar_imports/india_ndap/ndap/manifest.json create mode 100644 statvar_imports/india_ndap/ndap/metadata.csv create mode 100644 statvar_imports/india_ndap/ndap/places_resolved.csv create mode 100644 statvar_imports/india_ndap/ndap/pv_map.csv diff --git a/statvar_imports/india_ndap/ndap/README.md b/statvar_imports/india_ndap/ndap/README.md new file mode 100644 index 0000000000..f23339ccbd --- /dev/null +++ b/statvar_imports/india_ndap/ndap/README.md @@ -0,0 +1,19 @@ +# NDAP - India Life Expectancy + +- source: `https://ndap.niti.gov.in/dataset/7375`, + +- how to download data: The files can be downloaded using `python3 scripts/world_bank/worldbank_ids/download.py`. + +- type of place: State. + +- statvars: Health + +- years: 1997 to 2020 + +- place_resolution: Places are resolved based on name. + +### How to run: + +`python3 donwload_script.py` + +`python3 stat_var_processor.py --input_data=../../statvar_imports/india_ndap/ndap/input_files/India_LifeExpectancy.csv --pv_map=../../statvar_imports/india_ndap/ndap/pv_map.csv --config_file=../../statvar_imports/india_ndap/ndap/metadata.csv --output_path=../../statvar_imports/india_ndap/ndap/output/Life_expectancy"` diff --git a/statvar_imports/india_ndap/ndap/config.py b/statvar_imports/india_ndap/ndap/config.py new file mode 100644 index 0000000000..f7fee18e27 --- /dev/null +++ b/statvar_imports/india_ndap/ndap/config.py @@ -0,0 +1,2 @@ +url='https://loadqa.ndapapi.com/v1/openapi?API_Key=gAAAAABnrzyB1tiwHrggYMoK81fqeUFDhVWacu2CFsVRzMJ4A23xSK4Ov6JLZYHpVi3fQK2ikFB65Xo56PpgeISlZB0OEYfPltYSRIYRfPqb27TNDZd_AK-gdnEIbkyqVuaZCkO9RQYh44TG2mGZwaRC3Yo16s8Ks8-FXtTf2RsV5c-L243hAfGRI984Bed6UTUJMWmDe4Fh&ind=I7375_4&dim=Country,StateName,StateCode,Year,GENDER' +input_files='input_files/' \ No newline at end of file diff --git a/statvar_imports/india_ndap/ndap/download_script.py b/statvar_imports/india_ndap/ndap/download_script.py new file mode 100644 index 0000000000..e0df8acf15 --- /dev/null +++ b/statvar_imports/india_ndap/ndap/download_script.py @@ -0,0 +1,61 @@ +import requests +import pandas as pd +import config +import os +from retry import retry +"logging configuration " +import logging +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) +handler = logging.StreamHandler() +formatter = logging.Formatter('%(message)s') +handler.setFormatter(formatter) +logger.addHandler(handler) + +@retry(tries=3, delay=5, backoff=2) +def retry_method(url, headers=None): + response = requests.get(url, headers=headers, timeout=10) + response.raise_for_status() + return response +def get_api_response(api_url): + """Fetches data from the API and returns the JSON response.""" + try: + response = retry_method(api_url) + response.raise_for_status() + return response.json() + except requests.exceptions.RequestException as e: + logger.exception(f"Error fetching data from API: {e}") + return None + except ValueError as e: + logger.exception(f"Invalid JSON response: {e}. Response text was: {response.text}") + return None + except Exception as e: + logger.exception(f"A general error occurred: {e}") + return None + +if __name__=="__main__": + all_data = [] + + for page_num in range(1, 3): + api_url = f"{config.url}&pageno={page_num}" + response_data = get_api_response(api_url) + + if response_data: + keys = list(response_data.values())[5] + + for i in keys: + a = a=i['StateName'],i['Year'].split(",")[-1].strip(),i['GENDER'],i['I7375_4']['TotalPopulationWeight'],i['Year'].split(",")[-1].strip(),i['Year'] + all_data.append(a) + else: + logger.info(f"Failed to retrieve data from page {page_num}. Stopping.") + break + + if all_data: + df = pd.DataFrame(all_data, columns=['srcStateName', 'srcYear', 'GENDER', 'Life Expectancy', 'YearCode', 'Year']) + os.makedirs(config.input_files, exist_ok=True) + input_filename = os.path.join(config.input_files, 'India_LifeExpectancy.csv') + df.to_csv(input_filename, index=False) + logger.info("Data saved to India_LifeExpectancy_input.csv") + else: + logger.info("No data was retrieved from the API.") + diff --git a/statvar_imports/india_ndap/ndap/manifest.json b/statvar_imports/india_ndap/ndap/manifest.json new file mode 100644 index 0000000000..ff45a574fc --- /dev/null +++ b/statvar_imports/india_ndap/ndap/manifest.json @@ -0,0 +1,23 @@ +{ + "import_specifications": [ + { + "import_name": "India_LifeExpectancy", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://ndap.niti.gov.in/dataset/7375", + "provenance_description": "Life expectancy of a person in India based on gender at state level.", + "scripts": [ + "download_script.py", + "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/India_LifeExpectancy.csv --pv_map='pv_map.csv' --config_file=metadata.csv --output_path=output/Life_expectancy" + ], + "import_inputs": [ + { + "template_mcf": "output/Life_expectancy.tmcf", + "cleaned_csv": "output/Life_expectancy.csv" + } + ], + "cron_schedule": "00 11 1,15 * *" + } + ] +} \ No newline at end of file diff --git a/statvar_imports/india_ndap/ndap/metadata.csv b/statvar_imports/india_ndap/ndap/metadata.csv new file mode 100644 index 0000000000..f3ee245407 --- /dev/null +++ b/statvar_imports/india_ndap/ndap/metadata.csv @@ -0,0 +1,10 @@ +config,value +provenance_url,https://ndap.niti.gov.in/dataset/7375 +header_rows,1 +output_columns,"observationAbout,observationDate,value,variableMeasured,unit" +#places_within,country/IND +#place_types,"AdministrativeArea,AdministrativeArea1,AdministrativeArea2,State" +mapped_columns,3 +mapped_rows,1 +#input_rows,50 +#debug,1 diff --git a/statvar_imports/india_ndap/ndap/places_resolved.csv b/statvar_imports/india_ndap/ndap/places_resolved.csv new file mode 100644 index 0000000000..8ac24a7313 --- /dev/null +++ b/statvar_imports/india_ndap/ndap/places_resolved.csv @@ -0,0 +1,24 @@ +place_name,observationAbout,dcid +jammu and kashmir,observationAbout,wikidataId/Q1180 +himachal pradesh,observationAbout,wikidataId/Q1177 +punjab,observationAbout,wikidataId/Q22424 +uttarakhand,observationAbout,wikidataId/Q1499 +haryana,observationAbout,wikidataId/Q1174 +delhi,observationAbout,wikidataId/Q1353 +rajasthan,observationAbout,wikidataId/Q1437 +uttar pradesh,observationAbout,wikidataId/Q1498 +bihar,observationAbout,wikidataId/Q1165 +assam,observationAbout,wikidataId/Q1164 +West Bengal,observationAbout,wikidataId/Q1356 +jharkhand,observationAbout,wikidataId/Q1184 +odisha,observationAbout,wikidataId/Q22048 +chhattisgarh,observationAbout,wikidataId/Q1168 +madhya pradesh,observationAbout,wikidataId/Q1188 +gujarat,observationAbout,wikidataId/Q1061 +maharashtra,observationAbout,wikidataId/Q1191 +kerala,observationAbout,wikidataId/Q1186 +andhra pradesh,observationAbout,wikidataId/Q1159 +karnataka,observationAbout,wikidataId/Q1185 +tamil nadu,observationAbout,wikidataId/Q1445 +telangana,observationAbout,wikidataId/Q677037 +ladakh,observationAbout,wikidataId/Q200667 \ No newline at end of file diff --git a/statvar_imports/india_ndap/ndap/pv_map.csv b/statvar_imports/india_ndap/ndap/pv_map.csv new file mode 100644 index 0000000000..0b79b22af4 --- /dev/null +++ b/statvar_imports/india_ndap/ndap/pv_map.csv @@ -0,0 +1,29 @@ +key,prop,val,,,,,, +life expectancy,measuredProperty,lifeExpectancy,value,{Number},populationType,Person,unit,Year +male,gender,Male,,,,,, +female,gender,Female,,,,,, +Total,gender,"""""",,,,,, +srcyear,observationDate,{Number},,,,,, +jammu and kashmir,observationAbout,wikidataId/Q1180 +himachal pradesh,observationAbout,wikidataId/Q1177 +punjab,observationAbout,wikidataId/Q22424 +uttarakhand,observationAbout,wikidataId/Q1499 +haryana,observationAbout,wikidataId/Q1174 +delhi,observationAbout,wikidataId/Q1353 +rajasthan,observationAbout,wikidataId/Q1437 +uttar pradesh,observationAbout,wikidataId/Q1498 +bihar,observationAbout,wikidataId/Q1165 +assam,observationAbout,wikidataId/Q1164 +West Bengal,observationAbout,wikidataId/Q1356 +jharkhand,observationAbout,wikidataId/Q1184 +odisha,observationAbout,wikidataId/Q22048 +chhattisgarh,observationAbout,wikidataId/Q1168 +madhya pradesh,observationAbout,wikidataId/Q1188 +gujarat,observationAbout,wikidataId/Q1061 +maharashtra,observationAbout,wikidataId/Q1191 +kerala,observationAbout,wikidataId/Q1186 +andhra pradesh,observationAbout,wikidataId/Q1159 +karnataka,observationAbout,wikidataId/Q1185 +tamil nadu,observationAbout,wikidataId/Q1445 +telangana,observationAbout,wikidataId/Q677037 +ladakh,observationAbout,wikidataId/Q200667 \ No newline at end of file From 363a77e93c632dd9721f5a36d7699114c069920d Mon Sep 17 00:00:00 2001 From: Chandrashekar Harish Date: Fri, 13 Jun 2025 10:27:10 +0000 Subject: [PATCH 2/7] added test data --- ....csv => India_LifeExpectancy_metadata.csv} | 0 ...> India_LifeExpectancy_placesresolved.csv} | 0 ...map.csv => India_LifeExpectancy_pvmap.csv} | 0 statvar_imports/india_ndap/ndap/README.md | 2 +- .../india_ndap/ndap/download_script.py | 2 +- statvar_imports/india_ndap/ndap/manifest.json | 2 +- .../ndap/test_data/India_LifeExpectancy.csv | 96 +++++++++++++++ .../ndap/test_data/India_LifeExpectancy.tmcf | 7 ++ .../test_data/India_LifeExpectancy_input.csv | 115 ++++++++++++++++++ 9 files changed, 221 insertions(+), 3 deletions(-) rename statvar_imports/india_ndap/ndap/{metadata.csv => India_LifeExpectancy_metadata.csv} (100%) rename statvar_imports/india_ndap/ndap/{places_resolved.csv => India_LifeExpectancy_placesresolved.csv} (100%) rename statvar_imports/india_ndap/ndap/{pv_map.csv => India_LifeExpectancy_pvmap.csv} (100%) create mode 100644 statvar_imports/india_ndap/ndap/test_data/India_LifeExpectancy.csv create mode 100644 statvar_imports/india_ndap/ndap/test_data/India_LifeExpectancy.tmcf create mode 100644 statvar_imports/india_ndap/ndap/test_data/India_LifeExpectancy_input.csv diff --git a/statvar_imports/india_ndap/ndap/metadata.csv b/statvar_imports/india_ndap/ndap/India_LifeExpectancy_metadata.csv similarity index 100% rename from statvar_imports/india_ndap/ndap/metadata.csv rename to statvar_imports/india_ndap/ndap/India_LifeExpectancy_metadata.csv diff --git a/statvar_imports/india_ndap/ndap/places_resolved.csv b/statvar_imports/india_ndap/ndap/India_LifeExpectancy_placesresolved.csv similarity index 100% rename from statvar_imports/india_ndap/ndap/places_resolved.csv rename to statvar_imports/india_ndap/ndap/India_LifeExpectancy_placesresolved.csv diff --git a/statvar_imports/india_ndap/ndap/pv_map.csv b/statvar_imports/india_ndap/ndap/India_LifeExpectancy_pvmap.csv similarity index 100% rename from statvar_imports/india_ndap/ndap/pv_map.csv rename to statvar_imports/india_ndap/ndap/India_LifeExpectancy_pvmap.csv diff --git a/statvar_imports/india_ndap/ndap/README.md b/statvar_imports/india_ndap/ndap/README.md index f23339ccbd..4461c726d3 100644 --- a/statvar_imports/india_ndap/ndap/README.md +++ b/statvar_imports/india_ndap/ndap/README.md @@ -16,4 +16,4 @@ `python3 donwload_script.py` -`python3 stat_var_processor.py --input_data=../../statvar_imports/india_ndap/ndap/input_files/India_LifeExpectancy.csv --pv_map=../../statvar_imports/india_ndap/ndap/pv_map.csv --config_file=../../statvar_imports/india_ndap/ndap/metadata.csv --output_path=../../statvar_imports/india_ndap/ndap/output/Life_expectancy"` +`python3 stat_var_processor.py --input_data=../../statvar_imports/india_ndap/ndap/input_files/India_LifeExpectancy_input.csv --pv_map=../../statvar_imports/india_ndap/ndap/India_LifeExpectancy_pvmap.csv --config_file=../../statvar_imports/india_ndap/ndap/India_LifeExpectancy_metadata.csv --output_path=../../statvar_imports/india_ndap/ndap/output/Life_expectancy` diff --git a/statvar_imports/india_ndap/ndap/download_script.py b/statvar_imports/india_ndap/ndap/download_script.py index e0df8acf15..277d01a687 100644 --- a/statvar_imports/india_ndap/ndap/download_script.py +++ b/statvar_imports/india_ndap/ndap/download_script.py @@ -53,7 +53,7 @@ def get_api_response(api_url): if all_data: df = pd.DataFrame(all_data, columns=['srcStateName', 'srcYear', 'GENDER', 'Life Expectancy', 'YearCode', 'Year']) os.makedirs(config.input_files, exist_ok=True) - input_filename = os.path.join(config.input_files, 'India_LifeExpectancy.csv') + input_filename = os.path.join(config.input_files, 'India_LifeExpectancy_input.csv') df.to_csv(input_filename, index=False) logger.info("Data saved to India_LifeExpectancy_input.csv") else: diff --git a/statvar_imports/india_ndap/ndap/manifest.json b/statvar_imports/india_ndap/ndap/manifest.json index ff45a574fc..bc0b9524d0 100644 --- a/statvar_imports/india_ndap/ndap/manifest.json +++ b/statvar_imports/india_ndap/ndap/manifest.json @@ -9,7 +9,7 @@ "provenance_description": "Life expectancy of a person in India based on gender at state level.", "scripts": [ "download_script.py", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/India_LifeExpectancy.csv --pv_map='pv_map.csv' --config_file=metadata.csv --output_path=output/Life_expectancy" + "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/India_LifeExpectancy_input.csv --pv_map='India_LifeExpectancy_pvmap.csv' --config_file=India_LifeExpectancy_metadata.csv --output_path=output/Life_expectancy" ], "import_inputs": [ { diff --git a/statvar_imports/india_ndap/ndap/test_data/India_LifeExpectancy.csv b/statvar_imports/india_ndap/ndap/test_data/India_LifeExpectancy.csv new file mode 100644 index 0000000000..00aa94e15b --- /dev/null +++ b/statvar_imports/india_ndap/ndap/test_data/India_LifeExpectancy.csv @@ -0,0 +1,96 @@ +observationAbout,observationDate,value,variableMeasured,unit +wikidataId/Q1180,2002,67.3,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1180,2002,64.7,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1180,2002,65.9,dcid:LifeExpectancy_Person,Year +wikidataId/Q1180,2003,68,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1180,2003,64.4,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1180,2003,66,dcid:LifeExpectancy_Person,Year +wikidataId/Q1180,2004,68.9,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1180,2004,65.9,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1180,2004,67.3,dcid:LifeExpectancy_Person,Year +wikidataId/Q1180,2005,69.9,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1180,2005,67.1,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1180,2005,68.4,dcid:LifeExpectancy_Person,Year +wikidataId/Q1180,2006,71,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1180,2006,69.2,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1180,2006,70,dcid:LifeExpectancy_Person,Year +wikidataId/Q1180,2007,70.8,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1180,2007,68.6,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1180,2007,69.6,dcid:LifeExpectancy_Person,Year +wikidataId/Q1180,2008,70.6,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1180,2008,69,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1180,2008,69.8,dcid:LifeExpectancy_Person,Year +wikidataId/Q1180,2009,71.2,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1180,2009,68.9,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1180,2009,70,dcid:LifeExpectancy_Person,Year +wikidataId/Q1180,2010,71.1,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1180,2010,69.2,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1180,2010,70.1,dcid:LifeExpectancy_Person,Year +wikidataId/Q1180,2011,71.9,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1180,2011,69.4,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1180,2011,70.5,dcid:LifeExpectancy_Person,Year +wikidataId/Q1180,2012,72.4,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1180,2012,69.9,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1180,2012,71,dcid:LifeExpectancy_Person,Year +wikidataId/Q1180,2013,74,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1180,2013,70.6,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1180,2013,72,dcid:LifeExpectancy_Person,Year +wikidataId/Q1180,2014,74.9,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1180,2014,70.9,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1180,2014,72.6,dcid:LifeExpectancy_Person,Year +wikidataId/Q1180,2015,76.1,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1180,2015,71.2,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1180,2015,73.2,dcid:LifeExpectancy_Person,Year +wikidataId/Q1180,2016,68.5,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1180,2016,71.6,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1180,2016,73.5,dcid:LifeExpectancy_Person,Year +wikidataId/Q1180,2017,76.7,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1180,2017,72.1,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1180,2017,74.1,dcid:LifeExpectancy_Person,Year +wikidataId/Q1180,2018,76.2,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1180,2018,72.2,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1180,2018,74,dcid:LifeExpectancy_Person,Year +wikidataId/Q1180,2019,76.1,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1180,2019,72.6,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1180,2019,74.2,dcid:LifeExpectancy_Person,Year +wikidataId/Q1180,2020,76.3,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1180,2020,72.6,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1180,2020,74.3,dcid:LifeExpectancy_Person,Year +wikidataId/Q1177,1997,65.2,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1177,1997,64.6,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1177,1997,64.9,dcid:LifeExpectancy_Person,Year +wikidataId/Q1177,1998,65.5,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1177,1998,64.8,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1177,1998,65.2,dcid:LifeExpectancy_Person,Year +wikidataId/Q1177,1999,67.7,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1177,1999,64.8,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1177,1999,66.2,dcid:LifeExpectancy_Person,Year +wikidataId/Q1177,2000,68.3,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1177,2000,65.7,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1177,2000,67,dcid:LifeExpectancy_Person,Year +wikidataId/Q1177,2001,69.4,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1177,2001,66.3,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1177,2001,67.8,dcid:LifeExpectancy_Person,Year +wikidataId/Q1177,2002,70.2,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1177,2002,66.6,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1177,2002,68.3,dcid:LifeExpectancy_Person,Year +wikidataId/Q1177,2003,71.4,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1177,2003,67.1,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1177,2003,69.1,dcid:LifeExpectancy_Person,Year +wikidataId/Q1177,2004,72,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1177,2004,67.3,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1177,2004,69.5,dcid:LifeExpectancy_Person,Year +wikidataId/Q1177,2005,71.8,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1177,2005,67.4,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1177,2005,69.5,dcid:LifeExpectancy_Person,Year +wikidataId/Q1177,2006,72,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1177,2006,67.4,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1177,2006,69.6,dcid:LifeExpectancy_Person,Year +wikidataId/Q1177,2007,72.2,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1177,2007,67.8,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1177,2007,69.9,dcid:LifeExpectancy_Person,Year +wikidataId/Q1177,2008,72.1,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1177,2008,67.7,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1177,2008,69.8,dcid:LifeExpectancy_Person,Year +wikidataId/Q1177,2009,72,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1177,2009,67.7,dcid:LifeExpectancy_Person_Male,Year diff --git a/statvar_imports/india_ndap/ndap/test_data/India_LifeExpectancy.tmcf b/statvar_imports/india_ndap/ndap/test_data/India_LifeExpectancy.tmcf new file mode 100644 index 0000000000..4bea313414 --- /dev/null +++ b/statvar_imports/india_ndap/ndap/test_data/India_LifeExpectancy.tmcf @@ -0,0 +1,7 @@ +Node: E:India_LifeExpectancy->E0 +observationAbout: C:India_LifeExpectancy->observationAbout +observationDate: C:India_LifeExpectancy->observationDate +value: C:India_LifeExpectancy->value +variableMeasured: C:India_LifeExpectancy->variableMeasured +unit: C:India_LifeExpectancy->unit +typeOf: dcs:StatVarObservation diff --git a/statvar_imports/india_ndap/ndap/test_data/India_LifeExpectancy_input.csv b/statvar_imports/india_ndap/ndap/test_data/India_LifeExpectancy_input.csv new file mode 100644 index 0000000000..d1d1db809a --- /dev/null +++ b/statvar_imports/india_ndap/ndap/test_data/India_LifeExpectancy_input.csv @@ -0,0 +1,115 @@ +srcStateName,srcYear,GENDER,Life Expectancy,YearCode,Year +Jammu And Kashmir,1995,Total,,1995,"Calendar Year (Jan - Dec), 1995" +Jammu And Kashmir,1997,Female,,1997,"Calendar Year (Jan - Dec), 1997" +Jammu And Kashmir,1997,Male,,1997,"Calendar Year (Jan - Dec), 1997" +Jammu And Kashmir,1997,Total,,1997,"Calendar Year (Jan - Dec), 1997" +Jammu And Kashmir,1998,Female,,1998,"Calendar Year (Jan - Dec), 1998" +Jammu And Kashmir,1998,Male,,1998,"Calendar Year (Jan - Dec), 1998" +Jammu And Kashmir,1998,Total,,1998,"Calendar Year (Jan - Dec), 1998" +Jammu And Kashmir,1999,Female,,1999,"Calendar Year (Jan - Dec), 1999" +Jammu And Kashmir,1999,Male,,1999,"Calendar Year (Jan - Dec), 1999" +Jammu And Kashmir,1999,Total,,1999,"Calendar Year (Jan - Dec), 1999" +Jammu And Kashmir,2000,Female,,2000,"Calendar Year (Jan - Dec), 2000" +Jammu And Kashmir,2000,Male,,2000,"Calendar Year (Jan - Dec), 2000" +Jammu And Kashmir,2000,Total,,2000,"Calendar Year (Jan - Dec), 2000" +Jammu And Kashmir,2001,Female,,2001,"Calendar Year (Jan - Dec), 2001" +Jammu And Kashmir,2001,Male,,2001,"Calendar Year (Jan - Dec), 2001" +Jammu And Kashmir,2001,Total,,2001,"Calendar Year (Jan - Dec), 2001" +Jammu And Kashmir,2002,Female,67.3,2002,"Calendar Year (Jan - Dec), 2002" +Jammu And Kashmir,2002,Male,64.7,2002,"Calendar Year (Jan - Dec), 2002" +Jammu And Kashmir,2002,Total,65.9,2002,"Calendar Year (Jan - Dec), 2002" +Jammu And Kashmir,2003,Female,68.0,2003,"Calendar Year (Jan - Dec), 2003" +Jammu And Kashmir,2003,Male,64.4,2003,"Calendar Year (Jan - Dec), 2003" +Jammu And Kashmir,2003,Total,66.0,2003,"Calendar Year (Jan - Dec), 2003" +Jammu And Kashmir,2004,Female,68.9,2004,"Calendar Year (Jan - Dec), 2004" +Jammu And Kashmir,2004,Male,65.9,2004,"Calendar Year (Jan - Dec), 2004" +Jammu And Kashmir,2004,Total,67.3,2004,"Calendar Year (Jan - Dec), 2004" +Jammu And Kashmir,2005,Female,69.9,2005,"Calendar Year (Jan - Dec), 2005" +Jammu And Kashmir,2005,Male,67.1,2005,"Calendar Year (Jan - Dec), 2005" +Jammu And Kashmir,2005,Total,68.4,2005,"Calendar Year (Jan - Dec), 2005" +Jammu And Kashmir,2006,Female,71.0,2006,"Calendar Year (Jan - Dec), 2006" +Jammu And Kashmir,2006,Male,69.2,2006,"Calendar Year (Jan - Dec), 2006" +Jammu And Kashmir,2006,Total,70.0,2006,"Calendar Year (Jan - Dec), 2006" +Jammu And Kashmir,2007,Female,70.8,2007,"Calendar Year (Jan - Dec), 2007" +Jammu And Kashmir,2007,Male,68.6,2007,"Calendar Year (Jan - Dec), 2007" +Jammu And Kashmir,2007,Total,69.6,2007,"Calendar Year (Jan - Dec), 2007" +Uttarakhand,1995,Total,,1995,"Calendar Year (Jan - Dec), 1995" +Jammu And Kashmir,2008,Female,70.6,2008,"Calendar Year (Jan - Dec), 2008" +Jammu And Kashmir,2008,Male,69.0,2008,"Calendar Year (Jan - Dec), 2008" +Jammu And Kashmir,2008,Total,69.8,2008,"Calendar Year (Jan - Dec), 2008" +Jammu And Kashmir,2009,Female,71.2,2009,"Calendar Year (Jan - Dec), 2009" +Jammu And Kashmir,2009,Male,68.9,2009,"Calendar Year (Jan - Dec), 2009" +Jammu And Kashmir,2009,Total,70.0,2009,"Calendar Year (Jan - Dec), 2009" +Jammu And Kashmir,2010,Female,71.1,2010,"Calendar Year (Jan - Dec), 2010" +Jammu And Kashmir,2010,Male,69.2,2010,"Calendar Year (Jan - Dec), 2010" +Jammu And Kashmir,2010,Total,70.1,2010,"Calendar Year (Jan - Dec), 2010" +Jammu And Kashmir,2011,Female,71.9,2011,"Calendar Year (Jan - Dec), 2011" +Jammu And Kashmir,2011,Male,69.4,2011,"Calendar Year (Jan - Dec), 2011" +Jammu And Kashmir,2011,Total,70.5,2011,"Calendar Year (Jan - Dec), 2011" +Jammu And Kashmir,2012,Female,72.4,2012,"Calendar Year (Jan - Dec), 2012" +Jammu And Kashmir,2012,Male,69.9,2012,"Calendar Year (Jan - Dec), 2012" +Jammu And Kashmir,2012,Total,71.0,2012,"Calendar Year (Jan - Dec), 2012" +Jammu And Kashmir,2013,Female,74.0,2013,"Calendar Year (Jan - Dec), 2013" +Jammu And Kashmir,2013,Male,70.6,2013,"Calendar Year (Jan - Dec), 2013" +Jammu And Kashmir,2013,Total,72.0,2013,"Calendar Year (Jan - Dec), 2013" +Jammu And Kashmir,2014,Female,74.9,2014,"Calendar Year (Jan - Dec), 2014" +Jammu And Kashmir,2014,Male,70.9,2014,"Calendar Year (Jan - Dec), 2014" +Jammu And Kashmir,2014,Total,72.6,2014,"Calendar Year (Jan - Dec), 2014" +Jammu And Kashmir,2015,Female,76.1,2015,"Calendar Year (Jan - Dec), 2015" +Jammu And Kashmir,2015,Male,71.2,2015,"Calendar Year (Jan - Dec), 2015" +Jammu And Kashmir,2015,Total,73.2,2015,"Calendar Year (Jan - Dec), 2015" +Jammu And Kashmir,2016,Female,68.5,2016,"Calendar Year (Jan - Dec), 2016" +Jammu And Kashmir,2016,Male,71.6,2016,"Calendar Year (Jan - Dec), 2016" +Jammu And Kashmir,2016,Total,73.5,2016,"Calendar Year (Jan - Dec), 2016" +Jammu And Kashmir,2017,Female,76.7,2017,"Calendar Year (Jan - Dec), 2017" +Jammu And Kashmir,2017,Male,72.1,2017,"Calendar Year (Jan - Dec), 2017" +Jammu And Kashmir,2017,Total,74.1,2017,"Calendar Year (Jan - Dec), 2017" +Jammu And Kashmir,2018,Female,76.2,2018,"Calendar Year (Jan - Dec), 2018" +Jammu And Kashmir,2018,Male,72.2,2018,"Calendar Year (Jan - Dec), 2018" +Jammu And Kashmir,2018,Total,74.0,2018,"Calendar Year (Jan - Dec), 2018" +Jammu And Kashmir,2019,Female,76.1,2019,"Calendar Year (Jan - Dec), 2019" +Jammu And Kashmir,2019,Male,72.6,2019,"Calendar Year (Jan - Dec), 2019" +Jammu And Kashmir,2019,Total,74.2,2019,"Calendar Year (Jan - Dec), 2019" +Jammu And Kashmir,2020,Female,76.3,2020,"Calendar Year (Jan - Dec), 2020" +Jammu And Kashmir,2020,Male,72.6,2020,"Calendar Year (Jan - Dec), 2020" +Jammu And Kashmir,2020,Total,74.3,2020,"Calendar Year (Jan - Dec), 2020" +Himachal Pradesh,1995,Total,,1995,"Calendar Year (Jan - Dec), 1995" +Himachal Pradesh,1997,Female,65.2,1997,"Calendar Year (Jan - Dec), 1997" +Himachal Pradesh,1997,Male,64.6,1997,"Calendar Year (Jan - Dec), 1997" +Himachal Pradesh,1997,Total,64.9,1997,"Calendar Year (Jan - Dec), 1997" +Himachal Pradesh,1998,Female,65.5,1998,"Calendar Year (Jan - Dec), 1998" +Himachal Pradesh,1998,Male,64.8,1998,"Calendar Year (Jan - Dec), 1998" +Himachal Pradesh,1998,Total,65.2,1998,"Calendar Year (Jan - Dec), 1998" +Himachal Pradesh,1999,Female,67.7,1999,"Calendar Year (Jan - Dec), 1999" +Himachal Pradesh,1999,Male,64.8,1999,"Calendar Year (Jan - Dec), 1999" +Himachal Pradesh,1999,Total,66.2,1999,"Calendar Year (Jan - Dec), 1999" +Himachal Pradesh,2000,Female,68.3,2000,"Calendar Year (Jan - Dec), 2000" +Himachal Pradesh,2000,Male,65.7,2000,"Calendar Year (Jan - Dec), 2000" +Himachal Pradesh,2000,Total,67.0,2000,"Calendar Year (Jan - Dec), 2000" +Himachal Pradesh,2001,Female,69.4,2001,"Calendar Year (Jan - Dec), 2001" +Himachal Pradesh,2001,Male,66.3,2001,"Calendar Year (Jan - Dec), 2001" +Himachal Pradesh,2001,Total,67.8,2001,"Calendar Year (Jan - Dec), 2001" +Uttarakhand,1997,Female,,1997,"Calendar Year (Jan - Dec), 1997" +Himachal Pradesh,2002,Female,70.2,2002,"Calendar Year (Jan - Dec), 2002" +Himachal Pradesh,2002,Male,66.6,2002,"Calendar Year (Jan - Dec), 2002" +Himachal Pradesh,2002,Total,68.3,2002,"Calendar Year (Jan - Dec), 2002" +Himachal Pradesh,2003,Female,71.4,2003,"Calendar Year (Jan - Dec), 2003" +Himachal Pradesh,2003,Male,67.1,2003,"Calendar Year (Jan - Dec), 2003" +Himachal Pradesh,2003,Total,69.1,2003,"Calendar Year (Jan - Dec), 2003" +Himachal Pradesh,2004,Female,72.0,2004,"Calendar Year (Jan - Dec), 2004" +Himachal Pradesh,2004,Male,67.3,2004,"Calendar Year (Jan - Dec), 2004" +Himachal Pradesh,2004,Total,69.5,2004,"Calendar Year (Jan - Dec), 2004" +Himachal Pradesh,2005,Female,71.8,2005,"Calendar Year (Jan - Dec), 2005" +Himachal Pradesh,2005,Male,67.4,2005,"Calendar Year (Jan - Dec), 2005" +Himachal Pradesh,2005,Total,69.5,2005,"Calendar Year (Jan - Dec), 2005" +Himachal Pradesh,2006,Female,72.0,2006,"Calendar Year (Jan - Dec), 2006" +Himachal Pradesh,2006,Male,67.4,2006,"Calendar Year (Jan - Dec), 2006" +Himachal Pradesh,2006,Total,69.6,2006,"Calendar Year (Jan - Dec), 2006" +Himachal Pradesh,2007,Female,72.2,2007,"Calendar Year (Jan - Dec), 2007" +Himachal Pradesh,2007,Male,67.8,2007,"Calendar Year (Jan - Dec), 2007" +Himachal Pradesh,2007,Total,69.9,2007,"Calendar Year (Jan - Dec), 2007" +Himachal Pradesh,2008,Female,72.1,2008,"Calendar Year (Jan - Dec), 2008" +Himachal Pradesh,2008,Male,67.7,2008,"Calendar Year (Jan - Dec), 2008" +Himachal Pradesh,2008,Total,69.8,2008,"Calendar Year (Jan - Dec), 2008" +Himachal Pradesh,2009,Female,72.0,2009,"Calendar Year (Jan - Dec), 2009" +Himachal Pradesh,2009,Male,67.7,2009,"Calendar Year (Jan - Dec), 2009" \ No newline at end of file From a513f2ca2d2253120857688a36afe9768b4bc0a2 Mon Sep 17 00:00:00 2001 From: Chandrashekar Harish Date: Fri, 13 Jun 2025 17:53:53 +0000 Subject: [PATCH 3/7] reverted to comments --- .../india_ndap/ndap/download_script.py | 51 +++++++------------ statvar_imports/india_ndap/ndap/manifest.json | 3 ++ 2 files changed, 20 insertions(+), 34 deletions(-) diff --git a/statvar_imports/india_ndap/ndap/download_script.py b/statvar_imports/india_ndap/ndap/download_script.py index 277d01a687..a019c43ec6 100644 --- a/statvar_imports/india_ndap/ndap/download_script.py +++ b/statvar_imports/india_ndap/ndap/download_script.py @@ -1,44 +1,23 @@ +import sys import requests import pandas as pd import config import os +from absl import logging, app from retry import retry -"logging configuration " -import logging -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) -handler = logging.StreamHandler() -formatter = logging.Formatter('%(message)s') -handler.setFormatter(formatter) -logger.addHandler(handler) - -@retry(tries=3, delay=5, backoff=2) -def retry_method(url, headers=None): - response = requests.get(url, headers=headers, timeout=10) - response.raise_for_status() - return response -def get_api_response(api_url): - """Fetches data from the API and returns the JSON response.""" - try: - response = retry_method(api_url) - response.raise_for_status() - return response.json() - except requests.exceptions.RequestException as e: - logger.exception(f"Error fetching data from API: {e}") - return None - except ValueError as e: - logger.exception(f"Invalid JSON response: {e}. Response text was: {response.text}") - return None - except Exception as e: - logger.exception(f"A general error occurred: {e}") - return None -if __name__=="__main__": +_SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.join(_SCRIPT_PATH, '../../../util/')) +from download_util_script import _retry_method + + +def main(_): all_data = [] for page_num in range(1, 3): api_url = f"{config.url}&pageno={page_num}" - response_data = get_api_response(api_url) + response = _retry_method(api_url, None, 3, 5,2) + response_data = response.json() if response_data: keys = list(response_data.values())[5] @@ -47,7 +26,7 @@ def get_api_response(api_url): a = a=i['StateName'],i['Year'].split(",")[-1].strip(),i['GENDER'],i['I7375_4']['TotalPopulationWeight'],i['Year'].split(",")[-1].strip(),i['Year'] all_data.append(a) else: - logger.info(f"Failed to retrieve data from page {page_num}. Stopping.") + logging.info(f"Failed to retrieve data from page {page_num}. Stopping.") break if all_data: @@ -55,7 +34,11 @@ def get_api_response(api_url): os.makedirs(config.input_files, exist_ok=True) input_filename = os.path.join(config.input_files, 'India_LifeExpectancy_input.csv') df.to_csv(input_filename, index=False) - logger.info("Data saved to India_LifeExpectancy_input.csv") + logging.info("Data saved to India_LifeExpectancy_input.csv") else: - logger.info("No data was retrieved from the API.") + logging.info("No data was retrieved from the API.") + +if __name__=="__main__": + app.run(main) + diff --git a/statvar_imports/india_ndap/ndap/manifest.json b/statvar_imports/india_ndap/ndap/manifest.json index bc0b9524d0..822bc405ec 100644 --- a/statvar_imports/india_ndap/ndap/manifest.json +++ b/statvar_imports/india_ndap/ndap/manifest.json @@ -17,6 +17,9 @@ "cleaned_csv": "output/Life_expectancy.csv" } ], + "source_files": [ + "input_files/India_LifeExpectancy_input.csv" + ], "cron_schedule": "00 11 1,15 * *" } ] From 5dd18618a87c6c07ddbea7380179d85038f8317e Mon Sep 17 00:00:00 2001 From: Chandrashekar Harish Date: Tue, 17 Jun 2025 05:21:05 +0000 Subject: [PATCH 4/7] reverted to comments --- statvar_imports/india_ndap/ndap/download_script.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/statvar_imports/india_ndap/ndap/download_script.py b/statvar_imports/india_ndap/ndap/download_script.py index a019c43ec6..14b9d182c2 100644 --- a/statvar_imports/india_ndap/ndap/download_script.py +++ b/statvar_imports/india_ndap/ndap/download_script.py @@ -1,10 +1,8 @@ import sys -import requests import pandas as pd import config import os from absl import logging, app -from retry import retry _SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__)) sys.path.append(os.path.join(_SCRIPT_PATH, '../../../util/')) @@ -17,6 +15,7 @@ def main(_): for page_num in range(1, 3): api_url = f"{config.url}&pageno={page_num}" response = _retry_method(api_url, None, 3, 5,2) + response_data = response.json() if response_data: From e06fc6459aef904bbf5f0f3f4629bb515272056c Mon Sep 17 00:00:00 2001 From: Chandrashekar Harish Date: Tue, 17 Jun 2025 12:14:42 +0000 Subject: [PATCH 5/7] reverted to comments --- statvar_imports/india_ndap/ndap/download_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/statvar_imports/india_ndap/ndap/download_script.py b/statvar_imports/india_ndap/ndap/download_script.py index 14b9d182c2..b22845ea6e 100644 --- a/statvar_imports/india_ndap/ndap/download_script.py +++ b/statvar_imports/india_ndap/ndap/download_script.py @@ -25,7 +25,7 @@ def main(_): a = a=i['StateName'],i['Year'].split(",")[-1].strip(),i['GENDER'],i['I7375_4']['TotalPopulationWeight'],i['Year'].split(",")[-1].strip(),i['Year'] all_data.append(a) else: - logging.info(f"Failed to retrieve data from page {page_num}. Stopping.") + logging.fatal(f"Failed to retrieve data from page {page_num}. Stopping.") break if all_data: From 4c05c2fb38c3e43d12cc31bd4e84380bd7692f9f Mon Sep 17 00:00:00 2001 From: Chandrashekar Harish Date: Thu, 19 Jun 2025 08:49:49 +0000 Subject: [PATCH 6/7] reverted to comments --- .../india_ndap/ndap/download_script.py | 79 ++++++++++++++----- 1 file changed, 59 insertions(+), 20 deletions(-) diff --git a/statvar_imports/india_ndap/ndap/download_script.py b/statvar_imports/india_ndap/ndap/download_script.py index b22845ea6e..3171eca377 100644 --- a/statvar_imports/india_ndap/ndap/download_script.py +++ b/statvar_imports/india_ndap/ndap/download_script.py @@ -1,8 +1,31 @@ -import sys -import pandas as pd -import config +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 20 ('License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# How to run the script to download the files: +# python3 download_script.py +import json import os -from absl import logging, app +import pandas as pd +import sys +from absl import app +from absl import flags +from absl import logging +from google.cloud import storage +flags.DEFINE_string( + 'config_file_path', + 'gs://datcom-import-test/statvar_imports/india_ndap/ndap/config.json', + 'Input directory where config files downloaded.') _SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__)) sys.path.append(os.path.join(_SCRIPT_PATH, '../../../util/')) @@ -11,33 +34,49 @@ def main(_): all_data = [] - - for page_num in range(1, 3): - api_url = f"{config.url}&pageno={page_num}" - response = _retry_method(api_url, None, 3, 5,2) - + _FLAGS = flags.FLAGS + config_file_path = _FLAGS.config_file_path + storage_client = storage.Client() + bucket_name = config_file_path.split('/')[2] + bucket = storage_client.bucket(bucket_name) + blob_name = '/'.join(config_file_path.split('/')[3:]) + blob = bucket.blob(blob_name) + file_contents = blob.download_as_text() + try: + file_config = json.loads(file_contents) + url = file_config.get('url') + input_files = file_config.get('input_files') + except json.JSONDecodeError: + logging.fatal("Cannot extract url and input files path.") + page_num = 1 + while True: + api_url = f"{url}&pageno={page_num}" + response = _retry_method(api_url, None, 3, 5, 2) response_data = response.json() + if response_data and 'Data' in response_data and len(response_data['Data']) > 0: + keys = response_data['Data'] + try: + if not isinstance(keys, list): + logging.fatal(f"Value for key 'Data' is not a list.") + except KeyError as e: + logging.fatal(f"Missing expected key '{e}' in the API response.") + break - if response_data: - keys = list(response_data.values())[5] - for i in keys: a = a=i['StateName'],i['Year'].split(",")[-1].strip(),i['GENDER'],i['I7375_4']['TotalPopulationWeight'],i['Year'].split(",")[-1].strip(),i['Year'] all_data.append(a) + page_num += 1 else: - logging.fatal(f"Failed to retrieve data from page {page_num}. Stopping.") - break - + logging.error(f"failed to retrieve data from page {page_num}") + break if all_data: df = pd.DataFrame(all_data, columns=['srcStateName', 'srcYear', 'GENDER', 'Life Expectancy', 'YearCode', 'Year']) - os.makedirs(config.input_files, exist_ok=True) - input_filename = os.path.join(config.input_files, 'India_LifeExpectancy_input.csv') + os.makedirs(input_files, exist_ok=True) + input_filename = os.path.join(input_files, 'India_LifeExpectancy_input.csv') df.to_csv(input_filename, index=False) logging.info("Data saved to India_LifeExpectancy_input.csv") else: logging.info("No data was retrieved from the API.") if __name__=="__main__": - app.run(main) - - + app.run(main) \ No newline at end of file From 15823fdc32e93f578a10cf157c9f05fbd7ae1da8 Mon Sep 17 00:00:00 2001 From: Chandrashekar Harish Date: Thu, 19 Jun 2025 08:53:51 +0000 Subject: [PATCH 7/7] reverted to comments --- statvar_imports/india_ndap/ndap/download_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/statvar_imports/india_ndap/ndap/download_script.py b/statvar_imports/india_ndap/ndap/download_script.py index 3171eca377..fba0242ef2 100644 --- a/statvar_imports/india_ndap/ndap/download_script.py +++ b/statvar_imports/india_ndap/ndap/download_script.py @@ -61,7 +61,7 @@ def main(_): except KeyError as e: logging.fatal(f"Missing expected key '{e}' in the API response.") break - + # Considering the table id I7375_4 which is specific to the import. for i in keys: a = a=i['StateName'],i['Year'].split(",")[-1].strip(),i['GENDER'],i['I7375_4']['TotalPopulationWeight'],i['Year'].split(",")[-1].strip(),i['Year'] all_data.append(a)