From cad8d64c23f9ac8479c8eae67eb63e8416259aef Mon Sep 17 00:00:00 2001 From: Nick Heppler Date: Thu, 22 May 2025 11:25:41 -0400 Subject: [PATCH 1/2] feat: Add --reload flag to data synchronization script for loading JSON data - Introduced a new command-line argument `--reload` to allow loading data from a specified JSON file. - Updated `parse_arguments` to include the `--reload` flag. - Modified `main` function to truncate the feature layer and load data from the JSON file when the flag is used. - Enhanced documentation to reflect the new functionality and usage examples. --- README.md | 5 +++-- app.py | 24 +++++++++++++++++++----- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 90cf852..7c5b459 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ ## ๐Ÿš€ Overview -This script fetches enforcement data from an external API, truncates a specified feature layer in ArcGIS, and adds the fetched data as features to the layer. It also logs the operation, saves data to JSON files, and optionally purges old files. +This script fetches enforcement data from an external API, truncates a specified feature layer in ArcGIS, and adds the fetched data as features to the layer. It also logs the operation, saves data to JSON files, and optionally purges old files. Additionally, it supports reloading data from a JSON file without making API calls. --- @@ -88,6 +88,7 @@ python 753DataSync.py --results_per_page 100 |----------------------|---------------------------------------------| | `--results_per_page` | Optional. Number of results per API call (default: `100`) | | `--test` | Optional. If set, only fetch the first page of results. | +| `--reload` | Optional. Load data from a specified JSON file instead of fetching from the API. | --- @@ -101,6 +102,7 @@ python 753DataSync.py --results_per_page 100 6. **๐Ÿงน File Cleanup** โ€” Deletes `.json`/`.log` files older than `PURGE_DAYS`. 7. **๐Ÿ“‘ Dynamic Logs** โ€” Logs saved to `753DataSync_YYYY-MM-DD.log`. 8. **๐Ÿงช Test Mode** โ€” Use the `--test` flag to fetch only the first page of results for testing purposes. +9. **๐Ÿ”„ Reload Data** โ€” Use the `--reload` flag to truncate the feature layer and load data from a specified JSON file. --- @@ -114,7 +116,6 @@ python 753DataSync.py --results_per_page 100 ๐Ÿ“„ 753DataSync_2025-03-26.log ``` - --- ## ๐Ÿ“ Example Log diff --git a/app.py b/app.py index 5b5dfe5..cff11b2 100644 --- a/app.py +++ b/app.py @@ -149,10 +149,13 @@ def parse_arguments(): # Add a test flag parser.add_argument('--test', action='store_true', help="If set, only fetch the first page of results.") + # Add a reload flag + parser.add_argument('--reload', type=str, help="If set, load data from the specified file instead of fetching from the API.") + # Parse the arguments args = parser.parse_args() - return args.results_per_page, args.test + return args.results_per_page, args.test, args.reload def generate_token(username, password, url="https://www.arcgis.com/sharing/rest/generateToken"): """Generates an authentication token.""" @@ -308,8 +311,8 @@ def main(): purge_old_files(purge_days) # Parse command-line arguments - results_per_page, test_mode = parse_arguments() - logger.info(f"Parsed arguments: results_per_page={results_per_page}, test_mode={test_mode}") + results_per_page, test_mode, reload_file = parse_arguments() + logger.info(f"Parsed arguments: results_per_page={results_per_page}, test_mode={test_mode}, reload_file={reload_file}") # Load environment variables logger.info("Loading environment variables.") @@ -333,8 +336,19 @@ def main(): fs = os.getenv('FS') layer = os.getenv('LAYER') - # Truncate the layer before adding new features - truncate(token, hostname, instance, fs, layer) + # If --reload flag is set, load data from the specified file + if reload_file: + logger.info(f"Reloading data from file: {reload_file}") + truncate(token, hostname, instance, fs, layer) # Truncate the layer + + # Load data from the specified file + with open(reload_file, 'r', encoding='utf-8') as f: + aggregated_data = json.load(f) + + # Add the features to the feature layer + response = add_features(token, hostname, instance, fs, layer, aggregated_data) + logger.info("Data reloaded successfully from the specified file.") + return all_data = [] page_number = 1 From 3afe292b23924c40564290cf82acb2723c861060 Mon Sep 17 00:00:00 2001 From: Nick Heppler Date: Thu, 22 May 2025 12:12:01 -0400 Subject: [PATCH 2/2] fix: Ensure feature layer is truncated at the start of script execution - Moved the call to the truncate function to always execute at the beginning of the main function. - Ensured that the feature layer is cleared before loading data from a JSON file or fetching from the API. --- app.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app.py b/app.py index cff11b2..366e5ef 100644 --- a/app.py +++ b/app.py @@ -336,10 +336,12 @@ def main(): fs = os.getenv('FS') layer = os.getenv('LAYER') + logger.info("Truncating the feature layer.") + truncate(token, hostname, instance, fs, layer) + # If --reload flag is set, load data from the specified file if reload_file: logger.info(f"Reloading data from file: {reload_file}") - truncate(token, hostname, instance, fs, layer) # Truncate the layer # Load data from the specified file with open(reload_file, 'r', encoding='utf-8') as f: