nick/753-Data-Sync
1
0
Fork 0
Code Issues 4 Pull Requests Releases 2 Activity

feat: Add --test flag to fetch only the first page of results

Browse Source 
- Introduced a new command-line argument `--test` to limit data fetching to the first page.
- Updated `parse_arguments` function to include the `--test` flag.
- Modified the pagination logic in the `main` function to break after the first page when `--test` is enabled.
- Enhanced documentation to reflect the new functionality and usage examples.
This commit is contained in:
Nick Heppler 2025-05-22 10:27:36 -04:00
parent 364e365e9c
commit b7ca534211
1 changed files with 11 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
app.py
View File

@ -146,10 +146,13 @@ def parse_arguments():
# Add arguments for results per page # Add arguments for results per page
parser.add_argument('--results_per_page', type=int, default=100, help="Number of results per page (default: 100)") parser.add_argument('--results_per_page', type=int, default=100, help="Number of results per page (default: 100)")
# Add a test flag
parser.add_argument('--test', action='store_true', help="If set, only fetch the first page of results.")
# Parse the arguments # Parse the arguments
args = parser.parse_args() args = parser.parse_args()
return args.results_per_page return args.results_per_page, args.test
def generate_token(username, password, url="https://www.arcgis.com/sharing/rest/generateToken"): def generate_token(username, password, url="https://www.arcgis.com/sharing/rest/generateToken"):
"""Generates an authentication token.""" """Generates an authentication token."""
@ -305,8 +308,8 @@ def main():
purge_old_files(purge_days) purge_old_files(purge_days)
# Parse command-line arguments # Parse command-line arguments
results_per_page = parse_arguments() results_per_page, test_mode = parse_arguments()
logger.info(f"Parsed arguments: results_per_page={results_per_page}") logger.info(f"Parsed arguments: results_per_page={results_per_page}, test_mode={test_mode}")
# Load environment variables # Load environment variables
logger.info("Loading environment variables.") logger.info("Loading environment variables.")
@ -356,6 +359,11 @@ def main():
logger.info("No more data to fetch, stopping pagination.") logger.info("No more data to fetch, stopping pagination.")
break break
# Break the loop if in test mode
if test_mode:
logger.info("Test mode is enabled, stopping after the first page.")
break
page_number += 1 page_number += 1
except Exception as e: except Exception as e:
logger.error(f"Error fetching or saving data for page {page_number}: {e}", exc_info=True) logger.error(f"Error fetching or saving data for page {page_number}: {e}", exc_info=True)