From aaa9aa4e09c60377b9b4409bfdc12e39a5c08c80 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Thu, 9 May 2024 16:23:51 +0200 Subject: [PATCH] Agencies, regions --- amarillo-gtfs-generator/gtfs_generator.py | 85 +++++++++++--------- amarillo-gtfs-generator/services/agencies.py | 24 ++++++ amarillo-gtfs-generator/services/regions.py | 21 +++++ 3 files changed, 91 insertions(+), 39 deletions(-) create mode 100644 amarillo-gtfs-generator/services/agencies.py create mode 100644 amarillo-gtfs-generator/services/regions.py diff --git a/amarillo-gtfs-generator/gtfs_generator.py b/amarillo-gtfs-generator/gtfs_generator.py index de35a5e..8a90fee 100644 --- a/amarillo-gtfs-generator/gtfs_generator.py +++ b/amarillo-gtfs-generator/gtfs_generator.py @@ -12,6 +12,7 @@ import schedule import threading import time import logging +import os from watchdog.observers import Observer from watchdog.events import FileSystemEventHandler from .models.Carpool import Carpool, Region @@ -19,8 +20,8 @@ from .router import _assert_region_exists from .services import stops #TODO: make stop service its own package?? from .services.trips import TripStore, Trip from .services.carpools import CarpoolService -from amarillo.services.agencies import AgencyService -from amarillo.services.regions import RegionService +from .services.agencies import AgencyService +from .services.regions import RegionService from amarillo.utils.utils import agency_carpool_ids_from_filename @@ -53,6 +54,11 @@ class EventHandler(FileSystemEventHandler): def init(): + logger.info(f"Current working directory is {os.path.abspath(os.getcwd())}") + if not os.path.isdir('data/agency'): + logger.error(f'{os.path.abspath("data/agency")} directory does not exist') + + container['agencies'] = AgencyService() logger.info("Loaded %d agencies", len(container['agencies'].agencies)) @@ -76,12 +82,13 @@ def init(): logger.info("Restore carpools...") for agency_id in container['agencies'].agencies: - for carpool_file_name in glob(f'data/carpool/{agency_id}/*.json'): + for carpool_file_name in glob(f'data/enhanced/{agency_id}/*.json'): try: with open(carpool_file_name) as carpool_file: carpool = Carpool(**(json.load(carpool_file))) #TODO: convert to trip and add to tripstore directly container['carpools'].put(carpool.agency, carpool.id, carpool) + logger.info(f"Restored carpool {carpool_file_name}") except Exception as e: logger.warning("Issue during restore of carpool %s: %s", carpool_file_name, repr(e)) @@ -90,21 +97,21 @@ def init(): observer.schedule(EventHandler(), 'data/enhanced', recursive=True) observer.start() -def run_schedule(): +# def run_schedule(): - while 1: - try: - schedule.run_pending() - except Exception as e: - logger.exception(e) - time.sleep(1) +# while 1: +# try: +# schedule.run_pending() +# except Exception as e: +# logger.exception(e) +# time.sleep(1) -def midnight(): - container['stops_store'].load_stop_sources() - # container['trips_store'].unflag_unrecent_updates() - # container['carpools'].purge_outdated_offers() +# def midnight(): +# container['stops_store'].load_stop_sources() +# # container['trips_store'].unflag_unrecent_updates() +# # container['carpools'].purge_outdated_offers() - generate_gtfs() +# generate_gtfs() #TODO: generate for a specific region only #TODO: what happens when there are no trips? @@ -128,20 +135,20 @@ def generate_gtfs_rt(): for region in container['regions'].regions.values(): rt = producer.export_feed(time.time(), f"data/gtfs/amarillo.{region.id}.gtfsrt", bbox=region.bbox) -def start_schedule(): - schedule.every().day.at("00:00").do(midnight) - schedule.every(60).seconds.do(generate_gtfs_rt) - # Create all feeds once at startup - schedule.run_all() - job_thread = threading.Thread(target=run_schedule, daemon=True) - job_thread.start() +# def start_schedule(): +# # schedule.every().day.at("00:00").do(midnight) +# schedule.every(60).seconds.do(generate_gtfs_rt) +# # Create all feeds once at startup +# schedule.run_all() +# job_thread = threading.Thread(target=run_schedule, daemon=True) +# job_thread.start() -def setup(app : FastAPI): - # TODO: Create all feeds once at startup - # configure_enhancer_services() - # app.include_router(router) - # start_schedule() - pass +# def setup(app : FastAPI): +# # TODO: Create all feeds once at startup +# # configure_enhancer_services() +# # app.include_router(router) +# # start_schedule() +# pass logging.config.fileConfig('logging.conf', disable_existing_loggers=False) logger = logging.getLogger("gtfs-generator") @@ -243,9 +250,9 @@ async def get_file(region_id: str): # verify_permission("gtfs", requesting_user) return FileResponse(f'data/gtfs/amarillo.{region_id}.gtfs.zip') -@app.get("/region/{region_id}/grfs-rt/", - summary="Return GRFS-RT Feed for this region", - response_description="GRFS-RT-Feed", +@app.get("/region/{region_id}/gtfs-rt/", + summary="Return GTFS-RT Feed for this region", + response_description="GTFS-RT-Feed", response_class=FileResponse, responses={ status.HTTP_404_NOT_FOUND: {"description": "Region not found"}, @@ -256,20 +263,20 @@ async def get_file(region_id: str, format: str = 'protobuf'): generate_gtfs_rt() _assert_region_exists(region_id) if format == 'json': - return FileResponse(f'data/grfs/amarillo.{region_id}.gtfsrt.json') + return FileResponse(f'data/gtfs/amarillo.{region_id}.gtfsrt.json') elif format == 'protobuf': - return FileResponse(f'data/grfs/amarillo.{region_id}.gtfsrt.pbf') + return FileResponse(f'data/gtfs/amarillo.{region_id}.gtfsrt.pbf') else: message = "Specified format is not supported, i.e. neither protobuf nor json." raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=message) #TODO: sync endpoint that calls midnight -@app.post("/sync", - operation_id="sync") -#TODO: add examples -async def post_sync(): +# @app.post("/sync", +# operation_id="sync") +# #TODO: add examples +# async def post_sync(): - logger.info(f"Sync") +# logger.info(f"Sync") - midnight() \ No newline at end of file +# midnight() \ No newline at end of file diff --git a/amarillo-gtfs-generator/services/agencies.py b/amarillo-gtfs-generator/services/agencies.py new file mode 100644 index 0000000..e7450aa --- /dev/null +++ b/amarillo-gtfs-generator/services/agencies.py @@ -0,0 +1,24 @@ +import json +from glob import glob +from typing import Dict + +from amarillo.models.Carpool import Agency + +# TODO FG HB this service should also listen to pyinotify +# because the (updated) agencies are needed in the enhancer +# as well. + +class AgencyService: + + def __init__(self): + self.agencies: Dict[str, Agency] = {} + for agency_file_name in glob('data/agency/*.json'): + with open(agency_file_name) as agency_file: + dict = json.load(agency_file) + agency = Agency(**dict) + agency_id = agency.id + self.agencies[agency_id] = agency + + def get_agency(self, agency_id: str) -> Agency: + agency = self.agencies.get(agency_id) + return agency diff --git a/amarillo-gtfs-generator/services/regions.py b/amarillo-gtfs-generator/services/regions.py new file mode 100644 index 0000000..425b3ac --- /dev/null +++ b/amarillo-gtfs-generator/services/regions.py @@ -0,0 +1,21 @@ +import json +from glob import glob +from typing import Dict + +from amarillo.models.Carpool import Region + + +class RegionService: + + def __init__(self): + self.regions: Dict[str, Region] = {} + for region_file_name in glob('data/region/*.json'): + with open(region_file_name) as region_file: + dict = json.load(region_file) + region = Region(**dict) + region_id = region.id + self.regions[region_id] = region + + def get_region(self, region_id: str) -> Region: + region = self.regions.get(region_id) + return region