Agencies, regions

amarillo-gtfs-generator/gtfs_generator.py

@@ -12,6 +12,7 @@ import schedule
 import threading
 import time
 import logging
+import os
 from watchdog.observers import Observer
 from watchdog.events import FileSystemEventHandler
 from .models.Carpool import Carpool, Region
@@ -19,8 +20,8 @@ from .router import _assert_region_exists
 from .services import stops #TODO: make stop service its own package??
 from .services.trips import TripStore, Trip
 from .services.carpools import CarpoolService
-from amarillo.services.agencies import AgencyService
+from .services.agencies import AgencyService
-from amarillo.services.regions import RegionService
+from .services.regions import RegionService
 from amarillo.utils.utils import agency_carpool_ids_from_filename
 
 
@@ -53,6 +54,11 @@ class EventHandler(FileSystemEventHandler):
 
 
 def init():
+	logger.info(f"Current working directory is {os.path.abspath(os.getcwd())}")
+	if not os.path.isdir('data/agency'):
+		logger.error(f'{os.path.abspath("data/agency")} directory does not exist')
+	
+
 	container['agencies'] = AgencyService()
 	logger.info("Loaded %d agencies", len(container['agencies'].agencies))
 
@@ -76,12 +82,13 @@ def init():
 	logger.info("Restore carpools...")
 
 	for agency_id in container['agencies'].agencies:
-		for carpool_file_name in glob(f'data/carpool/{agency_id}/*.json'):
+		for carpool_file_name in glob(f'data/enhanced/{agency_id}/*.json'):
 			try:
 				with open(carpool_file_name) as carpool_file:
 					carpool = Carpool(**(json.load(carpool_file)))
 					#TODO: convert to trip and add to tripstore directly
 					container['carpools'].put(carpool.agency, carpool.id, carpool)
+					logger.info(f"Restored carpool {carpool_file_name}")
 			except Exception as e:
 				logger.warning("Issue during restore of carpool %s: %s", carpool_file_name, repr(e))
 
@@ -90,21 +97,21 @@ def init():
 	observer.schedule(EventHandler(), 'data/enhanced', recursive=True)
 	observer.start()
 
-def run_schedule():
+# def run_schedule():
 
-	while 1:
+# 	while 1:
-		try:
+# 		try:
-			schedule.run_pending()
+# 			schedule.run_pending()
-		except Exception as e:
+# 		except Exception as e:
-			logger.exception(e)
+# 			logger.exception(e)
-		time.sleep(1)
+# 		time.sleep(1)
 
-def midnight():
+# def midnight():
-	container['stops_store'].load_stop_sources()
+# 	container['stops_store'].load_stop_sources()
-	# container['trips_store'].unflag_unrecent_updates()
+# 	# container['trips_store'].unflag_unrecent_updates()
-	# container['carpools'].purge_outdated_offers()
+# 	# container['carpools'].purge_outdated_offers()
 
-	generate_gtfs()
+# 	generate_gtfs()
 
 #TODO: generate for a specific region only
 #TODO: what happens when there are no trips?
@@ -128,20 +135,20 @@ def generate_gtfs_rt():
 	for region in container['regions'].regions.values():
 		rt = producer.export_feed(time.time(), f"data/gtfs/amarillo.{region.id}.gtfsrt", bbox=region.bbox)
 
-def start_schedule():
+# def start_schedule():
-	schedule.every().day.at("00:00").do(midnight)
+# 	# schedule.every().day.at("00:00").do(midnight)
-	schedule.every(60).seconds.do(generate_gtfs_rt)
+# 	schedule.every(60).seconds.do(generate_gtfs_rt)
-	# Create all feeds once at startup
+# 	# Create all feeds once at startup
-	schedule.run_all()
+# 	schedule.run_all()
-	job_thread = threading.Thread(target=run_schedule, daemon=True)
+# 	job_thread = threading.Thread(target=run_schedule, daemon=True)
-	job_thread.start()
+# 	job_thread.start()
 
-def setup(app : FastAPI):
+# def setup(app : FastAPI):
-	# TODO: Create all feeds once at startup
+# 	# TODO: Create all feeds once at startup
-	# configure_enhancer_services()
+# 	# configure_enhancer_services()
-	# app.include_router(router)
+# 	# app.include_router(router)
-	# start_schedule()
+# 	# start_schedule()
-	pass
+# 	pass
 
 logging.config.fileConfig('logging.conf', disable_existing_loggers=False)
 logger = logging.getLogger("gtfs-generator")
@@ -243,9 +250,9 @@ async def get_file(region_id: str):
 	# verify_permission("gtfs", requesting_user)
 	return FileResponse(f'data/gtfs/amarillo.{region_id}.gtfs.zip')
 
-@app.get("/region/{region_id}/grfs-rt/",
+@app.get("/region/{region_id}/gtfs-rt/",
-    summary="Return GRFS-RT Feed for this region",
+    summary="Return GTFS-RT Feed for this region",
-    response_description="GRFS-RT-Feed",
+    response_description="GTFS-RT-Feed",
     response_class=FileResponse,
     responses={
                 status.HTTP_404_NOT_FOUND: {"description": "Region not found"},
@@ -256,20 +263,20 @@ async def get_file(region_id: str, format: str = 'protobuf'):
     generate_gtfs_rt()
     _assert_region_exists(region_id)
     if format == 'json':
-        return FileResponse(f'data/grfs/amarillo.{region_id}.gtfsrt.json')
+        return FileResponse(f'data/gtfs/amarillo.{region_id}.gtfsrt.json')
     elif format == 'protobuf':
-        return FileResponse(f'data/grfs/amarillo.{region_id}.gtfsrt.pbf')
+        return FileResponse(f'data/gtfs/amarillo.{region_id}.gtfsrt.pbf')
     else:
         message = "Specified format is not supported, i.e. neither protobuf nor json."
         raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=message)
 
 #TODO: sync endpoint that calls midnight
 
-@app.post("/sync",
+# @app.post("/sync",
-	operation_id="sync")
+# 	operation_id="sync")
-#TODO: add examples
+# #TODO: add examples
-async def post_sync():
+# async def post_sync():
 
-	logger.info(f"Sync")
+# 	logger.info(f"Sync")
 
-	midnight()
+# 	midnight()

amarillo-gtfs-generator/services/agencies.py

@@ -0,0 +1,24 @@
+import json
+from glob import glob
+from typing import Dict
+
+from amarillo.models.Carpool import Agency
+
+# TODO FG HB this service should also listen to pyinotify
+# because the (updated) agencies are needed in the enhancer
+# as well.
+
+class AgencyService:
+
+    def __init__(self):
+        self.agencies: Dict[str, Agency] = {}
+        for agency_file_name in glob('data/agency/*.json'):
+            with open(agency_file_name) as agency_file:
+                dict = json.load(agency_file)
+                agency = Agency(**dict)
+                agency_id = agency.id
+                self.agencies[agency_id] = agency
+
+    def get_agency(self, agency_id: str) -> Agency:
+        agency = self.agencies.get(agency_id)
+        return agency

amarillo-gtfs-generator/services/regions.py

@@ -0,0 +1,21 @@
+import json
+from glob import glob
+from typing import Dict
+
+from amarillo.models.Carpool import Region
+
+
+class RegionService:
+
+    def __init__(self):
+        self.regions: Dict[str, Region] = {}
+        for region_file_name in glob('data/region/*.json'):
+            with open(region_file_name) as region_file:
+                dict = json.load(region_file)
+                region = Region(**dict)
+                region_id = region.id
+                self.regions[region_id] = region
+
+    def get_region(self, region_id: str) -> Region:
+        region = self.regions.get(region_id)
+        return region