Fixed configuration and relative imports

amarillo-enhancer/configuration.py

@@ -5,24 +5,49 @@ import logging
 from glob import glob
 
 from amarillo.models.Carpool import Carpool
-from amarillo.plugins.enhancer.services import stops
-from amarillo.plugins.enhancer.services import trips
-from amarillo.plugins.enhancer.services.carpools import CarpoolService
-from amarillo.services.config import config
-from amarillo.configuration import configure_services
+from .services import stops
+from .services import trips
+from .services.carpools import CarpoolService
+from .services.config import config
+from amarillo.utils.utils import assert_folder_exists
 from .services.trips import TripTransformer
+from amarillo.services.agencies import AgencyService
+from amarillo.services.regions import RegionService
 
 logger = logging.getLogger(__name__)
 
-enhancer_configured = False
+# enhancer_configured = False
+
+def create_required_directories():
+    logger.info("Checking that necessary directories exist")
+    # Folder to serve GTFS(-RT) from
+    assert_folder_exists('data/gtfs')
+    # Temp folder for GTFS generation
+    assert_folder_exists('data/tmp')
+
+    for agency_id in container['agencies'].agencies:
+        for subdir in ['carpool', 'trash', 'enhanced', 'failed']:
+            foldername = f'data/{subdir}/{agency_id}'
+            logger.debug("Checking that necessary %s exist", foldername)
+            assert_folder_exists(f'data/{subdir}/{agency_id}')
+
+
+def configure_services():
+    container['agencies'] = AgencyService()
+    logger.info("Loaded %d agencies", len(container['agencies'].agencies))
+    
+    container['regions'] = RegionService()
+    logger.info("Loaded %d regions", len(container['regions'].regions))
+
+    create_required_directories()
 
 def configure_enhancer_services():
     #Make sure configuration only happens once
-    global enhancer_configured 
+    # global enhancer_configured 
     global transformer
-    if enhancer_configured:
-        logger.info("Enhancer is already configured")
-        return
+    # if enhancer_configured:
+    #     logger.info("Enhancer is already configured")
+    #     return
 
     configure_services()
 
@@ -37,28 +62,28 @@ def configure_enhancer_services():
     stop_store.load_stop_sources()
     # TODO: do we need container?
     container['stops_store'] = stop_store
-    container['trips_store'] = trips.TripStore(stop_store)
-    container['carpools'] = CarpoolService(container['trips_store'])
+    # container['trips_store'] = trips.TripStore(stop_store)
+    # container['carpools'] = CarpoolService(container['trips_store'])
 
     transformer = TripTransformer(stop_store)
 
-    logger.info("Restore carpools...")
+    # logger.info("Restore carpools...")
 
-    for agency_id in container['agencies'].agencies:
-        for carpool_file_name in glob(f'data/carpool/{agency_id}/*.json'):
-            try:
-                with open(carpool_file_name) as carpool_file:
-                    carpool = Carpool(**(json.load(carpool_file)))
-                    container['carpools'].put(carpool.agency, carpool.id, carpool)
-            except Exception as e:
-                logger.warning("Issue during restore of carpool %s: %s", carpool_file_name, repr(e))
+    # for agency_id in container['agencies'].agencies:
+    #     for carpool_file_name in glob(f'data/carpool/{agency_id}/*.json'):
+    #         try:
+    #             with open(carpool_file_name) as carpool_file:
+    #                 carpool = Carpool(**(json.load(carpool_file)))
+    #                 container['carpools'].put(carpool.agency, carpool.id, carpool)
+    #         except Exception as e:
+    #             logger.warning("Issue during restore of carpool %s: %s", carpool_file_name, repr(e))
                     
-        # notify carpool about carpools in trash, as delete notifications must be sent
-        for carpool_file_name in glob(f'data/trash/{agency_id}/*.json'):
-            with open(carpool_file_name) as carpool_file:
-                carpool = Carpool(**(json.load(carpool_file)))
-                container['carpools'].delete(carpool.agency, carpool.id)
+    #     # notify carpool about carpools in trash, as delete notifications must be sent
+    #     for carpool_file_name in glob(f'data/trash/{agency_id}/*.json'):
+    #         with open(carpool_file_name) as carpool_file:
+    #             carpool = Carpool(**(json.load(carpool_file)))
+    #             container['carpools'].delete(carpool.agency, carpool.id)
 
-    logger.info("Restored carpools: %s", container['carpools'].get_all_ids())
+    # logger.info("Restored carpools: %s", container['carpools'].get_all_ids())
 
-    enhancer_configured = True
+    # enhancer_configured = True

amarillo-enhancer/enhancer.py

@@ -71,7 +71,7 @@ app = FastAPI(title="Amarillo Enhancer",
 configure_enhancer_services()
 stops_store = container['stops_store']
 transformer : TripTransformer = TripTransformer(stops_store)
-logger.info(transformer)
+# logger.info(transformer)
 
 @app.post("/",
              operation_id="enhancecarpool",

amarillo-enhancer/models/gtfs.py

@@ -0,0 +1,30 @@
+# TODO: move to enhancer
+from collections import namedtuple
+from datetime import timedelta
+
+GtfsFeedInfo = namedtuple('GtfsFeedInfo', 'feed_id feed_publisher_name feed_publisher_url feed_lang feed_version')
+GtfsAgency = namedtuple('GtfsAgency', 'agency_id agency_name agency_url agency_timezone agency_lang agency_email')
+GtfsRoute = namedtuple('GtfsRoute',  'agency_id route_id route_long_name route_type route_url route_short_name')
+GtfsStop = namedtuple('GtfsStop', 'stop_id stop_lat stop_lon stop_name')
+GtfsStopTime = namedtuple('GtfsStopTime', 'trip_id departure_time arrival_time stop_id stop_sequence pickup_type drop_off_type timepoint')
+GtfsTrip = namedtuple('GtfsTrip', 'route_id trip_id service_id shape_id trip_headsign bikes_allowed')
+GtfsCalendar = namedtuple('GtfsCalendar', 'service_id start_date end_date monday tuesday wednesday thursday friday saturday sunday')
+GtfsCalendarDate = namedtuple('GtfsCalendarDate', 'service_id date exception_type')
+GtfsShape = namedtuple('GtfsShape','shape_id shape_pt_lon shape_pt_lat shape_pt_sequence')
+
+# TODO Move to utils
+class GtfsTimeDelta(timedelta):
+    def __str__(self):
+        seconds = self.total_seconds()
+        hours = seconds // 3600
+        minutes = (seconds % 3600) // 60
+        seconds = seconds % 60
+        str = '{:02d}:{:02d}:{:02d}'.format(int(hours), int(minutes), int(seconds))
+        return (str)
+        
+    def __add__(self, other):
+        if isinstance(other, timedelta):
+            return self.__class__(self.days + other.days,
+                                  self.seconds + other.seconds,
+                                  self.microseconds + other.microseconds)
+        return NotImplemented

amarillo-enhancer/services/carpools.py

@@ -2,7 +2,7 @@ import json
 import logging
 from datetime import datetime
 from typing import Dict
-from amarillo.models.Carpool import Carpool
+from ..models.Carpool import Carpool
 from amarillo.utils.utils import yesterday, is_older_than_days
 
 logger = logging.getLogger(__name__)

amarillo-enhancer/services/config.py

@@ -0,0 +1,11 @@
+from pydantic import ConfigDict
+from pydantic_settings import BaseSettings
+
+
+class Config(BaseSettings):
+    env: str = 'DEV'
+    graphhopper_base_url: str = 'https://api.mfdz.de/gh'
+    stop_sources_file: str = 'data/stop_sources.json'
+    model_config = ConfigDict(extra='allow')
+
+config = Config(_env_file='config', _env_file_encoding='utf-8')

amarillo-enhancer/services/gtfs_constants.py

@@ -0,0 +1,14 @@
+# Constants
+
+NO_BIKES_ALLOWED = 2
+RIDESHARING_ROUTE_TYPE = 1551
+CALENDAR_DATES_EXCEPTION_TYPE_ADDED = 1
+CALENDAR_DATES_EXCEPTION_TYPE_REMOVED = 2
+STOP_TIMES_STOP_TYPE_REGULARLY = 0
+STOP_TIMES_STOP_TYPE_NONE = 1
+STOP_TIMES_STOP_TYPE_PHONE_AGENCY = 2
+STOP_TIMES_STOP_TYPE_COORDINATE_DRIVER = 3
+STOP_TIMES_TIMEPOINT_APPROXIMATE = 0 
+STOP_TIMES_TIMEPOINT_EXACT = 1
+
+MFDZ_DEFAULT_UNCERTAINITY = 600

amarillo-enhancer/services/stops.py

@@ -1,7 +1,7 @@
 import csv
 import geopandas as gpd
 import pandas as pd
-from amarillo.models.Carpool import StopTime
+from ..models.Carpool import StopTime
 from contextlib import closing
 from shapely.geometry import Point, LineString
 from shapely.ops import transform

amarillo-enhancer/services/trips.py

@@ -1,9 +1,9 @@
-from amarillo.plugins.enhancer.models.gtfs import GtfsTimeDelta, GtfsStopTime
-from amarillo.models.Carpool import MAX_STOPS_PER_TRIP, Carpool, Weekday, StopTime, PickupDropoffType, Driver, RidesharingInfo
-from amarillo.services.config import config
-from amarillo.plugins.enhancer.services.gtfs_constants import *
-from amarillo.plugins.enhancer.services.routing import RoutingService, RoutingException
-from amarillo.plugins.enhancer.services.stops import is_carpooling_stop
+from ..models.gtfs import GtfsTimeDelta, GtfsStopTime
+from ..models.Carpool import MAX_STOPS_PER_TRIP, Carpool, Weekday, StopTime, PickupDropoffType, Driver, RidesharingInfo
+from ..services.config import config
+from ..services.gtfs_constants import *
+from ..services.routing import RoutingService, RoutingException
+from ..services.stops import is_carpooling_stop
 from amarillo.utils.utils import assert_folder_exists, is_older_than_days, yesterday, geodesic_distance_in_m
 from shapely.geometry import Point, LineString, box
 from geojson_pydantic.geometries import LineString as GeoJSONLineString