Added routers

amarillo/app/main.py

@@ -10,11 +10,11 @@ import mimetypes
 from starlette.staticfiles import StaticFiles
 
 
-# from app.routers import carpool, agency, agencyconf, metrics, region
+from amarillo.app.routers import carpool, agency, agencyconf, region
 from fastapi import FastAPI
 
 # https://pydantic-docs.helpmanual.io/usage/settings/
-# from app.views import home
+# from amarillo.app.views import home
 
 logger.info("Hello Amarillo!")
 
@@ -69,20 +69,12 @@ app = FastAPI(title="Amarillo - The Carpooling Intermediary",
               redoc_url=None
               )
 
-# app.include_router(carpool.router)
-# app.include_router(agency.router)
-# app.include_router(agencyconf.router)
-# app.include_router(region.router)
-# app.include_router(metrics.router)
+app.include_router(carpool.router)
+app.include_router(agency.router)
+app.include_router(agencyconf.router)
+app.include_router(region.router)
 
 
-# instrumentator = Instrumentator().instrument(app)
-# instrumentator.add(pfi_metrics.default())
-# instrumentator.add(metrics.amarillo_trips_number_total())
-
-
-# instrumentator.instrument(app)
-
 import importlib
 import pkgutil
 

amarillo/app/routers/agency.py

@@ -0,0 +1,84 @@
+import logging
+import time
+from typing import List
+
+from fastapi import APIRouter, HTTPException, status, Depends
+
+from amarillo.app.models.Carpool import Carpool, Agency
+from amarillo.app.routers.agencyconf import verify_api_key, verify_admin_api_key, verify_permission_for_same_agency_or_admin
+# TODO should move this to service
+from amarillo.app.routers.carpool import store_carpool, delete_agency_carpools_older_than
+from amarillo.app.services.agencies import AgencyService
+from amarillo.app.services.importing.ride2go import import_ride2go
+from amarillo.app.utils.container import container
+from fastapi.responses import FileResponse
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(
+    prefix="/agency",
+    tags=["agency"]
+)
+
+
+@router.get("/{agency_id}",
+            operation_id="getAgencyById",
+            summary="Find agency by ID",
+            response_model=Agency,
+            description="Find agency by ID",
+            # TODO next to the status codes are "Links". There is nothing shown now.
+            # Either show something there, or hide the Links, or do nothing.
+            responses={ 
+                status.HTTP_404_NOT_FOUND: {"description": "Agency not found"},
+            },
+            )
+async def get_agency(agency_id: str, admin_api_key: str = Depends(verify_api_key)) -> Agency:
+    agencies: AgencyService = container['agencies']
+    agency = agencies.get_agency(agency_id)
+    agency_exists = agency is not None
+
+    if not agency_exists:
+        message = f"Agency with id {agency_id} does not exist."
+        logger.error(message)
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=message)
+
+    logger.info(f"Get agency {agency_id}.")
+
+    return agency
+
+# TODO add push batch endpoint
+
+@router.post("/{agency_id}/sync",
+             operation_id="sync",
+             summary="Synchronizes all carpool offers",
+             response_model=List[Carpool],
+             responses={
+                 status.HTTP_200_OK: {
+                     "description": "Carpool created"},
+                 status.HTTP_404_NOT_FOUND: {
+                     "description": "Agency does not exist"},
+                 status.HTTP_500_INTERNAL_SERVER_ERROR: {
+                     "description": "Import error"}
+             })
+async def sync(agency_id: str, requesting_agency_id: str = Depends(verify_api_key)) -> List[Carpool]:
+    await verify_permission_for_same_agency_or_admin(agency_id, requesting_agency_id)
+
+    if agency_id == "ride2go":
+        import_function = import_ride2go
+    else:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Agency does not exist or does not support sync.")
+
+    try:
+        carpools = import_function()
+        # Reduce current time by a minute to avoid inter process timestamp issues
+        synced_files_older_than = time.time() - 60
+        result = [await store_carpool(cp) for cp in carpools]
+        await delete_agency_carpools_older_than(agency_id, synced_files_older_than)
+        return result
+    except BaseException as e:
+        logger.exception("Error on sync for agency %s", agency_id)
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Something went wrong during import.")

amarillo/app/routers/agencyconf.py

@@ -0,0 +1,103 @@
+import logging
+from typing import List
+
+from fastapi import APIRouter, HTTPException, status, Header, Depends
+
+from amarillo.app.models.AgencyConf import AgencyConf
+from amarillo.app.services.agencyconf import AgencyConfService
+from amarillo.app.services.config import config
+from amarillo.app.utils.container import container
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(
+    prefix="/agencyconf",
+    tags=["agencyconf"]
+)
+
+# This endpoint is not shown in PROD installations, only in development
+# TODO make this an explicit config option
+include_in_schema = config.env != 'PROD'
+
+
+# noinspection PyPep8Naming
+# X_API_Key is upper case for OpenAPI
+async def verify_admin_api_key(X_API_Key: str = Header(...)):
+    if X_API_Key != config.admin_token:
+        message="X-API-Key header invalid"
+        logger.error(message)
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=message)
+
+    return "admin"
+
+
+# noinspection PyPep8Naming
+# X_API_Key is upper case for OpenAPI
+async def verify_api_key(X_API_Key: str = Header(...)):
+    agency_conf_service: AgencyConfService = container['agencyconf']
+
+    return agency_conf_service.check_api_key(X_API_Key)
+
+# TODO Return code 403 Unauthoized (in response_status_codes as well...)
+async def verify_permission_for_same_agency_or_admin(agency_id_in_path_or_body, agency_id_from_api_key):
+    """Verifies that an agency is accessing something it owns or the user is admin
+
+    The agency_id is part of some paths, or when not in the path it is in the body, e.g. in PUT /carpool.
+
+    This function encapsulates the formula 'working with own stuff, or admin'.
+    """
+    is_permitted = agency_id_in_path_or_body == agency_id_from_api_key or agency_id_from_api_key == "admin"
+
+    if not is_permitted:
+        message = f"Working with {agency_id_in_path_or_body} resources is not permitted for {agency_id_from_api_key}."
+        logger.error(message)
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=message)
+
+
+@router.get("/",
+            include_in_schema=include_in_schema,
+            operation_id="getAgencyIdsWhichHaveAConfiguration",
+            summary="Get agency_ids which have a configuration",
+            response_model=List[str],
+            description="Returns the agency_ids but not the details.",
+            status_code=status.HTTP_200_OK)
+async def get_agency_ids(admin_api_key: str = Depends(verify_api_key)) -> [str]:
+    return container['agencyconf'].get_agency_ids()
+
+
+@router.post("/",
+             include_in_schema=include_in_schema,
+             operation_id="postNewAgencyConf",
+             summary="Post a new AgencyConf")
+async def post_agency_conf(agency_conf: AgencyConf, admin_api_key: str = Depends(verify_admin_api_key)):
+    agency_conf_service: AgencyConfService = container['agencyconf']
+    agency_conf_service.add(agency_conf)
+
+# TODO 400->403
+@router.delete("/{agency_id}",
+               include_in_schema=include_in_schema,
+               operation_id="deleteAgencyConf",
+               status_code=status.HTTP_200_OK,
+               summary="Delete configuration of an agency. Returns true if the token for the agency existed, "
+                       "false if it didn't exist."
+               )
+async def delete_agency_conf(agency_id: str, requesting_agency_id: str = Depends(verify_api_key)):
+    agency_may_delete_own = requesting_agency_id == agency_id
+    admin_may_delete_everything = requesting_agency_id == "admin"
+    is_permitted = agency_may_delete_own or admin_may_delete_everything
+
+    if not is_permitted:
+        message = f"The API key for {requesting_agency_id} can not delete the configuration for {agency_id}"
+        logger.error(message)
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=message)
+
+    agency_conf_service: AgencyConfService = container['agencyconf']
+
+    agency_exists = agency_id in agency_conf_service.get_agency_ids()
+
+    if not agency_exists:
+        message = f"No config for {agency_id}"
+        logger.error(message)
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=message)
+
+    agency_conf_service.delete(agency_id)

amarillo/app/routers/carpool.py

@@ -0,0 +1,137 @@
+import logging
+import json
+import os
+import os.path
+import re
+from glob import glob
+
+from fastapi import APIRouter, Body, Header, HTTPException, status, Depends
+from datetime import datetime
+
+from amarillo.app.models.Carpool import Carpool
+from amarillo.app.routers.agencyconf import verify_api_key, verify_permission_for_same_agency_or_admin
+from amarillo.app.tests.sampledata import examples
+
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(
+    prefix="/carpool",
+    tags=["carpool"]
+)
+
+@router.post("/",
+             operation_id="addcarpool",
+             summary="Add a new or update existing carpool",
+             description="Carpool object to be created or updated",
+             response_model=Carpool,
+             responses={
+                 status.HTTP_404_NOT_FOUND: {
+                     "description": "Agency does not exist"},
+                 
+                })
+async def post_carpool(carpool: Carpool = Body(..., examples=examples),
+                       requesting_agency_id: str = Depends(verify_api_key)) -> Carpool:
+    await verify_permission_for_same_agency_or_admin(carpool.agency, requesting_agency_id)
+
+    logger.info(f"POST trip {carpool.agency}:{carpool.id}.")
+    await assert_agency_exists(carpool.agency)
+
+    await set_lastUpdated_if_unset(carpool)
+
+    await save_carpool(carpool)
+
+    return carpool
+    
+# TODO 403
+@router.get("/{agency_id}/{carpool_id}",
+            operation_id="getcarpoolById",
+            summary="Find carpool by ID",
+            response_model=Carpool,
+            description="Find carpool by ID",
+            responses={
+                status.HTTP_404_NOT_FOUND: {"description": "Carpool not found"},
+            },
+            )
+async def get_carpool(agency_id: str, carpool_id: str, api_key: str = Depends(verify_api_key)) -> Carpool:
+    logger.info(f"Get trip {agency_id}:{carpool_id}.")
+    await assert_agency_exists(agency_id)
+    await assert_carpool_exists(agency_id, carpool_id)
+
+    carpool = await load_carpool(agency_id, carpool_id)
+
+    return carpool
+
+
+@router.delete("/{agency_id}/{carpool_id}",
+               operation_id="deletecarpool",
+               summary="Deletes a carpool",
+               description="Carpool id to delete",
+               responses={
+                   status.HTTP_404_NOT_FOUND: {
+                       "description": "Carpool or agency not found"},
+               },
+               )
+async def delete_carpool(agency_id: str, carpool_id: str, requesting_agency_id: str = Depends(verify_api_key)):
+    await verify_permission_for_same_agency_or_admin(agency_id, requesting_agency_id)
+
+    logger.info(f"Delete trip {agency_id}:{carpool_id}.")
+    await assert_agency_exists(agency_id)
+    await assert_carpool_exists(agency_id, carpool_id)
+    
+    return await _delete_carpool(agency_id, carpool_id)
+
+async def _delete_carpool(agency_id: str, carpool_id: str):
+    logger.info(f"Delete carpool {agency_id}:{carpool_id}.")
+    cp = await load_carpool(agency_id, carpool_id)
+    logger.info(f"Loaded carpool {agency_id}:{carpool_id}.")
+    # load and store, to receive pyinotify events and have file timestamp updated
+    await save_carpool(cp, 'data/trash')
+    logger.info(f"Saved carpool {agency_id}:{carpool_id} in trash.")
+    os.remove(f"data/carpool/{agency_id}/{carpool_id}.json")
+
+async def store_carpool(carpool: Carpool) -> Carpool:
+    await set_lastUpdated_if_unset(carpool)
+    await save_carpool(carpool)
+
+    return carpool
+
+async def set_lastUpdated_if_unset(carpool):
+    if carpool.lastUpdated is None:
+        carpool.lastUpdated = datetime.now()
+
+
+async def load_carpool(agency_id, carpool_id) -> Carpool:
+    with open(f'data/carpool/{agency_id}/{carpool_id}.json', 'r', encoding='utf-8') as f:
+        dict = json.load(f)
+        carpool = Carpool(**dict)
+    return carpool
+
+
+async def save_carpool(carpool, folder: str = 'data/carpool'):
+    with open(f'{folder}/{carpool.agency}/{carpool.id}.json', 'w', encoding='utf-8') as f:
+        f.write(carpool.json())
+
+
+async def assert_agency_exists(agency_id: str):
+    agency_exists = os.path.exists(f"conf/agency/{agency_id}.json")
+    if not agency_exists:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Agency with id {agency_id} does not exist.")
+
+
+async def assert_carpool_exists(agency_id: str, carpool_id: str):
+    carpool_exists = os.path.exists(f"data/carpool/{agency_id}/{carpool_id}.json")
+    if not carpool_exists:
+        raise HTTPException(
+            status_code=404,
+            detail=f"Carpool with id {carpool_id} for agency {agency_id} not found")
+
+
+async def delete_agency_carpools_older_than(agency_id, timestamp):
+    for carpool_file_name in glob(f'data/carpool/{agency_id}/*.json'):
+        if os.path.getmtime(carpool_file_name) < timestamp:
+            m = re.search(r'([a-zA-Z0-9_-]+)\.json$', carpool_file_name)
+            # TODO log deletion
+            await _delete_carpool(agency_id, m[1])

amarillo/app/routers/region.py

@@ -0,0 +1,88 @@
+import logging
+import time
+from typing import List
+
+from fastapi import APIRouter, HTTPException, status, Depends
+
+from amarillo.app.models.Carpool import Region
+from amarillo.app.routers.agencyconf import verify_admin_api_key
+from amarillo.app.services.regions import RegionService
+from amarillo.app.utils.container import container
+from fastapi.responses import FileResponse
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(
+    prefix="/region",
+    tags=["region"]
+)
+
+
+@router.get("/",
+            operation_id="getRegions",
+            summary="Return all regions",
+            response_model=List[Region],
+            responses={
+            },
+            )
+async def get_regions() -> List[Region]:
+    service: RegionService = container['regions']
+    return list(service.regions.values())
+
+@router.get("/{region_id}",
+            operation_id="getRegionById",
+            summary="Find region by ID",
+            response_model=Region,
+            description="Find region by ID",
+            responses={
+                status.HTTP_404_NOT_FOUND: {"description": "Region not found"},
+            },
+            )
+async def get_region(region_id: str) -> Region:
+    region = _assert_region_exists(region_id)
+    logger.info(f"Get region {region_id}.")
+
+    return region
+
+def _assert_region_exists(region_id: str) -> Region:
+    regions: RegionService = container['regions']
+    region = regions.get_region(region_id)
+    region_exists = region is not None
+
+    if not region_exists:
+        message = f"Region with id {region_id} does not exist."
+        logger.error(message)
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=message)
+
+    return region
+
+@router.get("/{region_id}/gtfs", 
+    summary="Return GTFS Feed for this region",
+    response_description="GTFS-Feed (zip-file)",
+    response_class=FileResponse,
+    responses={
+                status.HTTP_404_NOT_FOUND: {"description": "Region not found"},
+        }
+    )
+async def get_file(region_id: str, user: str = Depends(verify_admin_api_key)):
+    _assert_region_exists(region_id)
+    return FileResponse(f'data/gtfs/amarillo.{region_id}.gtfs.zip')
+
+@router.get("/{region_id}/gtfs-rt",
+    summary="Return GTFS-RT Feed for this region",
+    response_description="GTFS-RT-Feed",
+    response_class=FileResponse,
+    responses={
+                status.HTTP_404_NOT_FOUND: {"description": "Region not found"},
+                status.HTTP_400_BAD_REQUEST: {"description": "Bad request, e.g. because format is not supported, i.e. neither protobuf nor json."}
+        }
+    )
+async def get_file(region_id: str, format: str = 'protobuf', user: str = Depends(verify_admin_api_key)):
+    _assert_region_exists(region_id)
+    if format == 'json':
+        return FileResponse(f'data/gtfs/amarillo.{region_id}.gtfsrt.json')
+    elif format == 'protobuf':
+        return FileResponse(f'data/gtfs/amarillo.{region_id}.gtfsrt.pbf')
+    else:
+        message = "Specified format is not supported, i.e. neither protobuf nor json."
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=message)

amarillo/app/services/importing/ride2go.py

@@ -0,0 +1,68 @@
+import logging
+from typing import List
+
+import requests
+from amarillo.app.models.Carpool import Carpool, StopTime
+from amarillo.app.services.config import config
+
+from amarillo.app.services.secrets import secrets
+import re
+
+logger = logging.getLogger(__name__)
+
+def as_StopTime(stop):
+    return StopTime(
+        # id="todo",
+        name=stop['address'],
+        lat=stop['coordinates']['lat'],
+        lon=stop['coordinates']['lon']
+    )
+
+
+def as_Carpool(dict) -> Carpool:
+    (agency, id) = re.findall(r'https?://(.*)\..*/?trip=([0-9]+)', dict['deeplink'])[0]
+
+    carpool = Carpool(id=id,
+                      agency=agency,
+                      deeplink=dict['deeplink'],
+                      stops=[as_StopTime(s) for s in dict.get('stops')],
+                      departureTime=dict.get('departTime'),
+                      departureDate=dict.get('departDate') if dict.get('departDate') else dict.get('weekdays'),
+                      lastUpdated=dict.get('lastUpdated'))
+
+    return carpool
+
+def import_ride2go() -> List[Carpool]:
+    ride2go_query_data = config.ride2go_query_data
+
+    ride2go_url = "https://ride2go.com/api/v1/trips/export"
+
+    api_key = secrets.ride2go_token
+
+    ride2go_headers = {
+        'Content-type': 'text/plain;charset=UTF-8',
+        'X-API-Key': f"{api_key}"
+    }
+
+    try:
+        result = requests.get(
+            ride2go_url,
+            data=ride2go_query_data,
+            headers=ride2go_headers
+        )
+        if result.status_code == 200:
+            json_results = result.json()
+            carpools = [as_Carpool(cp) for cp in json_results]
+
+            return carpools
+        else:
+            logger.error("ride2go request returned with status_code %s", result.status_code)
+            json_results = result.json()
+            if 'status' in json_results:
+                logger.error("Error was: %s", result.json()['status'])
+            
+            raise ValueError("Sync failed with error. See logs") 
+
+    except BaseException as e:
+        logger.exception("Error on import for agency ride2go")
+        raise e

amarillo/app/tests/sampledata.py

@@ -0,0 +1,91 @@
+from amarillo.app.models.Carpool import Carpool, StopTime, Weekday
+
+# TODO use meanigful values for id and lat, lon
+stops_1234 = [
+    StopTime(
+        id="de:08115:4802:0:3",
+        name="Herrenberg",
+        lat=48.5948979,
+        lon=8.8684534),
+    StopTime(
+        id="de:08111:6221:3:6",
+        name="Stuttgart Feuersee",
+        lat= 48.7733275,
+        lon=9.1671590)]
+
+carpool_1234 = Carpool(
+    id="1234",
+    agency="mfdz",
+    deeplink="https://mfdz.de/trip/1234",
+    stops=stops_1234,
+    departureTime="07:00",
+    departureDate="2022-03-30",
+)
+
+carpool_repeating = Carpool(
+    id="12345",
+    agency="mfdz",
+    deeplink="https://mfdz.de/trip/12345",
+    stops=stops_1234,
+    departureTime="06:00",
+    departureDate=[Weekday.monday, Weekday.tuesday, Weekday.wednesday,
+                   Weekday.thursday, Weekday.friday],
+)
+
+examples = {
+    "one-time trip with date": {
+        "summary": "one-time trip with date",
+        "description": "carpool object that should to be added or modified",
+        "value": carpool_1234},
+    "repeating trip Mon-Fri": {
+        "summary": "repeating trip Mon-Fri",
+        "description": "carpool object that should to be added or modified",
+        "value": carpool_repeating}
+}
+
+data1 = {
+    'id': "Eins",
+    'agency': "mfdz",
+    'deeplink': "https://mfdz.de/trip/123",
+    'stops': [
+        {'id': "mfdz:12073:001", 'name': "abc", 'lat': 53.11901, 'lon': 14.015776},
+        {'id': "de:12073:900340137::3", 'name': "xyz", 'lat': 53.011459, 'lon': 13.94945}],
+    'departureTime': "23:59",
+    'departureDate': "2022-05-30",
+}
+
+carpool_repeating_json = {
+    'id': "Zwei",
+    'agency': "mfdz",
+    'deeplink': "https://mfdz.de/trip/123",
+    'stops': [
+        {'id': "mfdz:12073:001", 'name': "abc", 'lat': 53.11901, 'lon': 14.015776},
+        {'id': "de:12073:900340137::3", 'name': "xyz", 'lat': 53.011459, 'lon': 13.94945}],
+    'departureTime': "15:00",
+    'departureDate': ["monday"],
+}
+
+
+cp1 = Carpool(**data1)
+
+# JSON string for trying out the API in Swagger
+cp2 = """
+{
+  "id": "Vier",
+  "agency": "string",
+  "deeplink": "http://mfdz.de",
+  "stops": [
+    {
+      "id": "de:12073:900340137::4", "name": "drei", "lat": 45, "lon": 9
+    },
+    {
+      "id": "de:12073:900340137::5", "name": "drei b", "lat": 45, "lon": 9
+    }
+  ],
+  "departureTime": "12:34",
+  "departureDate": "2022-03-30",
+  "lastUpdated": "2022-03-30 12:34"
+}
+"""
+
+stop_issue = {"id": "106727", "agency": "ride2go", "deeplink": "https://ride2go.com/?trip=106727", "stops": [{"id": None, "name": "Mitfahrbank Angerm\u00fcnde, Einkaufscenter, Prenzlauer Stra\u00dfe, 16278 Angerm\u00fcnde", "departureTime": None, "arrivalTime": None, "lat": 53.0220209, "lon": 13.9999447, "pickup_dropoff": None}, {"id": None, "name": "Mitfahrbank B\u00f6lkendorf, B\u00f6lkendorfer Stra\u00dfe, 16278 Angerm\u00fcnde", "departureTime": None, "arrivalTime": None, "lat": 52.949856, "lon": 14.003533, "pickup_dropoff": None}], "departureTime": "17:00:00", "departureDate": "2022-06-22", "path": None, "lastUpdated": "2022-06-22T11:04:22"}