diff --git a/amarillo/app/main.py b/amarillo/app/main.py index d9abb6a..6935f96 100644 --- a/amarillo/app/main.py +++ b/amarillo/app/main.py @@ -10,11 +10,11 @@ import mimetypes from starlette.staticfiles import StaticFiles -# from app.routers import carpool, agency, agencyconf, metrics, region +from amarillo.app.routers import carpool, agency, agencyconf, region from fastapi import FastAPI # https://pydantic-docs.helpmanual.io/usage/settings/ -# from app.views import home +# from amarillo.app.views import home logger.info("Hello Amarillo!") @@ -69,20 +69,12 @@ app = FastAPI(title="Amarillo - The Carpooling Intermediary", redoc_url=None ) -# app.include_router(carpool.router) -# app.include_router(agency.router) -# app.include_router(agencyconf.router) -# app.include_router(region.router) -# app.include_router(metrics.router) +app.include_router(carpool.router) +app.include_router(agency.router) +app.include_router(agencyconf.router) +app.include_router(region.router) -# instrumentator = Instrumentator().instrument(app) -# instrumentator.add(pfi_metrics.default()) -# instrumentator.add(metrics.amarillo_trips_number_total()) - - -# instrumentator.instrument(app) - import importlib import pkgutil diff --git a/amarillo/app/routers/__init__.py b/amarillo/app/routers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/amarillo/app/routers/agency.py b/amarillo/app/routers/agency.py new file mode 100644 index 0000000..81aabad --- /dev/null +++ b/amarillo/app/routers/agency.py @@ -0,0 +1,84 @@ +import logging +import time +from typing import List + +from fastapi import APIRouter, HTTPException, status, Depends + +from amarillo.app.models.Carpool import Carpool, Agency +from amarillo.app.routers.agencyconf import verify_api_key, verify_admin_api_key, verify_permission_for_same_agency_or_admin +# TODO should move this to service +from amarillo.app.routers.carpool import store_carpool, delete_agency_carpools_older_than +from amarillo.app.services.agencies import AgencyService +from amarillo.app.services.importing.ride2go import import_ride2go +from amarillo.app.utils.container import container +from fastapi.responses import FileResponse + +logger = logging.getLogger(__name__) + +router = APIRouter( + prefix="/agency", + tags=["agency"] +) + + +@router.get("/{agency_id}", + operation_id="getAgencyById", + summary="Find agency by ID", + response_model=Agency, + description="Find agency by ID", + # TODO next to the status codes are "Links". There is nothing shown now. + # Either show something there, or hide the Links, or do nothing. + responses={ + status.HTTP_404_NOT_FOUND: {"description": "Agency not found"}, + }, + ) +async def get_agency(agency_id: str, admin_api_key: str = Depends(verify_api_key)) -> Agency: + agencies: AgencyService = container['agencies'] + agency = agencies.get_agency(agency_id) + agency_exists = agency is not None + + if not agency_exists: + message = f"Agency with id {agency_id} does not exist." + logger.error(message) + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=message) + + logger.info(f"Get agency {agency_id}.") + + return agency + +# TODO add push batch endpoint + +@router.post("/{agency_id}/sync", + operation_id="sync", + summary="Synchronizes all carpool offers", + response_model=List[Carpool], + responses={ + status.HTTP_200_OK: { + "description": "Carpool created"}, + status.HTTP_404_NOT_FOUND: { + "description": "Agency does not exist"}, + status.HTTP_500_INTERNAL_SERVER_ERROR: { + "description": "Import error"} + }) +async def sync(agency_id: str, requesting_agency_id: str = Depends(verify_api_key)) -> List[Carpool]: + await verify_permission_for_same_agency_or_admin(agency_id, requesting_agency_id) + + if agency_id == "ride2go": + import_function = import_ride2go + else: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Agency does not exist or does not support sync.") + + try: + carpools = import_function() + # Reduce current time by a minute to avoid inter process timestamp issues + synced_files_older_than = time.time() - 60 + result = [await store_carpool(cp) for cp in carpools] + await delete_agency_carpools_older_than(agency_id, synced_files_older_than) + return result + except BaseException as e: + logger.exception("Error on sync for agency %s", agency_id) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Something went wrong during import.") diff --git a/amarillo/app/routers/agencyconf.py b/amarillo/app/routers/agencyconf.py new file mode 100644 index 0000000..3b57639 --- /dev/null +++ b/amarillo/app/routers/agencyconf.py @@ -0,0 +1,103 @@ +import logging +from typing import List + +from fastapi import APIRouter, HTTPException, status, Header, Depends + +from amarillo.app.models.AgencyConf import AgencyConf +from amarillo.app.services.agencyconf import AgencyConfService +from amarillo.app.services.config import config +from amarillo.app.utils.container import container + +logger = logging.getLogger(__name__) + +router = APIRouter( + prefix="/agencyconf", + tags=["agencyconf"] +) + +# This endpoint is not shown in PROD installations, only in development +# TODO make this an explicit config option +include_in_schema = config.env != 'PROD' + + +# noinspection PyPep8Naming +# X_API_Key is upper case for OpenAPI +async def verify_admin_api_key(X_API_Key: str = Header(...)): + if X_API_Key != config.admin_token: + message="X-API-Key header invalid" + logger.error(message) + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=message) + + return "admin" + + +# noinspection PyPep8Naming +# X_API_Key is upper case for OpenAPI +async def verify_api_key(X_API_Key: str = Header(...)): + agency_conf_service: AgencyConfService = container['agencyconf'] + + return agency_conf_service.check_api_key(X_API_Key) + +# TODO Return code 403 Unauthoized (in response_status_codes as well...) +async def verify_permission_for_same_agency_or_admin(agency_id_in_path_or_body, agency_id_from_api_key): + """Verifies that an agency is accessing something it owns or the user is admin + + The agency_id is part of some paths, or when not in the path it is in the body, e.g. in PUT /carpool. + + This function encapsulates the formula 'working with own stuff, or admin'. + """ + is_permitted = agency_id_in_path_or_body == agency_id_from_api_key or agency_id_from_api_key == "admin" + + if not is_permitted: + message = f"Working with {agency_id_in_path_or_body} resources is not permitted for {agency_id_from_api_key}." + logger.error(message) + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=message) + + +@router.get("/", + include_in_schema=include_in_schema, + operation_id="getAgencyIdsWhichHaveAConfiguration", + summary="Get agency_ids which have a configuration", + response_model=List[str], + description="Returns the agency_ids but not the details.", + status_code=status.HTTP_200_OK) +async def get_agency_ids(admin_api_key: str = Depends(verify_api_key)) -> [str]: + return container['agencyconf'].get_agency_ids() + + +@router.post("/", + include_in_schema=include_in_schema, + operation_id="postNewAgencyConf", + summary="Post a new AgencyConf") +async def post_agency_conf(agency_conf: AgencyConf, admin_api_key: str = Depends(verify_admin_api_key)): + agency_conf_service: AgencyConfService = container['agencyconf'] + agency_conf_service.add(agency_conf) + +# TODO 400->403 +@router.delete("/{agency_id}", + include_in_schema=include_in_schema, + operation_id="deleteAgencyConf", + status_code=status.HTTP_200_OK, + summary="Delete configuration of an agency. Returns true if the token for the agency existed, " + "false if it didn't exist." + ) +async def delete_agency_conf(agency_id: str, requesting_agency_id: str = Depends(verify_api_key)): + agency_may_delete_own = requesting_agency_id == agency_id + admin_may_delete_everything = requesting_agency_id == "admin" + is_permitted = agency_may_delete_own or admin_may_delete_everything + + if not is_permitted: + message = f"The API key for {requesting_agency_id} can not delete the configuration for {agency_id}" + logger.error(message) + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=message) + + agency_conf_service: AgencyConfService = container['agencyconf'] + + agency_exists = agency_id in agency_conf_service.get_agency_ids() + + if not agency_exists: + message = f"No config for {agency_id}" + logger.error(message) + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=message) + + agency_conf_service.delete(agency_id) diff --git a/amarillo/app/routers/carpool.py b/amarillo/app/routers/carpool.py new file mode 100644 index 0000000..4ede57c --- /dev/null +++ b/amarillo/app/routers/carpool.py @@ -0,0 +1,137 @@ +import logging +import json +import os +import os.path +import re +from glob import glob + +from fastapi import APIRouter, Body, Header, HTTPException, status, Depends +from datetime import datetime + +from amarillo.app.models.Carpool import Carpool +from amarillo.app.routers.agencyconf import verify_api_key, verify_permission_for_same_agency_or_admin +from amarillo.app.tests.sampledata import examples + + +logger = logging.getLogger(__name__) + +router = APIRouter( + prefix="/carpool", + tags=["carpool"] +) + +@router.post("/", + operation_id="addcarpool", + summary="Add a new or update existing carpool", + description="Carpool object to be created or updated", + response_model=Carpool, + responses={ + status.HTTP_404_NOT_FOUND: { + "description": "Agency does not exist"}, + + }) +async def post_carpool(carpool: Carpool = Body(..., examples=examples), + requesting_agency_id: str = Depends(verify_api_key)) -> Carpool: + await verify_permission_for_same_agency_or_admin(carpool.agency, requesting_agency_id) + + logger.info(f"POST trip {carpool.agency}:{carpool.id}.") + await assert_agency_exists(carpool.agency) + + await set_lastUpdated_if_unset(carpool) + + await save_carpool(carpool) + + return carpool + +# TODO 403 +@router.get("/{agency_id}/{carpool_id}", + operation_id="getcarpoolById", + summary="Find carpool by ID", + response_model=Carpool, + description="Find carpool by ID", + responses={ + status.HTTP_404_NOT_FOUND: {"description": "Carpool not found"}, + }, + ) +async def get_carpool(agency_id: str, carpool_id: str, api_key: str = Depends(verify_api_key)) -> Carpool: + logger.info(f"Get trip {agency_id}:{carpool_id}.") + await assert_agency_exists(agency_id) + await assert_carpool_exists(agency_id, carpool_id) + + carpool = await load_carpool(agency_id, carpool_id) + + return carpool + + +@router.delete("/{agency_id}/{carpool_id}", + operation_id="deletecarpool", + summary="Deletes a carpool", + description="Carpool id to delete", + responses={ + status.HTTP_404_NOT_FOUND: { + "description": "Carpool or agency not found"}, + }, + ) +async def delete_carpool(agency_id: str, carpool_id: str, requesting_agency_id: str = Depends(verify_api_key)): + await verify_permission_for_same_agency_or_admin(agency_id, requesting_agency_id) + + logger.info(f"Delete trip {agency_id}:{carpool_id}.") + await assert_agency_exists(agency_id) + await assert_carpool_exists(agency_id, carpool_id) + + return await _delete_carpool(agency_id, carpool_id) + +async def _delete_carpool(agency_id: str, carpool_id: str): + logger.info(f"Delete carpool {agency_id}:{carpool_id}.") + cp = await load_carpool(agency_id, carpool_id) + logger.info(f"Loaded carpool {agency_id}:{carpool_id}.") + # load and store, to receive pyinotify events and have file timestamp updated + await save_carpool(cp, 'data/trash') + logger.info(f"Saved carpool {agency_id}:{carpool_id} in trash.") + os.remove(f"data/carpool/{agency_id}/{carpool_id}.json") + +async def store_carpool(carpool: Carpool) -> Carpool: + await set_lastUpdated_if_unset(carpool) + await save_carpool(carpool) + + return carpool + +async def set_lastUpdated_if_unset(carpool): + if carpool.lastUpdated is None: + carpool.lastUpdated = datetime.now() + + +async def load_carpool(agency_id, carpool_id) -> Carpool: + with open(f'data/carpool/{agency_id}/{carpool_id}.json', 'r', encoding='utf-8') as f: + dict = json.load(f) + carpool = Carpool(**dict) + return carpool + + +async def save_carpool(carpool, folder: str = 'data/carpool'): + with open(f'{folder}/{carpool.agency}/{carpool.id}.json', 'w', encoding='utf-8') as f: + f.write(carpool.json()) + + +async def assert_agency_exists(agency_id: str): + agency_exists = os.path.exists(f"conf/agency/{agency_id}.json") + if not agency_exists: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Agency with id {agency_id} does not exist.") + + +async def assert_carpool_exists(agency_id: str, carpool_id: str): + carpool_exists = os.path.exists(f"data/carpool/{agency_id}/{carpool_id}.json") + if not carpool_exists: + raise HTTPException( + status_code=404, + detail=f"Carpool with id {carpool_id} for agency {agency_id} not found") + + +async def delete_agency_carpools_older_than(agency_id, timestamp): + for carpool_file_name in glob(f'data/carpool/{agency_id}/*.json'): + if os.path.getmtime(carpool_file_name) < timestamp: + m = re.search(r'([a-zA-Z0-9_-]+)\.json$', carpool_file_name) + # TODO log deletion + await _delete_carpool(agency_id, m[1]) diff --git a/amarillo/app/routers/region.py b/amarillo/app/routers/region.py new file mode 100644 index 0000000..2674c76 --- /dev/null +++ b/amarillo/app/routers/region.py @@ -0,0 +1,88 @@ +import logging +import time +from typing import List + +from fastapi import APIRouter, HTTPException, status, Depends + +from amarillo.app.models.Carpool import Region +from amarillo.app.routers.agencyconf import verify_admin_api_key +from amarillo.app.services.regions import RegionService +from amarillo.app.utils.container import container +from fastapi.responses import FileResponse + +logger = logging.getLogger(__name__) + +router = APIRouter( + prefix="/region", + tags=["region"] +) + + +@router.get("/", + operation_id="getRegions", + summary="Return all regions", + response_model=List[Region], + responses={ + }, + ) +async def get_regions() -> List[Region]: + service: RegionService = container['regions'] + return list(service.regions.values()) + +@router.get("/{region_id}", + operation_id="getRegionById", + summary="Find region by ID", + response_model=Region, + description="Find region by ID", + responses={ + status.HTTP_404_NOT_FOUND: {"description": "Region not found"}, + }, + ) +async def get_region(region_id: str) -> Region: + region = _assert_region_exists(region_id) + logger.info(f"Get region {region_id}.") + + return region + +def _assert_region_exists(region_id: str) -> Region: + regions: RegionService = container['regions'] + region = regions.get_region(region_id) + region_exists = region is not None + + if not region_exists: + message = f"Region with id {region_id} does not exist." + logger.error(message) + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=message) + + return region + +@router.get("/{region_id}/gtfs", + summary="Return GTFS Feed for this region", + response_description="GTFS-Feed (zip-file)", + response_class=FileResponse, + responses={ + status.HTTP_404_NOT_FOUND: {"description": "Region not found"}, + } + ) +async def get_file(region_id: str, user: str = Depends(verify_admin_api_key)): + _assert_region_exists(region_id) + return FileResponse(f'data/gtfs/amarillo.{region_id}.gtfs.zip') + +@router.get("/{region_id}/gtfs-rt", + summary="Return GTFS-RT Feed for this region", + response_description="GTFS-RT-Feed", + response_class=FileResponse, + responses={ + status.HTTP_404_NOT_FOUND: {"description": "Region not found"}, + status.HTTP_400_BAD_REQUEST: {"description": "Bad request, e.g. because format is not supported, i.e. neither protobuf nor json."} + } + ) +async def get_file(region_id: str, format: str = 'protobuf', user: str = Depends(verify_admin_api_key)): + _assert_region_exists(region_id) + if format == 'json': + return FileResponse(f'data/gtfs/amarillo.{region_id}.gtfsrt.json') + elif format == 'protobuf': + return FileResponse(f'data/gtfs/amarillo.{region_id}.gtfsrt.pbf') + else: + message = "Specified format is not supported, i.e. neither protobuf nor json." + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=message) diff --git a/amarillo/app/services/importing/__init__.py b/amarillo/app/services/importing/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/amarillo/app/services/importing/ride2go.py b/amarillo/app/services/importing/ride2go.py new file mode 100644 index 0000000..7fc56c7 --- /dev/null +++ b/amarillo/app/services/importing/ride2go.py @@ -0,0 +1,68 @@ +import logging +from typing import List + +import requests +from amarillo.app.models.Carpool import Carpool, StopTime +from amarillo.app.services.config import config + +from amarillo.app.services.secrets import secrets +import re + +logger = logging.getLogger(__name__) + +def as_StopTime(stop): + return StopTime( + # id="todo", + name=stop['address'], + lat=stop['coordinates']['lat'], + lon=stop['coordinates']['lon'] + ) + + +def as_Carpool(dict) -> Carpool: + (agency, id) = re.findall(r'https?://(.*)\..*/?trip=([0-9]+)', dict['deeplink'])[0] + + carpool = Carpool(id=id, + agency=agency, + deeplink=dict['deeplink'], + stops=[as_StopTime(s) for s in dict.get('stops')], + departureTime=dict.get('departTime'), + departureDate=dict.get('departDate') if dict.get('departDate') else dict.get('weekdays'), + lastUpdated=dict.get('lastUpdated')) + + return carpool + +def import_ride2go() -> List[Carpool]: + ride2go_query_data = config.ride2go_query_data + + ride2go_url = "https://ride2go.com/api/v1/trips/export" + + api_key = secrets.ride2go_token + + ride2go_headers = { + 'Content-type': 'text/plain;charset=UTF-8', + 'X-API-Key': f"{api_key}" + } + + try: + result = requests.get( + ride2go_url, + data=ride2go_query_data, + headers=ride2go_headers + ) + if result.status_code == 200: + json_results = result.json() + carpools = [as_Carpool(cp) for cp in json_results] + + return carpools + else: + logger.error("ride2go request returned with status_code %s", result.status_code) + json_results = result.json() + if 'status' in json_results: + logger.error("Error was: %s", result.json()['status']) + + raise ValueError("Sync failed with error. See logs") + + except BaseException as e: + logger.exception("Error on import for agency ride2go") + raise e diff --git a/amarillo/app/tests/sampledata.py b/amarillo/app/tests/sampledata.py new file mode 100644 index 0000000..407eba9 --- /dev/null +++ b/amarillo/app/tests/sampledata.py @@ -0,0 +1,91 @@ +from amarillo.app.models.Carpool import Carpool, StopTime, Weekday + +# TODO use meanigful values for id and lat, lon +stops_1234 = [ + StopTime( + id="de:08115:4802:0:3", + name="Herrenberg", + lat=48.5948979, + lon=8.8684534), + StopTime( + id="de:08111:6221:3:6", + name="Stuttgart Feuersee", + lat= 48.7733275, + lon=9.1671590)] + +carpool_1234 = Carpool( + id="1234", + agency="mfdz", + deeplink="https://mfdz.de/trip/1234", + stops=stops_1234, + departureTime="07:00", + departureDate="2022-03-30", +) + +carpool_repeating = Carpool( + id="12345", + agency="mfdz", + deeplink="https://mfdz.de/trip/12345", + stops=stops_1234, + departureTime="06:00", + departureDate=[Weekday.monday, Weekday.tuesday, Weekday.wednesday, + Weekday.thursday, Weekday.friday], +) + +examples = { + "one-time trip with date": { + "summary": "one-time trip with date", + "description": "carpool object that should to be added or modified", + "value": carpool_1234}, + "repeating trip Mon-Fri": { + "summary": "repeating trip Mon-Fri", + "description": "carpool object that should to be added or modified", + "value": carpool_repeating} +} + +data1 = { + 'id': "Eins", + 'agency': "mfdz", + 'deeplink': "https://mfdz.de/trip/123", + 'stops': [ + {'id': "mfdz:12073:001", 'name': "abc", 'lat': 53.11901, 'lon': 14.015776}, + {'id': "de:12073:900340137::3", 'name': "xyz", 'lat': 53.011459, 'lon': 13.94945}], + 'departureTime': "23:59", + 'departureDate': "2022-05-30", +} + +carpool_repeating_json = { + 'id': "Zwei", + 'agency': "mfdz", + 'deeplink': "https://mfdz.de/trip/123", + 'stops': [ + {'id': "mfdz:12073:001", 'name': "abc", 'lat': 53.11901, 'lon': 14.015776}, + {'id': "de:12073:900340137::3", 'name': "xyz", 'lat': 53.011459, 'lon': 13.94945}], + 'departureTime': "15:00", + 'departureDate': ["monday"], +} + + +cp1 = Carpool(**data1) + +# JSON string for trying out the API in Swagger +cp2 = """ +{ + "id": "Vier", + "agency": "string", + "deeplink": "http://mfdz.de", + "stops": [ + { + "id": "de:12073:900340137::4", "name": "drei", "lat": 45, "lon": 9 + }, + { + "id": "de:12073:900340137::5", "name": "drei b", "lat": 45, "lon": 9 + } + ], + "departureTime": "12:34", + "departureDate": "2022-03-30", + "lastUpdated": "2022-03-30 12:34" +} +""" + +stop_issue = {"id": "106727", "agency": "ride2go", "deeplink": "https://ride2go.com/?trip=106727", "stops": [{"id": None, "name": "Mitfahrbank Angerm\u00fcnde, Einkaufscenter, Prenzlauer Stra\u00dfe, 16278 Angerm\u00fcnde", "departureTime": None, "arrivalTime": None, "lat": 53.0220209, "lon": 13.9999447, "pickup_dropoff": None}, {"id": None, "name": "Mitfahrbank B\u00f6lkendorf, B\u00f6lkendorfer Stra\u00dfe, 16278 Angerm\u00fcnde", "departureTime": None, "arrivalTime": None, "lat": 52.949856, "lon": 14.003533, "pickup_dropoff": None}], "departureTime": "17:00:00", "departureDate": "2022-06-22", "path": None, "lastUpdated": "2022-06-22T11:04:22"} \ No newline at end of file