From ece0dd89fe7c7c3cbba134b355c007756a43557a Mon Sep 17 00:00:00 2001 From: Csaba Date: Thu, 7 Dec 2023 16:12:46 +0100 Subject: [PATCH 01/33] Initial commit --- .gitignore | 162 +++++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 3 + 2 files changed, 165 insertions(+) create mode 100644 .gitignore create mode 100644 README.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5d381cc --- /dev/null +++ b/.gitignore @@ -0,0 +1,162 @@ +# ---> Python +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + diff --git a/README.md b/README.md new file mode 100644 index 0000000..d96cdc1 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# amarillo-enhancer + +Trip enhancement plugin for Amarillo \ No newline at end of file From d845dec08bf2830937f3a921d0752f31564d1ba7 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Fri, 8 Dec 2023 16:16:41 +0100 Subject: [PATCH 02/33] Added pyproject.toml --- pyproject.toml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..7da48e0 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,7 @@ +[project] +name = "amarillo-enhancer" +version = "0.0.1" +dependencies = [ + "amarillo-core", + "watchdog", +] \ No newline at end of file From 5747854755d6503c712ca7c92b68dc9c8e21853f Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Mon, 11 Dec 2023 12:32:30 +0100 Subject: [PATCH 03/33] Added gtfs_export and gtfs_generator --- amarillo/app/services/gtfs_export.py | 229 ++++++++++++++++++++++++ amarillo/app/services/gtfs_generator.py | 71 ++++++++ 2 files changed, 300 insertions(+) create mode 100644 amarillo/app/services/gtfs_export.py create mode 100644 amarillo/app/services/gtfs_generator.py diff --git a/amarillo/app/services/gtfs_export.py b/amarillo/app/services/gtfs_export.py new file mode 100644 index 0000000..b60af4e --- /dev/null +++ b/amarillo/app/services/gtfs_export.py @@ -0,0 +1,229 @@ + +from collections.abc import Iterable +from datetime import datetime, timedelta +from zipfile import ZipFile +import csv +import gettext +import logging +import re + +from app.utils.utils import assert_folder_exists +from app.models.gtfs import GtfsTimeDelta, GtfsFeedInfo, GtfsAgency, GtfsRoute, GtfsStop, GtfsStopTime, GtfsTrip, GtfsCalendar, GtfsCalendarDate, GtfsShape +from app.services.stops import is_carpooling_stop +from app.services.gtfs_constants import * + + +logger = logging.getLogger(__name__) + +class GtfsExport: + + stops_counter = 0 + trips_counter = 0 + routes_counter = 0 + + stored_stops = {} + + def __init__(self, agencies, feed_info, ridestore, stopstore, bbox = None): + self.stops = {} + self.routes = [] + self.calendar_dates = [] + self.calendar = [] + self.trips = [] + self.stop_times = [] + self.calendar = [] + self.shapes = [] + self.agencies = agencies + self.feed_info = feed_info + self.localized_to = " nach " + self.localized_short_name = "Mitfahrgelegenheit" + self.stopstore = stopstore + self.ridestore = ridestore + self.bbox = bbox + + def export(self, gtfszip_filename, gtfsfolder): + assert_folder_exists(gtfsfolder) + self._prepare_gtfs_feed(self.ridestore, self.stopstore) + self._write_csvfile(gtfsfolder, 'agency.txt', self.agencies) + self._write_csvfile(gtfsfolder, 'feed_info.txt', self.feed_info) + self._write_csvfile(gtfsfolder, 'routes.txt', self.routes) + self._write_csvfile(gtfsfolder, 'trips.txt', self.trips) + self._write_csvfile(gtfsfolder, 'calendar.txt', self.calendar) + self._write_csvfile(gtfsfolder, 'calendar_dates.txt', self.calendar_dates) + self._write_csvfile(gtfsfolder, 'stops.txt', self.stops.values()) + self._write_csvfile(gtfsfolder, 'stop_times.txt', self.stop_times) + self._write_csvfile(gtfsfolder, 'shapes.txt', self.shapes) + self._zip_files(gtfszip_filename, gtfsfolder) + + def _zip_files(self, gtfszip_filename, gtfsfolder): + gtfsfiles = ['agency.txt', 'feed_info.txt', 'routes.txt', 'trips.txt', + 'calendar.txt', 'calendar_dates.txt', 'stops.txt', 'stop_times.txt', 'shapes.txt'] + with ZipFile(gtfszip_filename, 'w') as gtfszip: + for gtfsfile in gtfsfiles: + gtfszip.write(gtfsfolder+'/'+gtfsfile, gtfsfile) + + def _prepare_gtfs_feed(self, ridestore, stopstore): + """ + Prepares all gtfs objects in memory before they are written + to their respective streams. + + For all wellknown stops a GTFS stop is created and + afterwards all ride offers are transformed into their + gtfs equivalents. + """ + for stopSet in stopstore.stopsDataFrames: + for stop in stopSet["stops"].itertuples(): + self._load_stored_stop(stop) + cloned_trips = dict(ridestore.trips) + for url, trip in cloned_trips.items(): + if self.bbox is None or trip.intersects(self.bbox): + self._convert_trip(trip) + + def _convert_trip(self, trip): + self.routes_counter += 1 + self.routes.append(self._create_route(trip)) + self.calendar.append(self._create_calendar(trip)) + if not trip.runs_regularly: + self.calendar_dates.append(self._create_calendar_date(trip)) + self.trips.append(self._create_trip(trip, self.routes_counter)) + self._append_stops_and_stop_times(trip) + self._append_shapes(trip, self.routes_counter) + + def _trip_headsign(self, destination): + destination = destination.replace('(Deutschland)', '') + destination = destination.replace(', Deutschland', '') + appendix = '' + if 'Schweiz' in destination or 'Switzerland' in destination: + appendix = ', Schweiz' + destination = destination.replace('(Schweiz)', '') + destination = destination.replace(', Schweiz', '') + destination = destination.replace('(Switzerland)', '') + + try: + matches = re.match(r"(.*,)? ?(\d{4,5})? ?(.*)", destination) + + match = matches.group(3).strip() if matches != None else destination.strip() + if match[-1]==')' and not '(' in match: + match = match[0:-1] + + return match + appendix + except Exception as ex: + logger.error("error for "+destination ) + logger.exception(ex) + return destination + + def _create_route(self, trip): + return GtfsRoute(trip.agency, trip.trip_id, trip.route_long_name(), RIDESHARING_ROUTE_TYPE, trip.url, "") + + def _create_calendar(self, trip): + # TODO currently, calendar is not provided by Fahrgemeinschaft.de interface. + # We could apply some heuristics like requesting multiple days and extrapolate + # if multiple trips are found, but better would be to have these provided by the + # offical interface. Then validity periods should be provided as well (not + # sure if these are available) + # For fahrgemeinschaft.de, regurlar trips are recognizable via their url + # which contains "regelmaessig". However, we don't know on which days of the week, + # nor until when. As a first guess, if datetime is a mo-fr, we assume each workday, + # if it's sa/su, only this... + + feed_start_date = datetime.today() + stop_date = self._convert_stop_date(feed_start_date) + return GtfsCalendar(trip.trip_id, stop_date, self._convert_stop_date(feed_start_date + timedelta(days=31)), *(trip.weekdays)) + + def _create_calendar_date(self, trip): + return GtfsCalendarDate(trip.trip_id, self._convert_stop_date(trip.start), CALENDAR_DATES_EXCEPTION_TYPE_ADDED) + + def _create_trip(self, trip, shape_id): + return GtfsTrip(trip.trip_id, trip.trip_id, trip.trip_id, shape_id, trip.trip_headsign, NO_BIKES_ALLOWED) + + def _convert_stop(self, stop): + """ + Converts a stop represented as pandas row to a gtfs stop. + Expected attributes of stop: id, stop_name, x, y (in wgs84) + """ + if stop.id: + id = stop.id + else: + self.stops_counter += 1 + id = "tmp-{}".format(self.stops_counter) + + stop_name = "k.A." if stop.stop_name is None else stop.stop_name + return GtfsStop(id, stop.y, stop.x, stop_name) + + def _append_stops_and_stop_times(self, trip): + # Assumptions: + # arrival_time = departure_time + # pickup_type, drop_off_type for origin: = coordinate/none + # pickup_type, drop_off_type for destination: = none/coordinate + # timepoint = approximate for origin and destination (not sure what consequences this might have for trip planners) + for stop_time in trip.stop_times: + # retrieve stop from stored_stops and mark it to be exported + wkn_stop = self.stored_stops.get(stop_time.stop_id) + if not wkn_stop: + logger.warning("No stop found in stop_store for %s. Will skip stop_time %s of trip %s", stop_time.stop_id, stop_time.stop_sequence, trip.trip_id) + else: + self.stops[stop_time.stop_id] = wkn_stop + # Append stop_time + self.stop_times.append(stop_time) + + def _append_shapes(self, trip, shape_id): + counter = 0 + for point in trip.path.coordinates: + counter += 1 + self.shapes.append(GtfsShape(shape_id, point[0], point[1], counter)) + + def _stop_hash(self, stop): + return "{}#{}#{}".format(stop.stop_name,stop.x,stop.y) + + def _should_always_export(self, stop): + """ + Returns true, if the given stop shall be exported to GTFS, + regardless, if it's part of a trip or not. + + This is necessary, as potential stops are required + to be part of the GTFS to be referenced later on + by dynamicly added trips. + """ + if self.bbox: + return (self.bbox[0] <= stop.stop_lon <= self.bbox[2] and + self.bbox[1] <= stop.stop_lat <= self.bbox[3]) + else: + return is_carpooling_stop(stop.stop_id, stop.stop_name) + + def _load_stored_stop(self, stop): + gtfsstop = self._convert_stop(stop) + stop_hash = self._stop_hash(stop) + self.stored_stops[gtfsstop.stop_id] = gtfsstop + if self._should_always_export(gtfsstop): + self.stops[gtfsstop.stop_id] = gtfsstop + + def _get_stop_by_hash(self, stop_hash): + return self.stops.get(stop_hash, self.stored_stops.get(stop_hash)) + + def _get_or_create_stop(self, stop): + stop_hash = self._stop_hash(stop) + gtfsstop = self.stops.get(stop_hash) + if gtfsstop is None: + gtfsstop = self.stored_stops.get(stop_hash, self._convert_stop(stop)) + self.stops[stop_hash] = gtfsstop + return gtfsstop + + def _convert_stop_date(self, date_time): + return date_time.strftime("%Y%m%d") + + def _write_csvfile(self, gtfsfolder, filename, content): + with open(gtfsfolder+"/"+filename, 'w', newline="\n", encoding="utf-8") as csvfile: + self._write_csv(csvfile, content) + + def _write_csv(self, csvfile, content): + if hasattr(content, '_fields'): + writer = csv.DictWriter(csvfile, content._fields) + writer.writeheader() + writer.writerow(content._asdict()) + else: + if content: + writer = csv.DictWriter(csvfile, next(iter(content))._fields) + writer.writeheader() + for record in content: + writer.writerow(record._asdict()) + + \ No newline at end of file diff --git a/amarillo/app/services/gtfs_generator.py b/amarillo/app/services/gtfs_generator.py new file mode 100644 index 0000000..2698494 --- /dev/null +++ b/amarillo/app/services/gtfs_generator.py @@ -0,0 +1,71 @@ +from app.models.Carpool import Region +from app.services.gtfs_export import GtfsExport, GtfsFeedInfo, GtfsAgency +from app.services.gtfs import GtfsRtProducer +from app.utils.container import container +from glob import glob +import json +import schedule +import threading +import time +import logging +from datetime import date, timedelta + +logger = logging.getLogger(__name__) + +regions = {} +for region_file_name in glob('conf/region/*.json'): + with open(region_file_name) as region_file: + dict = json.load(region_file) + region = Region(**dict) + region_id = region.id + regions[region_id] = region + +agencies = [] +for agency_file_name in glob('conf/agency/*.json'): + with open(agency_file_name) as agency_file: + dict = json.load(agency_file) + agency = GtfsAgency(dict["id"], dict["name"], dict["url"], dict["timezone"], dict["lang"], dict["email"]) + agency_id = agency.agency_id + agencies.append(agency) + +def run_schedule(): + while 1: + try: + schedule.run_pending() + except Exception as e: + logger.exception(e) + time.sleep(1) + +def midnight(): + container['stops_store'].load_stop_sources() + container['trips_store'].unflag_unrecent_updates() + container['carpools'].purge_outdated_offers() + generate_gtfs() + +def generate_gtfs(): + logger.info("Generate GTFS") + + for region in regions.values(): + # TODO make feed producer infos configurable + feed_info = GtfsFeedInfo('mfdz', 'MITFAHR|DE|ZENTRALE', 'http://www.mitfahrdezentrale.de', 'de', 1) + exporter = GtfsExport( + agencies, + feed_info, + container['trips_store'], + container['stops_store'], + region.bbox) + exporter.export(f"data/gtfs/amarillo.{region.id}.gtfs.zip", "data/tmp/") + +def generate_gtfs_rt(): + logger.info("Generate GTFS-RT") + producer = GtfsRtProducer(container['trips_store']) + for region in regions.values(): + rt = producer.export_feed(time.time(), f"data/gtfs/amarillo.{region.id}.gtfsrt", bbox=region.bbox) + +def start_schedule(): + schedule.every().day.at("00:00").do(midnight) + schedule.every(60).seconds.do(generate_gtfs_rt) + # Create all feeds once at startup + schedule.run_all() + job_thread = threading.Thread(target=run_schedule, daemon=True) + job_thread.start() \ No newline at end of file From 11142f496102b99add8be9bc7e509765702fff68 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Mon, 11 Dec 2023 13:13:29 +0100 Subject: [PATCH 04/33] Added enhancer.py and enhancer configuration --- amarillo/plugins/enhancer.py | 70 ++++++++++++++++++++++ amarillo/plugins/enhancer_configuration.py | 51 ++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 amarillo/plugins/enhancer.py create mode 100644 amarillo/plugins/enhancer_configuration.py diff --git a/amarillo/plugins/enhancer.py b/amarillo/plugins/enhancer.py new file mode 100644 index 0000000..4f22248 --- /dev/null +++ b/amarillo/plugins/enhancer.py @@ -0,0 +1,70 @@ +import json +import time +import logging +import logging.config +from watchdog.observers import Observer +from watchdog.events import FileSystemEventHandler + +from enhancer_configuration import configure_enhancer_services +from amarillo.app.utils.container import container +from amarillo.app.models.Carpool import Carpool +from amarillo.app.utils.utils import agency_carpool_ids_from_filename + +logging.config.fileConfig('logging.conf', disable_existing_loggers=False) +logger = logging.getLogger("enhancer") + +logger.info("Hello Enhancer") + +configure_enhancer_services() + +observer = Observer() # Watch Manager + + +class EventHandler(FileSystemEventHandler): + # TODO FG HB should watch for both carpools and agencies + # in data/agency, data/agencyconf, see AgencyConfService + + def on_closed(self, event): + + logger.info("CLOSE_WRITE: Created %s", event.src_path) + try: + with open(event.src_path, 'r', encoding='utf-8') as f: + dict = json.load(f) + carpool = Carpool(**dict) + + container['carpools'].put(carpool.agency, carpool.id, carpool) + except FileNotFoundError as e: + logger.error("Carpool could not be added, as already deleted (%s)", event.src_path) + except: + logger.exception("Eventhandler on_closed encountered exception") + + def on_deleted(self, event): + try: + logger.info("DELETE: Removing %s", event.src_path) + (agency_id, carpool_id) = agency_carpool_ids_from_filename(event.src_path) + container['carpools'].delete(agency_id, carpool_id) + except: + logger.exception("Eventhandler on_deleted encountered exception") + + +observer.schedule(EventHandler(), 'data/carpool', recursive=True) +observer.start() + +import time + +try: + # TODO FG Is this really needed? + cnt = 0 + ENHANCER_LOG_INTERVAL_IN_S = 600 + while True: + if cnt == ENHANCER_LOG_INTERVAL_IN_S: + logger.debug("Currently stored carpool ids: %s", container['carpools'].get_all_ids()) + cnt = 0 + + time.sleep(1) + cnt += 1 +finally: + observer.stop() + observer.join() + + logger.info("Goodbye Enhancer") diff --git a/amarillo/plugins/enhancer_configuration.py b/amarillo/plugins/enhancer_configuration.py new file mode 100644 index 0000000..5a19df7 --- /dev/null +++ b/amarillo/plugins/enhancer_configuration.py @@ -0,0 +1,51 @@ +# separate file so that it can be imported without initializing FastAPI +from amarillo.app.utils.container import container +import json +import logging +from glob import glob + +from amarillo.app.models.Carpool import Carpool +from amarillo.app.services import stops +from amarillo.app.services import trips +from amarillo.app.services.carpools import CarpoolService + +from amarillo.app.configuration import configure_services + +logger = logging.getLogger(__name__) + + +def configure_enhancer_services(): + configure_services() + + logger.info("Load stops...") + stop_sources = [ + {"url": "https://datahub.bbnavi.de/export/rideshare_points.geojson", "vicinity": 50}, + {"url": "https://data.mfdz.de/mfdz/stops/stops_zhv.csv", "vicinity": 50}, + {"url": "https://data.mfdz.de/mfdz/stops/parkings_osm.csv", "vicinity": 500}, + ] + stop_store = stops.StopsStore(stop_sources) + + stop_store.load_stop_sources() + container['stops_store'] = stop_store + container['trips_store'] = trips.TripStore(stop_store) + container['carpools'] = CarpoolService(container['trips_store']) + + logger.info("Restore carpools...") + + for agency_id in container['agencies'].agencies: + for carpool_file_name in glob(f'data/carpool/{agency_id}/*.json'): + try: + with open(carpool_file_name) as carpool_file: + carpool = Carpool(**(json.load(carpool_file))) + container['carpools'].put(carpool.agency, carpool.id, carpool) + except Exception as e: + logger.warning("Issue during restore of carpool %s: %s", carpool_file_name, repr(e)) + + # notify carpool about carpools in trash, as delete notifications must be sent + for carpool_file_name in glob(f'data/trash/{agency_id}/*.json'): + with open(carpool_file_name) as carpool_file: + carpool = Carpool(**(json.load(carpool_file))) + container['carpools'].delete(carpool.agency, carpool.id) + + logger.info("Restored carpools: %s", container['carpools'].get_all_ids()) + logger.info("Starting scheduler") \ No newline at end of file From bd36c1ec704b8224c3a05d6809a6047100aad0d2 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Mon, 11 Dec 2023 13:49:46 +0100 Subject: [PATCH 05/33] Added requirements.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..cce0be8 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +watchdog==3.0.0 \ No newline at end of file From c18f347aac445ca44b5464f7ff05e71c43d42956 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Mon, 11 Dec 2023 14:55:35 +0100 Subject: [PATCH 06/33] Moved code to plugins/enhancer folder --- amarillo/plugins/__init__.py | 1 + .../configuration.py} | 4 +- amarillo/plugins/{ => enhancer}/enhancer.py | 60 +++++++++++-------- 3 files changed, 38 insertions(+), 27 deletions(-) create mode 100644 amarillo/plugins/__init__.py rename amarillo/plugins/{enhancer_configuration.py => enhancer/configuration.py} (94%) rename amarillo/plugins/{ => enhancer}/enhancer.py (62%) diff --git a/amarillo/plugins/__init__.py b/amarillo/plugins/__init__.py new file mode 100644 index 0000000..0260537 --- /dev/null +++ b/amarillo/plugins/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) \ No newline at end of file diff --git a/amarillo/plugins/enhancer_configuration.py b/amarillo/plugins/enhancer/configuration.py similarity index 94% rename from amarillo/plugins/enhancer_configuration.py rename to amarillo/plugins/enhancer/configuration.py index 5a19df7..6e6760a 100644 --- a/amarillo/plugins/enhancer_configuration.py +++ b/amarillo/plugins/enhancer/configuration.py @@ -8,6 +8,7 @@ from amarillo.app.models.Carpool import Carpool from amarillo.app.services import stops from amarillo.app.services import trips from amarillo.app.services.carpools import CarpoolService +from amarillo.app.services import gtfs_generator from amarillo.app.configuration import configure_services @@ -48,4 +49,5 @@ def configure_enhancer_services(): container['carpools'].delete(carpool.agency, carpool.id) logger.info("Restored carpools: %s", container['carpools'].get_all_ids()) - logger.info("Starting scheduler") \ No newline at end of file + logger.info("Starting scheduler") + gtfs_generator.start_schedule() \ No newline at end of file diff --git a/amarillo/plugins/enhancer.py b/amarillo/plugins/enhancer/enhancer.py similarity index 62% rename from amarillo/plugins/enhancer.py rename to amarillo/plugins/enhancer/enhancer.py index 4f22248..c07a8b0 100644 --- a/amarillo/plugins/enhancer.py +++ b/amarillo/plugins/enhancer/enhancer.py @@ -1,11 +1,11 @@ import json -import time +from threading import Thread import logging import logging.config from watchdog.observers import Observer from watchdog.events import FileSystemEventHandler -from enhancer_configuration import configure_enhancer_services +from amarillo.plugins.enhancer.configuration import configure_enhancer_services from amarillo.app.utils.container import container from amarillo.app.models.Carpool import Carpool from amarillo.app.utils.utils import agency_carpool_ids_from_filename @@ -13,13 +13,6 @@ from amarillo.app.utils.utils import agency_carpool_ids_from_filename logging.config.fileConfig('logging.conf', disable_existing_loggers=False) logger = logging.getLogger("enhancer") -logger.info("Hello Enhancer") - -configure_enhancer_services() - -observer = Observer() # Watch Manager - - class EventHandler(FileSystemEventHandler): # TODO FG HB should watch for both carpools and agencies # in data/agency, data/agencyconf, see AgencyConfService @@ -47,24 +40,39 @@ class EventHandler(FileSystemEventHandler): logger.exception("Eventhandler on_deleted encountered exception") -observer.schedule(EventHandler(), 'data/carpool', recursive=True) -observer.start() +def run_enhancer(): + logger.info("Hello Enhancer") -import time + configure_enhancer_services() -try: - # TODO FG Is this really needed? - cnt = 0 - ENHANCER_LOG_INTERVAL_IN_S = 600 - while True: - if cnt == ENHANCER_LOG_INTERVAL_IN_S: - logger.debug("Currently stored carpool ids: %s", container['carpools'].get_all_ids()) - cnt = 0 + observer = Observer() # Watch Manager - time.sleep(1) - cnt += 1 -finally: - observer.stop() - observer.join() + observer.schedule(EventHandler(), 'data/carpool', recursive=True) + observer.start() - logger.info("Goodbye Enhancer") + import time + + try: + # TODO FG Is this really needed? + cnt = 0 + ENHANCER_LOG_INTERVAL_IN_S = 600 + while True: + if cnt == ENHANCER_LOG_INTERVAL_IN_S: + logger.debug("Currently stored carpool ids: %s", container['carpools'].get_all_ids()) + cnt = 0 + + time.sleep(1) + cnt += 1 + finally: + observer.stop() + observer.join() + + logger.info("Goodbye Enhancer") + +def setup(app): + thread = Thread(target=run_enhancer) + thread.start() + + +if __name__ == "__main__": + run_enhancer() \ No newline at end of file From a2543a1aa3bd701b4b1e7c4e380e17999a192f5b Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Mon, 11 Dec 2023 14:55:50 +0100 Subject: [PATCH 07/33] Added schedule dependency --- pyproject.toml | 3 ++- requirements.txt | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7da48e0..114ae51 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,8 @@ [project] name = "amarillo-enhancer" -version = "0.0.1" +version = "0.0.2" dependencies = [ "amarillo-core", "watchdog", + "schedule==1.2.1", ] \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index cce0be8..37767e7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ -watchdog==3.0.0 \ No newline at end of file +watchdog==3.0.0 +schedule==1.2.1 \ No newline at end of file From 04437b107ab87a59fdc18f7c5a358d8a93155365 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Mon, 11 Dec 2023 15:01:27 +0100 Subject: [PATCH 08/33] gtfs_export and gtfs_generator --- amarillo/app/__init__.py | 0 amarillo/app/services/__init__.py | 0 amarillo/app/services/gtfs_export.py | 8 ++++---- amarillo/app/services/gtfs_generator.py | 8 ++++---- 4 files changed, 8 insertions(+), 8 deletions(-) create mode 100644 amarillo/app/__init__.py create mode 100644 amarillo/app/services/__init__.py diff --git a/amarillo/app/__init__.py b/amarillo/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/amarillo/app/services/__init__.py b/amarillo/app/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/amarillo/app/services/gtfs_export.py b/amarillo/app/services/gtfs_export.py index b60af4e..9d9aea5 100644 --- a/amarillo/app/services/gtfs_export.py +++ b/amarillo/app/services/gtfs_export.py @@ -7,10 +7,10 @@ import gettext import logging import re -from app.utils.utils import assert_folder_exists -from app.models.gtfs import GtfsTimeDelta, GtfsFeedInfo, GtfsAgency, GtfsRoute, GtfsStop, GtfsStopTime, GtfsTrip, GtfsCalendar, GtfsCalendarDate, GtfsShape -from app.services.stops import is_carpooling_stop -from app.services.gtfs_constants import * +from amarillo.app.utils.utils import assert_folder_exists +from amarillo.app.models.gtfs import GtfsTimeDelta, GtfsFeedInfo, GtfsAgency, GtfsRoute, GtfsStop, GtfsStopTime, GtfsTrip, GtfsCalendar, GtfsCalendarDate, GtfsShape +from amarillo.app.services.stops import is_carpooling_stop +from amarillo.app.services.gtfs_constants import * logger = logging.getLogger(__name__) diff --git a/amarillo/app/services/gtfs_generator.py b/amarillo/app/services/gtfs_generator.py index 2698494..1459aca 100644 --- a/amarillo/app/services/gtfs_generator.py +++ b/amarillo/app/services/gtfs_generator.py @@ -1,7 +1,7 @@ -from app.models.Carpool import Region -from app.services.gtfs_export import GtfsExport, GtfsFeedInfo, GtfsAgency -from app.services.gtfs import GtfsRtProducer -from app.utils.container import container +from amarillo.app.models.Carpool import Region +from amarillo.app.services.gtfs_export import GtfsExport, GtfsFeedInfo, GtfsAgency +from amarillo.app.services.gtfs import GtfsRtProducer +from amarillo.app.utils.container import container from glob import glob import json import schedule From 4770845938efb713ea1cecde407b1ae76d0e027a Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Mon, 11 Dec 2023 15:17:55 +0100 Subject: [PATCH 09/33] Moved gtfs_export and gtfs_generator to enhancer/services --- amarillo/app/services/__init__.py | 0 amarillo/plugins/enhancer/__init__.py | 1 + amarillo/plugins/enhancer/configuration.py | 2 +- amarillo/{app => plugins/enhancer/services}/__init__.py | 0 amarillo/{app => plugins/enhancer}/services/gtfs_export.py | 0 amarillo/{app => plugins/enhancer}/services/gtfs_generator.py | 2 +- 6 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 amarillo/app/services/__init__.py create mode 100644 amarillo/plugins/enhancer/__init__.py rename amarillo/{app => plugins/enhancer/services}/__init__.py (100%) rename amarillo/{app => plugins/enhancer}/services/gtfs_export.py (100%) rename amarillo/{app => plugins/enhancer}/services/gtfs_generator.py (94%) diff --git a/amarillo/app/services/__init__.py b/amarillo/app/services/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/amarillo/plugins/enhancer/__init__.py b/amarillo/plugins/enhancer/__init__.py new file mode 100644 index 0000000..c8e9291 --- /dev/null +++ b/amarillo/plugins/enhancer/__init__.py @@ -0,0 +1 @@ +from .enhancer import * \ No newline at end of file diff --git a/amarillo/plugins/enhancer/configuration.py b/amarillo/plugins/enhancer/configuration.py index 6e6760a..0e19314 100644 --- a/amarillo/plugins/enhancer/configuration.py +++ b/amarillo/plugins/enhancer/configuration.py @@ -8,7 +8,7 @@ from amarillo.app.models.Carpool import Carpool from amarillo.app.services import stops from amarillo.app.services import trips from amarillo.app.services.carpools import CarpoolService -from amarillo.app.services import gtfs_generator +from amarillo.plugins.enhancer.services import gtfs_generator from amarillo.app.configuration import configure_services diff --git a/amarillo/app/__init__.py b/amarillo/plugins/enhancer/services/__init__.py similarity index 100% rename from amarillo/app/__init__.py rename to amarillo/plugins/enhancer/services/__init__.py diff --git a/amarillo/app/services/gtfs_export.py b/amarillo/plugins/enhancer/services/gtfs_export.py similarity index 100% rename from amarillo/app/services/gtfs_export.py rename to amarillo/plugins/enhancer/services/gtfs_export.py diff --git a/amarillo/app/services/gtfs_generator.py b/amarillo/plugins/enhancer/services/gtfs_generator.py similarity index 94% rename from amarillo/app/services/gtfs_generator.py rename to amarillo/plugins/enhancer/services/gtfs_generator.py index 1459aca..604faad 100644 --- a/amarillo/app/services/gtfs_generator.py +++ b/amarillo/plugins/enhancer/services/gtfs_generator.py @@ -1,5 +1,5 @@ from amarillo.app.models.Carpool import Region -from amarillo.app.services.gtfs_export import GtfsExport, GtfsFeedInfo, GtfsAgency +from amarillo.plugins.enhancer.services.gtfs_export import GtfsExport, GtfsFeedInfo, GtfsAgency from amarillo.app.services.gtfs import GtfsRtProducer from amarillo.app.utils.container import container from glob import glob From b422be2b36fbb102d8bb6585fd41b016894810d6 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Fri, 19 Jan 2024 15:16:57 +0100 Subject: [PATCH 10/33] #28 move CarpoolService and Tripstore from core to enhancer --- amarillo/plugins/enhancer/configuration.py | 4 +- .../plugins/enhancer/services/carpools.py | 60 +++ amarillo/plugins/enhancer/services/trips.py | 376 ++++++++++++++++++ 3 files changed, 438 insertions(+), 2 deletions(-) create mode 100644 amarillo/plugins/enhancer/services/carpools.py create mode 100644 amarillo/plugins/enhancer/services/trips.py diff --git a/amarillo/plugins/enhancer/configuration.py b/amarillo/plugins/enhancer/configuration.py index 0e19314..821e365 100644 --- a/amarillo/plugins/enhancer/configuration.py +++ b/amarillo/plugins/enhancer/configuration.py @@ -6,8 +6,8 @@ from glob import glob from amarillo.app.models.Carpool import Carpool from amarillo.app.services import stops -from amarillo.app.services import trips -from amarillo.app.services.carpools import CarpoolService +from amarillo.plugins.enhancer.services import trips +from amarillo.plugins.enhancer.services.carpools import CarpoolService from amarillo.plugins.enhancer.services import gtfs_generator from amarillo.app.configuration import configure_services diff --git a/amarillo/plugins/enhancer/services/carpools.py b/amarillo/plugins/enhancer/services/carpools.py new file mode 100644 index 0000000..5d6b97c --- /dev/null +++ b/amarillo/plugins/enhancer/services/carpools.py @@ -0,0 +1,60 @@ +import json +import logging +from datetime import datetime +from typing import Dict +from amarillo.app.models.Carpool import Carpool +from amarillo.app.utils.utils import yesterday, is_older_than_days + +logger = logging.getLogger(__name__) + +class CarpoolService(): + MAX_OFFER_AGE_IN_DAYS = 180 + + def __init__(self, trip_store): + + self.trip_store = trip_store + self.carpools: Dict[str, Carpool] = {} + + def is_outdated(self, carpool): + """ + A carpool offer is outdated, if + * it's completly in the past (if it's a single date offer). + As we know the start time but not latest arrival, we deem + offers starting the day before yesterday as outdated + * it's last update occured before MAX_OFFER_AGE_IN_DAYS + """ + runs_once = not isinstance(carpool.departureDate, set) + return (is_older_than_days(carpool.lastUpdated.date(), self.MAX_OFFER_AGE_IN_DAYS) or + (runs_once and carpool.departureDate < yesterday())) + + def purge_outdated_offers(self): + """ + Iterates over all carpools and deletes those which are outdated + """ + for key in list(self.carpools.keys()): + cp = self.carpools.get(key) + if cp and self.is_outdated(cp): + logger.info("Purge outdated offer %s", key) + self.delete(cp.agency, cp.id) + + def get(self, agency_id: str, carpool_id: str): + return self.carpools.get(f"{agency_id}:{carpool_id}") + + def get_all_ids(self): + return list(self.carpools) + + def put(self, agency_id: str, carpool_id: str, carpool): + self.carpools[f"{agency_id}:{carpool_id}"] = carpool + # Outdated trips (which might have been in the store) + # will be deleted + if self.is_outdated(carpool): + logger.info('Deleting outdated carpool %s:%s', agency_id, carpool_id) + self.delete(agency_id, carpool_id) + else: + self.trip_store.put_carpool(carpool) + + def delete(self, agency_id: str, carpool_id: str): + id = f"{agency_id}:{carpool_id}" + if id in self.carpools: + del self.carpools[id] + self.trip_store.delete_carpool(agency_id, carpool_id) diff --git a/amarillo/plugins/enhancer/services/trips.py b/amarillo/plugins/enhancer/services/trips.py new file mode 100644 index 0000000..6fb58ff --- /dev/null +++ b/amarillo/plugins/enhancer/services/trips.py @@ -0,0 +1,376 @@ +from amarillo.app.models.gtfs import GtfsTimeDelta, GtfsStopTime +from amarillo.app.models.Carpool import MAX_STOPS_PER_TRIP, Carpool, Weekday, StopTime, PickupDropoffType +from amarillo.app.services.gtfs_constants import * +from amarillo.app.services.routing import RoutingService, RoutingException +from amarillo.app.services.stops import is_carpooling_stop +from amarillo.app.utils.utils import assert_folder_exists, is_older_than_days, yesterday, geodesic_distance_in_m +from shapely.geometry import Point, LineString, box +from geojson_pydantic.geometries import LineString as GeoJSONLineString +from datetime import datetime, timedelta +import numpy as np +import os +import json +import logging + +logger = logging.getLogger(__name__) + +class Trip: + + # TODO: add driver attributes, additional ridesharing info + def __init__(self, trip_id, route_name, headsign, url, calendar, departureTime, path, agency, lastUpdated, stop_times, bbox): + if isinstance(calendar, set): + self.runs_regularly = True + self.weekdays = [ + 1 if Weekday.monday in calendar else 0, + 1 if Weekday.tuesday in calendar else 0, + 1 if Weekday.wednesday in calendar else 0, + 1 if Weekday.thursday in calendar else 0, + 1 if Weekday.friday in calendar else 0, + 1 if Weekday.saturday in calendar else 0, + 1 if Weekday.sunday in calendar else 0, + ] + start_in_day = self._total_seconds(departureTime) + else: + self.start = datetime.combine(calendar, departureTime) + self.runs_regularly = False + self.weekdays = [0,0,0,0,0,0,0] + + self.start_time = departureTime + self.path = path + self.trip_id = trip_id + self.url = url + self.agency = agency + self.stops = [] + self.lastUpdated = lastUpdated + self.stop_times = stop_times + self.bbox = bbox + self.route_name = route_name + self.trip_headsign = headsign + + def path_as_line_string(self): + return self.path + + def _total_seconds(self, instant): + return instant.hour * 3600 + instant.minute * 60 + instant.second + + def start_time_str(self): + return self.start_time.strftime("%H:%M:%S") + + def next_trip_dates(self, start_date, day_count=14): + if self.runs_regularly: + for single_date in (start_date + timedelta(n) for n in range(day_count)): + if self.weekdays[single_date.weekday()]==1: + yield single_date.strftime("%Y%m%d") + else: + yield self.start.strftime("%Y%m%d") + + def route_long_name(self): + return self.route_name + + def intersects(self, bbox): + return self.bbox.intersects(box(*bbox)) + + +class TripStore(): + """ + TripStore maintains the currently valid trips. A trip is a + carpool offer enhanced with all stops this + + Attributes: + trips Dict of currently valid trips. + deleted_trips Dict of recently deleted trips. + """ + + def __init__(self, stops_store): + self.transformer = TripTransformer(stops_store) + self.stops_store = stops_store + self.trips = {} + self.deleted_trips = {} + self.recent_trips = {} + + + def put_carpool(self, carpool: Carpool): + """ + Adds carpool to the TripStore. + """ + id = "{}:{}".format(carpool.agency, carpool.id) + filename = f'data/enhanced/{carpool.agency}/{carpool.id}.json' + try: + existing_carpool = self._load_carpool_if_exists(carpool.agency, carpool.id) + if existing_carpool and existing_carpool.lastUpdated == carpool.lastUpdated: + enhanced_carpool = existing_carpool + else: + if len(carpool.stops) < 2 or self.distance_in_m(carpool) < 1000: + logger.warning("Failed to add carpool %s:%s to TripStore, distance too low", carpool.agency, carpool.id) + self.handle_failed_carpool_enhancement(carpool) + return + enhanced_carpool = self.transformer.enhance_carpool(carpool) + # TODO should only store enhanced_carpool, if it has 2 or more stops + assert_folder_exists(f'data/enhanced/{carpool.agency}/') + with open(filename, 'w', encoding='utf-8') as f: + f.write(enhanced_carpool.json()) + logger.info("Added enhanced carpool %s:%s", carpool.agency, carpool.id) + + return self._load_as_trip(enhanced_carpool) + except RoutingException as err: + logger.warning("Failed to add carpool %s:%s to TripStore due to RoutingException %s", carpool.agency, carpool.id, getattr(err, 'message', repr(err))) + self.handle_failed_carpool_enhancement(carpool) + except Exception as err: + logger.error("Failed to add carpool %s:%s to TripStore.", carpool.agency, carpool.id, exc_info=True) + self.handle_failed_carpool_enhancement(carpool) + + def handle_failed_carpool_enhancement(sellf, carpool: Carpool): + assert_folder_exists(f'data/failed/{carpool.agency}/') + with open(f'data/failed/{carpool.agency}/{carpool.id}.json', 'w', encoding='utf-8') as f: + f.write(carpool.json()) + + def distance_in_m(self, carpool): + if len(carpool.stops) < 2: + return 0 + s1 = carpool.stops[0] + s2 = carpool.stops[-1] + return geodesic_distance_in_m((s1.lon, s1.lat),(s2.lon, s2.lat)) + + def recently_added_trips(self): + return list(self.recent_trips.values()) + + def recently_deleted_trips(self): + return list(self.deleted_trips.values()) + + def _load_carpool_if_exists(self, agency_id: str, carpool_id: str): + if carpool_exists(agency_id, carpool_id, 'data/enhanced'): + try: + return load_carpool(agency_id, carpool_id, 'data/enhanced') + except Exception as e: + # An error on restore could be caused by model changes, + # in such a case, it need's to be recreated + logger.warning("Could not restore enhanced trip %s:%s, reason: %s", agency_id, carpool_id, repr(e)) + + return None + + def _load_as_trip(self, carpool: Carpool): + trip = self.transformer.transform_to_trip(carpool) + id = trip.trip_id + self.trips[id] = trip + if not is_older_than_days(carpool.lastUpdated, 1): + self.recent_trips[id] = trip + logger.debug("Added trip %s", id) + + return trip + + def delete_carpool(self, agency_id: str, carpool_id: str): + """ + Deletes carpool from the TripStore. + """ + agencyScopedCarpoolId = f"{agency_id}:{carpool_id}" + trip_to_be_deleted = self.trips.get(agencyScopedCarpoolId) + if trip_to_be_deleted: + self.deleted_trips[agencyScopedCarpoolId] = trip_to_be_deleted + del self.trips[agencyScopedCarpoolId] + + if self.recent_trips.get(agencyScopedCarpoolId): + del self.recent_trips[agencyScopedCarpoolId] + + if carpool_exists(agency_id, carpool_id): + remove_carpool_file(agency_id, carpool_id) + + logger.debug("Deleted trip %s", id) + + def unflag_unrecent_updates(self): + """ + Trips that were last updated before yesterday, are not recent + any longer. As no updates need to be sent for them any longer, + they will be removed from recent recent_trips and deleted_trips. + """ + for key in list(self.recent_trips): + t = self.recent_trips.get(key) + if t and t.lastUpdated.date() < yesterday(): + del self.recent_trips[key] + + for key in list(self.deleted_trips): + t = self.deleted_trips.get(key) + if t and t.lastUpdated.date() < yesterday(): + del self.deleted_trips[key] + + +class TripTransformer: + REPLACE_CARPOOL_STOPS_BY_CLOSEST_TRANSIT_STOPS = True + REPLACEMENT_STOPS_SERACH_RADIUS_IN_M = 1000 + SIMPLIFY_TOLERANCE = 0.0001 + + router = RoutingService() + + def __init__(self, stops_store): + self.stops_store = stops_store + + def transform_to_trip(self, carpool): + stop_times = self._convert_stop_times(carpool) + route_name = carpool.stops[0].name + " nach " + carpool.stops[-1].name + headsign= carpool.stops[-1].name + trip_id = self._trip_id(carpool) + path = carpool.path + bbox = box( + min([pt[0] for pt in path.coordinates]), + min([pt[1] for pt in path.coordinates]), + max([pt[0] for pt in path.coordinates]), + max([pt[1] for pt in path.coordinates])) + + # TODO: pass driver and ridesharing info object to the Trip constructor + trip = Trip(trip_id, route_name, headsign, str(carpool.deeplink), carpool.departureDate, carpool.departureTime, carpool.path, carpool.agency, carpool.lastUpdated, stop_times, bbox) + + return trip + + def _trip_id(self, carpool): + return f"{carpool.agency}:{carpool.id}" + + def _replace_stops_by_transit_stops(self, carpool, max_search_distance): + new_stops = [] + for carpool_stop in carpool.stops: + new_stops.append(self.stops_store.find_closest_stop(carpool_stop, max_search_distance)) + return new_stops + + def enhance_carpool(self, carpool): + if self.REPLACE_CARPOOL_STOPS_BY_CLOSEST_TRANSIT_STOPS: + carpool.stops = self._replace_stops_by_transit_stops(carpool, self.REPLACEMENT_STOPS_SERACH_RADIUS_IN_M) + + path = self._path_for_ride(carpool) + lineString_shapely_wgs84 = LineString(coordinates = path["points"]["coordinates"]).simplify(0.0001) + lineString_wgs84 = GeoJSONLineString(type="LineString", coordinates=list(lineString_shapely_wgs84.coords)) + virtual_stops = self.stops_store.find_additional_stops_around(lineString_wgs84, carpool.stops) + if not virtual_stops.empty: + virtual_stops["time"] = self._estimate_times(path, virtual_stops['distance']) + logger.debug("Virtual stops found: {}".format(virtual_stops)) + if len(virtual_stops) > MAX_STOPS_PER_TRIP: + # in case we found more than MAX_STOPS_PER_TRIP, we retain first and last + # half of MAX_STOPS_PER_TRIP + virtual_stops = virtual_stops.iloc[np.r_[0:int(MAX_STOPS_PER_TRIP/2), int(MAX_STOPS_PER_TRIP/2):]] + + trip_id = f"{carpool.agency}:{carpool.id}" + stop_times = self._stops_and_stop_times(carpool.departureTime, trip_id, virtual_stops) + + enhanced_carpool = carpool.copy() + enhanced_carpool.stops = stop_times + enhanced_carpool.path = lineString_wgs84 + return enhanced_carpool + + def _convert_stop_times(self, carpool): + + stop_times = [GtfsStopTime( + self._trip_id(carpool), + stop.arrivalTime, + stop.departureTime, + stop.id, + seq_nr+1, + STOP_TIMES_STOP_TYPE_NONE if stop.pickup_dropoff == PickupDropoffType.only_dropoff else STOP_TIMES_STOP_TYPE_COORDINATE_DRIVER, + STOP_TIMES_STOP_TYPE_NONE if stop.pickup_dropoff == PickupDropoffType.only_pickup else STOP_TIMES_STOP_TYPE_COORDINATE_DRIVER, + STOP_TIMES_TIMEPOINT_APPROXIMATE) + for seq_nr, stop in enumerate(carpool.stops)] + return stop_times + + def _path_for_ride(self, carpool): + points = self._stop_coords(carpool.stops) + return self.router.path_for_stops(points) + + def _stop_coords(self, stops): + # Retrieve coordinates of all officially announced stops (start, intermediate, target) + return [Point(stop.lon, stop.lat) for stop in stops] + + def _estimate_times(self, path, distances_from_start): + cumulated_distance = 0 + cumulated_time = 0 + stop_times = [] + instructions = path["instructions"] + + cnt = 0 + instr_distance = instructions[cnt]["distance"] + instr_time = instructions[cnt]["time"] + + for distance in distances_from_start: + while cnt < len(instructions) and cumulated_distance + instructions[cnt]["distance"] < distance: + cumulated_distance = cumulated_distance + instructions[cnt]["distance"] + cumulated_time = cumulated_time + instructions[cnt]["time"] + cnt = cnt + 1 + + if cnt < len(instructions): + if instructions[cnt]["distance"] ==0: + raise RoutingException("Origin and destinaction too close") + percent_dist = (distance - cumulated_distance) / instructions[cnt]["distance"] + stop_time = cumulated_time + percent_dist * instructions[cnt]["time"] + stop_times.append(stop_time) + else: + logger.debug("distance {} exceeds total length {}, using max arrival time {}".format(distance, cumulated_distance, cumulated_time)) + stop_times.append(cumulated_time) + return stop_times + + def _stops_and_stop_times(self, start_time, trip_id, stops_frame): + # Assumptions: + # arrival_time = departure_time + # pickup_type, drop_off_type for origin: = coordinate/none + # pickup_type, drop_off_type for destination: = none/coordinate + # timepoint = approximate for origin and destination (not sure what consequences this might have for trip planners) + number_of_stops = len(stops_frame.index) + total_distance = stops_frame.iloc[number_of_stops-1]["distance"] + + first_stop_time = GtfsTimeDelta(hours = start_time.hour, minutes = start_time.minute, seconds = start_time.second) + stop_times = [] + seq_nr = 0 + for i in range(0, number_of_stops): + current_stop = stops_frame.iloc[i] + + if not current_stop.id: + continue + elif i == 0: + if (stops_frame.iloc[1].time-current_stop.time) < 1000: + # skip custom stop if there is an official stop very close by + logger.debug("Skipped stop %s", current_stop.id) + continue + else: + if (current_stop.time-stops_frame.iloc[i-1].time) < 5000 and not i==1 and not is_carpooling_stop(current_stop.id, current_stop.stop_name): + # skip latter stop if it's very close (<5 seconds drive) by the preceding + logger.debug("Skipped stop %s", current_stop.id) + continue + trip_time = timedelta(milliseconds=int(current_stop.time)) + is_dropoff = self._is_dropoff_stop(current_stop, total_distance) + is_pickup = self._is_pickup_stop(current_stop, total_distance) + # TODO would be nice if possible to publish a minimum shared distance + pickup_type = STOP_TIMES_STOP_TYPE_COORDINATE_DRIVER if is_pickup else STOP_TIMES_STOP_TYPE_NONE + dropoff_type = STOP_TIMES_STOP_TYPE_COORDINATE_DRIVER if is_dropoff else STOP_TIMES_STOP_TYPE_NONE + + if is_pickup and not is_dropoff: + pickup_dropoff = PickupDropoffType.only_pickup + elif not is_pickup and is_dropoff: + pickup_dropoff = PickupDropoffType.only_dropoff + else: + pickup_dropoff = PickupDropoffType.pickup_and_dropoff + + next_stop_time = first_stop_time + trip_time + seq_nr += 1 + stop_times.append(StopTime(**{ + 'arrivalTime': str(next_stop_time), + 'departureTime': str(next_stop_time), + 'id': current_stop.id, + 'pickup_dropoff': pickup_dropoff, + 'name': str(current_stop.stop_name), + 'lat': current_stop.y, + 'lon': current_stop.x + })) + + return stop_times + + def _is_dropoff_stop(self, current_stop, total_distance): + return current_stop["distance"] >= 0.5 * total_distance + + def _is_pickup_stop(self, current_stop, total_distance): + return current_stop["distance"] < 0.5 * total_distance + +def load_carpool(agency_id: str, carpool_id: str, folder: str ='data/enhanced') -> Carpool: + with open(f'{folder}/{agency_id}/{carpool_id}.json', 'r', encoding='utf-8') as f: + dict = json.load(f) + carpool = Carpool(**dict) + return carpool + +def carpool_exists(agency_id: str, carpool_id: str, folder: str ='data/enhanced'): + return os.path.exists(f"{folder}/{agency_id}/{carpool_id}.json") + +def remove_carpool_file(agency_id: str, carpool_id: str, folder: str ='data/enhanced'): + return os.remove(f"{folder}/{agency_id}/{carpool_id}.json") From d491c42f251ae297403106b41bc6118764aad60f Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Mon, 29 Jan 2024 15:37:00 +0100 Subject: [PATCH 11/33] Update pyproject.toml --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 114ae51..2e958c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,8 +1,8 @@ [project] name = "amarillo-enhancer" -version = "0.0.2" +version = "0.0.4" dependencies = [ - "amarillo-core", + "amarillo", "watchdog", "schedule==1.2.1", ] \ No newline at end of file From 4da7d134fb9e25a0c71de5fdaa43b9d1707af635 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Tue, 30 Jan 2024 11:16:05 +0100 Subject: [PATCH 12/33] Changed 'amarillo.app' to 'amarillo' --- amarillo/plugins/enhancer/configuration.py | 8 ++++---- amarillo/plugins/enhancer/enhancer.py | 6 +++--- amarillo/plugins/enhancer/services/carpools.py | 4 ++-- amarillo/plugins/enhancer/services/gtfs_export.py | 8 ++++---- amarillo/plugins/enhancer/services/gtfs_generator.py | 6 +++--- amarillo/plugins/enhancer/services/trips.py | 12 ++++++------ 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/amarillo/plugins/enhancer/configuration.py b/amarillo/plugins/enhancer/configuration.py index 821e365..dec2e88 100644 --- a/amarillo/plugins/enhancer/configuration.py +++ b/amarillo/plugins/enhancer/configuration.py @@ -1,16 +1,16 @@ # separate file so that it can be imported without initializing FastAPI -from amarillo.app.utils.container import container +from amarillo.utils.container import container import json import logging from glob import glob -from amarillo.app.models.Carpool import Carpool -from amarillo.app.services import stops +from amarillo.models.Carpool import Carpool +from amarillo.services import stops from amarillo.plugins.enhancer.services import trips from amarillo.plugins.enhancer.services.carpools import CarpoolService from amarillo.plugins.enhancer.services import gtfs_generator -from amarillo.app.configuration import configure_services +from amarillo.configuration import configure_services logger = logging.getLogger(__name__) diff --git a/amarillo/plugins/enhancer/enhancer.py b/amarillo/plugins/enhancer/enhancer.py index c07a8b0..30b44e6 100644 --- a/amarillo/plugins/enhancer/enhancer.py +++ b/amarillo/plugins/enhancer/enhancer.py @@ -6,9 +6,9 @@ from watchdog.observers import Observer from watchdog.events import FileSystemEventHandler from amarillo.plugins.enhancer.configuration import configure_enhancer_services -from amarillo.app.utils.container import container -from amarillo.app.models.Carpool import Carpool -from amarillo.app.utils.utils import agency_carpool_ids_from_filename +from amarillo.utils.container import container +from amarillo.models.Carpool import Carpool +from amarillo.utils.utils import agency_carpool_ids_from_filename logging.config.fileConfig('logging.conf', disable_existing_loggers=False) logger = logging.getLogger("enhancer") diff --git a/amarillo/plugins/enhancer/services/carpools.py b/amarillo/plugins/enhancer/services/carpools.py index 5d6b97c..e62d7fd 100644 --- a/amarillo/plugins/enhancer/services/carpools.py +++ b/amarillo/plugins/enhancer/services/carpools.py @@ -2,8 +2,8 @@ import json import logging from datetime import datetime from typing import Dict -from amarillo.app.models.Carpool import Carpool -from amarillo.app.utils.utils import yesterday, is_older_than_days +from amarillo.models.Carpool import Carpool +from amarillo.utils.utils import yesterday, is_older_than_days logger = logging.getLogger(__name__) diff --git a/amarillo/plugins/enhancer/services/gtfs_export.py b/amarillo/plugins/enhancer/services/gtfs_export.py index 9d9aea5..fb39425 100644 --- a/amarillo/plugins/enhancer/services/gtfs_export.py +++ b/amarillo/plugins/enhancer/services/gtfs_export.py @@ -7,10 +7,10 @@ import gettext import logging import re -from amarillo.app.utils.utils import assert_folder_exists -from amarillo.app.models.gtfs import GtfsTimeDelta, GtfsFeedInfo, GtfsAgency, GtfsRoute, GtfsStop, GtfsStopTime, GtfsTrip, GtfsCalendar, GtfsCalendarDate, GtfsShape -from amarillo.app.services.stops import is_carpooling_stop -from amarillo.app.services.gtfs_constants import * +from amarillo.utils.utils import assert_folder_exists +from amarillo.models.gtfs import GtfsTimeDelta, GtfsFeedInfo, GtfsAgency, GtfsRoute, GtfsStop, GtfsStopTime, GtfsTrip, GtfsCalendar, GtfsCalendarDate, GtfsShape +from amarillo.services.stops import is_carpooling_stop +from amarillo.services.gtfs_constants import * logger = logging.getLogger(__name__) diff --git a/amarillo/plugins/enhancer/services/gtfs_generator.py b/amarillo/plugins/enhancer/services/gtfs_generator.py index 604faad..ef1d7d6 100644 --- a/amarillo/plugins/enhancer/services/gtfs_generator.py +++ b/amarillo/plugins/enhancer/services/gtfs_generator.py @@ -1,7 +1,7 @@ -from amarillo.app.models.Carpool import Region +from amarillo.models.Carpool import Region from amarillo.plugins.enhancer.services.gtfs_export import GtfsExport, GtfsFeedInfo, GtfsAgency -from amarillo.app.services.gtfs import GtfsRtProducer -from amarillo.app.utils.container import container +from amarillo.services.gtfs import GtfsRtProducer +from amarillo.utils.container import container from glob import glob import json import schedule diff --git a/amarillo/plugins/enhancer/services/trips.py b/amarillo/plugins/enhancer/services/trips.py index 6fb58ff..eab61a3 100644 --- a/amarillo/plugins/enhancer/services/trips.py +++ b/amarillo/plugins/enhancer/services/trips.py @@ -1,9 +1,9 @@ -from amarillo.app.models.gtfs import GtfsTimeDelta, GtfsStopTime -from amarillo.app.models.Carpool import MAX_STOPS_PER_TRIP, Carpool, Weekday, StopTime, PickupDropoffType -from amarillo.app.services.gtfs_constants import * -from amarillo.app.services.routing import RoutingService, RoutingException -from amarillo.app.services.stops import is_carpooling_stop -from amarillo.app.utils.utils import assert_folder_exists, is_older_than_days, yesterday, geodesic_distance_in_m +from amarillo.models.gtfs import GtfsTimeDelta, GtfsStopTime +from amarillo.models.Carpool import MAX_STOPS_PER_TRIP, Carpool, Weekday, StopTime, PickupDropoffType +from amarillo.services.gtfs_constants import * +from amarillo.services.routing import RoutingService, RoutingException +from amarillo.services.stops import is_carpooling_stop +from amarillo.utils.utils import assert_folder_exists, is_older_than_days, yesterday, geodesic_distance_in_m from shapely.geometry import Point, LineString, box from geojson_pydantic.geometries import LineString as GeoJSONLineString from datetime import datetime, timedelta From 7ded4838a71d280a5b560dc1fe2f8b70e924d655 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Tue, 30 Jan 2024 15:33:49 +0100 Subject: [PATCH 13/33] moved services/gtfs.py to enhancer --- amarillo/__init__.py | 1 + amarillo/plugins/enhancer/services/gtfs.py | 137 ++++++++++++++++++ .../enhancer/services/gtfs_generator.py | 2 +- 3 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 amarillo/__init__.py create mode 100644 amarillo/plugins/enhancer/services/gtfs.py diff --git a/amarillo/__init__.py b/amarillo/__init__.py new file mode 100644 index 0000000..0260537 --- /dev/null +++ b/amarillo/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) \ No newline at end of file diff --git a/amarillo/plugins/enhancer/services/gtfs.py b/amarillo/plugins/enhancer/services/gtfs.py new file mode 100644 index 0000000..c45a626 --- /dev/null +++ b/amarillo/plugins/enhancer/services/gtfs.py @@ -0,0 +1,137 @@ +import amarillo.services.gtfsrt.gtfs_realtime_pb2 as gtfs_realtime_pb2 +import amarillo.services.gtfsrt.realtime_extension_pb2 as mfdzrte +from amarillo.services.gtfs_constants import * +from google.protobuf.json_format import MessageToDict +from google.protobuf.json_format import ParseDict +from datetime import datetime, timedelta +import json +import re +import time + +class GtfsRtProducer(): + + def __init__(self, trip_store): + self.trip_store = trip_store + + def generate_feed(self, time, format='protobuf', bbox=None): + # See https://developers.google.com/transit/gtfs-realtime/reference + # https://github.com/mfdz/carpool-gtfs-rt/blob/master/src/main/java/de/mfdz/resource/CarpoolResource.java + gtfsrt_dict = { + 'header': { + 'gtfsRealtimeVersion': '1.0', + 'timestamp': int(time) + }, + 'entity': self._get_trip_updates(bbox) + } + feed = gtfs_realtime_pb2.FeedMessage() + ParseDict(gtfsrt_dict, feed) + + if "message" == format.lower(): + return feed + elif "json" == format.lower(): + return MessageToDict(feed) + else: + return feed.SerializeToString() + + def export_feed(self, timestamp, file_path, bbox=None): + """ + Exports gtfs-rt feed as .json and .pbf file to file_path + """ + feed = self.generate_feed(timestamp, "message", bbox) + with open(f"{file_path}.pbf", "wb") as f: + f.write(feed.SerializeToString()) + with open(f"{file_path}.json", "w") as f: + json.dump(MessageToDict(feed), f) + + def _get_trip_updates(self, bbox = None): + trips = [] + trips.extend(self._get_added(bbox)) + trips.extend(self._get_deleted(bbox)) + trip_updates = [] + for num, trip in enumerate(trips): + trip_updates.append( { + 'id': f'carpool-update-{num}', + 'tripUpdate': trip + } + ) + return trip_updates + + def _get_deleted(self, bbox = None): + return self._get_updates( + self.trip_store.recently_deleted_trips(), + self._as_delete_updates, + bbox) + + def _get_added(self, bbox = None): + return self._get_updates( + self.trip_store.recently_added_trips(), + self._as_added_updates, + bbox) + + def _get_updates(self, trips, update_func, bbox = None): + updates = [] + today = datetime.today() + for t in trips: + if bbox == None or t.intersects(bbox): + updates.extend(update_func(t, today)) + return updates + + def _as_delete_updates(self, trip, fromdate): + return [{ + 'trip': { + 'tripId': trip.trip_id, + 'startTime': trip.start_time_str(), + 'startDate': trip_date, + 'scheduleRelationship': 'CANCELED', + 'routeId': trip.trip_id + } + } for trip_date in trip.next_trip_dates(fromdate)] + + def _to_seconds(self, fromdate, stop_time): + startdate = datetime.strptime(fromdate, '%Y%m%d') + m = re.search(r'(\d+):(\d+):(\d+)', stop_time) + delta = timedelta( + hours=int(m.group(1)), + minutes=int(m.group(2)), + seconds=int(m.group(3))) + return time.mktime((startdate + delta).timetuple()) + + def _to_stop_times(self, trip, fromdate): + return [{ + 'stopSequence': stoptime.stop_sequence, + 'arrival': { + 'time': self._to_seconds(fromdate, stoptime.arrival_time), + 'uncertainty': MFDZ_DEFAULT_UNCERTAINITY + }, + 'departure': { + 'time': self._to_seconds(fromdate, stoptime.departure_time), + 'uncertainty': MFDZ_DEFAULT_UNCERTAINITY + }, + 'stopId': stoptime.stop_id, + 'scheduleRelationship': 'SCHEDULED', + 'stop_time_properties': { + '[transit_realtime.stop_time_properties]': { + 'dropoffType': 'COORDINATE_WITH_DRIVER' if stoptime.drop_off_type == STOP_TIMES_STOP_TYPE_COORDINATE_DRIVER else 'NONE', + 'pickupType': 'COORDINATE_WITH_DRIVER' if stoptime.pickup_type == STOP_TIMES_STOP_TYPE_COORDINATE_DRIVER else 'NONE' + } + } + } + for stoptime in trip.stop_times] + + def _as_added_updates(self, trip, fromdate): + return [{ + 'trip': { + 'tripId': trip.trip_id, + 'startTime': trip.start_time_str(), + 'startDate': trip_date, + 'scheduleRelationship': 'ADDED', + 'routeId': trip.trip_id, + '[transit_realtime.trip_descriptor]': { + 'routeUrl' : trip.url, + 'agencyId' : trip.agency, + 'route_long_name' : trip.route_long_name(), + 'route_type': RIDESHARING_ROUTE_TYPE + } + }, + 'stopTimeUpdate': self._to_stop_times(trip, trip_date) + } for trip_date in trip.next_trip_dates(fromdate)] diff --git a/amarillo/plugins/enhancer/services/gtfs_generator.py b/amarillo/plugins/enhancer/services/gtfs_generator.py index ef1d7d6..60f024b 100644 --- a/amarillo/plugins/enhancer/services/gtfs_generator.py +++ b/amarillo/plugins/enhancer/services/gtfs_generator.py @@ -1,6 +1,6 @@ from amarillo.models.Carpool import Region from amarillo.plugins.enhancer.services.gtfs_export import GtfsExport, GtfsFeedInfo, GtfsAgency -from amarillo.services.gtfs import GtfsRtProducer +from amarillo.plugins.enhancer.services.gtfs import GtfsRtProducer from amarillo.utils.container import container from glob import glob import json From cddff3aea2391237ce563b338c6e911f8e6f9038 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Mon, 5 Feb 2024 14:41:07 +0100 Subject: [PATCH 14/33] moved gtfs_constants.py to enhancer --- .../plugins/enhancer/services/gtfs_constants.py | 14 ++++++++++++++ amarillo/plugins/enhancer/services/gtfs_export.py | 2 +- amarillo/plugins/enhancer/services/trips.py | 2 +- 3 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 amarillo/plugins/enhancer/services/gtfs_constants.py diff --git a/amarillo/plugins/enhancer/services/gtfs_constants.py b/amarillo/plugins/enhancer/services/gtfs_constants.py new file mode 100644 index 0000000..1e8f3af --- /dev/null +++ b/amarillo/plugins/enhancer/services/gtfs_constants.py @@ -0,0 +1,14 @@ +# Constants + +NO_BIKES_ALLOWED = 2 +RIDESHARING_ROUTE_TYPE = 1551 +CALENDAR_DATES_EXCEPTION_TYPE_ADDED = 1 +CALENDAR_DATES_EXCEPTION_TYPE_REMOVED = 2 +STOP_TIMES_STOP_TYPE_REGULARLY = 0 +STOP_TIMES_STOP_TYPE_NONE = 1 +STOP_TIMES_STOP_TYPE_PHONE_AGENCY = 2 +STOP_TIMES_STOP_TYPE_COORDINATE_DRIVER = 3 +STOP_TIMES_TIMEPOINT_APPROXIMATE = 0 +STOP_TIMES_TIMEPOINT_EXACT = 1 + +MFDZ_DEFAULT_UNCERTAINITY = 600 \ No newline at end of file diff --git a/amarillo/plugins/enhancer/services/gtfs_export.py b/amarillo/plugins/enhancer/services/gtfs_export.py index fb39425..9c0c26e 100644 --- a/amarillo/plugins/enhancer/services/gtfs_export.py +++ b/amarillo/plugins/enhancer/services/gtfs_export.py @@ -10,7 +10,7 @@ import re from amarillo.utils.utils import assert_folder_exists from amarillo.models.gtfs import GtfsTimeDelta, GtfsFeedInfo, GtfsAgency, GtfsRoute, GtfsStop, GtfsStopTime, GtfsTrip, GtfsCalendar, GtfsCalendarDate, GtfsShape from amarillo.services.stops import is_carpooling_stop -from amarillo.services.gtfs_constants import * +from amarillo.plugins.enhancer.services.gtfs_constants import * logger = logging.getLogger(__name__) diff --git a/amarillo/plugins/enhancer/services/trips.py b/amarillo/plugins/enhancer/services/trips.py index eab61a3..6a59f1d 100644 --- a/amarillo/plugins/enhancer/services/trips.py +++ b/amarillo/plugins/enhancer/services/trips.py @@ -1,6 +1,6 @@ from amarillo.models.gtfs import GtfsTimeDelta, GtfsStopTime from amarillo.models.Carpool import MAX_STOPS_PER_TRIP, Carpool, Weekday, StopTime, PickupDropoffType -from amarillo.services.gtfs_constants import * +from amarillo.plugins.enhancer.services.gtfs_constants import * from amarillo.services.routing import RoutingService, RoutingException from amarillo.services.stops import is_carpooling_stop from amarillo.utils.utils import assert_folder_exists, is_older_than_days, yesterday, geodesic_distance_in_m From 9e8174b917a13737c1ae2db2206838133a8e15d2 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Mon, 5 Feb 2024 15:15:30 +0100 Subject: [PATCH 15/33] moved services/stops.py to enhancer --- amarillo/plugins/enhancer/services/stops.py | 182 ++++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 amarillo/plugins/enhancer/services/stops.py diff --git a/amarillo/plugins/enhancer/services/stops.py b/amarillo/plugins/enhancer/services/stops.py new file mode 100644 index 0000000..1d3a1bd --- /dev/null +++ b/amarillo/plugins/enhancer/services/stops.py @@ -0,0 +1,182 @@ +import csv +import geopandas as gpd +import pandas as pd +from amarillo.models.Carpool import StopTime +from contextlib import closing +from shapely.geometry import Point, LineString +from shapely.ops import transform +from pyproj import Proj, Transformer +import re +import requests +from io import TextIOWrapper +import codecs +import logging + +logger = logging.getLogger(__name__) + +class StopsStore(): + + def __init__(self, stop_sources = [], internal_projection = "EPSG:32632"): + self.internal_projection = internal_projection + self.projection = Transformer.from_crs("EPSG:4326", internal_projection, always_xy=True).transform + self.stopsDataFrames = [] + self.stop_sources = stop_sources + + + def load_stop_sources(self): + """Imports stops from stop_sources and registers them with + the distance they are still associated with a trip. + E.g. bus stops should be registered with a distance of e.g. 30m, + while larger carpool parkings might be registered with e.g. 500m. + + Subsequent calls of load_stop_sources will reload all stop_sources + but replace the current stops only if all stops could be loaded successfully. + """ + stopsDataFrames = [] + error_occured = False + + for stops_source in self.stop_sources: + try: + stopsDataFrame =self._load_stops(stops_source["url"]) + stopsDataFrames.append({'distanceInMeter': stops_source["vicinity"], + 'stops': stopsDataFrame}) + except Exception as err: + error_occured = True + logger.error("Failed to load stops from %s to StopsStore.", stops_source["url"], exc_info=True) + + if not error_occured: + self.stopsDataFrames = stopsDataFrames + + def find_additional_stops_around(self, line, stops = None): + """Returns a GeoDataFrame with all stops in vicinity of the + given line, sorted by distance from origin of the line. + Note: for internal projection/distance calculations, the + lat/lon geometries of line and stops are converted to + """ + stops_frames = [] + if stops: + stops_frames.append(self._convert_to_dataframe(stops)) + transformedLine = transform(self.projection, LineString(line.coordinates)) + for stops_to_match in self.stopsDataFrames: + stops_frames.append(self._find_stops_around_transformed(stops_to_match['stops'], transformedLine, stops_to_match['distanceInMeter'])) + stops = gpd.GeoDataFrame( pd.concat(stops_frames, ignore_index=True, sort=True)) + if not stops.empty: + self._sort_by_distance(stops, transformedLine) + return stops + + def find_closest_stop(self, carpool_stop, max_search_distance): + transformedCoord = Point(self.projection(carpool_stop.lon, carpool_stop.lat)) + best_dist = max_search_distance + 1 + best_stop = None + for stops_with_dist in self.stopsDataFrames: + stops = stops_with_dist['stops'] + s, d = stops.sindex.nearest(transformedCoord, return_all= True, return_distance=True, max_distance=max_search_distance) + if len(d) > 0 and d[0] < best_dist: + best_dist = d[0] + row = s[1][0] + best_stop = StopTime(name=stops.at[row, 'stop_name'], lat=stops.at[row, 'y'], lon=stops.at[row, 'x']) + + return best_stop if best_stop else carpool_stop + + def _normalize_stop_name(self, stop_name): + default_name = 'P+R-Parkplatz' + if stop_name in ('', 'Park&Ride'): + return default_name + normalized_stop_name = re.sub(r"P(ark)?\s?[\+&]\s?R(ail|ide)?",'P+R', stop_name) + + return normalized_stop_name + + def _load_stops(self, source : str): + """Loads stops from given source and registers them with + the distance they are still associated with a trip. + E.g. bus stops should be registered with a distance of e.g. 30m, + while larger carpool parkings might be registered with e.g. 500m + """ + logger.info("Load stops from %s", source) + if source.startswith('http'): + if source.endswith('json'): + with requests.get(source) as json_source: + stopsDataFrame = self._load_stops_geojson(json_source.json()) + else: + with requests.get(source) as csv_source: + stopsDataFrame = self._load_stops_csv(codecs.iterdecode(csv_source.iter_lines(), 'utf-8')) + else: + with open(source, encoding='utf-8') as csv_source: + stopsDataFrame = self._load_stops_csv(csv_source) + + return stopsDataFrame + + def _load_stops_csv(self, csv_source): + id = [] + lat = [] + lon = [] + stop_name = [] + reader = csv.DictReader(csv_source, delimiter=';') + columns = ['stop_id', 'stop_lat', 'stop_lon', 'stop_name'] + lists = [id, lat, lon, stop_name] + for row in reader: + for col, lst in zip(columns, lists): + if col == "stop_lat" or col == "stop_lon": + lst.append(float(row[col].replace(",","."))) + elif col == "stop_name": + row_stop_name = self._normalize_stop_name(row[col]) + lst.append(row_stop_name) + else: + lst.append(row[col]) + + return self._as_dataframe(id, lat, lon, stop_name) + + def _load_stops_geojson(self, geojson_source): + id = [] + lat = [] + lon = [] + stop_name = [] + columns = ['stop_id', 'stop_lat', 'stop_lon', 'stop_name'] + lists = [id, lat, lon, stop_name] + for row in geojson_source['features']: + coord = row['geometry']['coordinates'] + if not coord or not row['properties'].get('name'): + logger.error('Stop feature {} has null coord or name'.format(row['id'])) + continue + for col, lst in zip(columns, lists): + if col == "stop_lat": + lst.append(coord[1]) + elif col == "stop_lon": + lst.append(coord[0]) + elif col == "stop_name": + row_stop_name = self._normalize_stop_name(row['properties']['name']) + lst.append(row_stop_name) + elif col == "stop_id": + lst.append(row['id']) + + return self._as_dataframe(id, lat, lon, stop_name) + + def _as_dataframe(self, id, lat, lon, stop_name): + + df = gpd.GeoDataFrame(data={'x':lon, 'y':lat, 'stop_name':stop_name, 'id':id}) + stopsGeoDataFrame = gpd.GeoDataFrame(df, geometry=gpd.points_from_xy(df.x, df.y, crs='EPSG:4326')) + stopsGeoDataFrame.to_crs(crs=self.internal_projection, inplace=True) + return stopsGeoDataFrame + + def _find_stops_around_transformed(self, stopsDataFrame, transformedLine, distance): + bufferedLine = transformedLine.buffer(distance) + sindex = stopsDataFrame.sindex + possible_matches_index = list(sindex.intersection(bufferedLine.bounds)) + possible_matches = stopsDataFrame.iloc[possible_matches_index] + exact_matches = possible_matches[possible_matches.intersects(bufferedLine)] + + return exact_matches + + def _convert_to_dataframe(self, stops): + return gpd.GeoDataFrame([[stop.name, stop.lon, stop.lat, + stop.id, Point(self.projection(stop.lon, stop.lat))] for stop in stops], columns = ['stop_name','x','y','id','geometry'], crs=self.internal_projection) + + def _sort_by_distance(self, stops, transformedLine): + stops['distance']=stops.apply(lambda row: transformedLine.project(row['geometry']), axis=1) + stops.sort_values('distance', inplace=True) + +def is_carpooling_stop(stop_id, name): + stop_name = name.lower() + # mfdz: or bbnavi: prefixed stops are custom stops which are explicitly meant to be carpooling stops + return stop_id.startswith('mfdz:') or stop_id.startswith('bbnavi:') or 'mitfahr' in stop_name or 'p&m' in stop_name + From edc428a3fd1a85d008a6f2fdf8fe29244c5b8453 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Mon, 5 Feb 2024 15:29:06 +0100 Subject: [PATCH 16/33] moved tests to enhancer --- .gitignore | 1 + amarillo/plugins/enhancer/tests/__init__.py | 0 amarillo/plugins/enhancer/tests/stops.csv | 5 + amarillo/plugins/enhancer/tests/stops.json | 39 +++++ amarillo/plugins/enhancer/tests/test_gtfs.py | 142 ++++++++++++++++++ .../enhancer/tests/test_stops_store.py | 24 +++ .../plugins/enhancer/tests/test_trip_store.py | 23 +++ config | 5 + logging.conf | 22 +++ 9 files changed, 261 insertions(+) create mode 100644 amarillo/plugins/enhancer/tests/__init__.py create mode 100644 amarillo/plugins/enhancer/tests/stops.csv create mode 100644 amarillo/plugins/enhancer/tests/stops.json create mode 100644 amarillo/plugins/enhancer/tests/test_gtfs.py create mode 100644 amarillo/plugins/enhancer/tests/test_stops_store.py create mode 100644 amarillo/plugins/enhancer/tests/test_trip_store.py create mode 100644 config create mode 100644 logging.conf diff --git a/.gitignore b/.gitignore index 5d381cc..d33bc9a 100644 --- a/.gitignore +++ b/.gitignore @@ -160,3 +160,4 @@ cython_debug/ # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ +data/ \ No newline at end of file diff --git a/amarillo/plugins/enhancer/tests/__init__.py b/amarillo/plugins/enhancer/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/amarillo/plugins/enhancer/tests/stops.csv b/amarillo/plugins/enhancer/tests/stops.csv new file mode 100644 index 0000000..ed419bb --- /dev/null +++ b/amarillo/plugins/enhancer/tests/stops.csv @@ -0,0 +1,5 @@ +stop_id;stop_code;stop_lat;stop_lon;stop_name +mfdz:x;x;52.11901;14.2;Stop x +mfdz:y;y;53.1;14.01;Stop y +mfdz:z;z;54.11;14.0;Stop z +mfdz:Ang001;Ang001;53.11901;14.015776;Mitfahrbank Biesenbrow diff --git a/amarillo/plugins/enhancer/tests/stops.json b/amarillo/plugins/enhancer/tests/stops.json new file mode 100644 index 0000000..5744c13 --- /dev/null +++ b/amarillo/plugins/enhancer/tests/stops.json @@ -0,0 +1,39 @@ +{ + "data": { + "pointsOfInterest": [ + { + "id": "14622", + "externalId": "bbnavi:12073:0001", + "name": "Parkbank", + "description": "Parkbank", + "dataProvider": { + "id": "1", + "name": "Administrator" + }, + "addresses": [ + { + "street": "Hauptstrasse", + "city": "Wittenberge", + "zip": "12345", + "geoLocation": { + "latitude": 52.9932971109789, + "longitude": 11.767383582547 + } + } + ], + "openStreetMap": { + "capacity": 112, + "capacityCharging": "2", + "capacityDisabled": "", + "fee": "No", + "lit": "Yes", + "parking": "", + "shelter": "No", + "surface": "", + "utilization": "", + "website": "" + } + } + ] + } +} \ No newline at end of file diff --git a/amarillo/plugins/enhancer/tests/test_gtfs.py b/amarillo/plugins/enhancer/tests/test_gtfs.py new file mode 100644 index 0000000..3fbe97c --- /dev/null +++ b/amarillo/plugins/enhancer/tests/test_gtfs.py @@ -0,0 +1,142 @@ +from amarillo.tests.sampledata import carpool_1234, data1, carpool_repeating_json, stop_issue +from amarillo.plugins.enhancer.services.gtfs_export import GtfsExport +from amarillo.plugins.enhancer.services.gtfs import GtfsRtProducer +from amarillo.plugins.enhancer.services.stops import StopsStore +from amarillo.plugins.enhancer.services.trips import TripStore +from amarillo.models.Carpool import Carpool +from datetime import datetime +import time +import pytest + + +def test_gtfs_generation(): + cp = Carpool(**data1) + stops_store = StopsStore() + trips_store = TripStore(stops_store) + trips_store.put_carpool(cp) + + exporter = GtfsExport(None, None, trips_store, stops_store) + exporter.export('target/tests/test_gtfs_generation/test.gtfs.zip', "target/tests/test_gtfs_generation") + +def test_correct_stops(): + cp = Carpool(**stop_issue) + stops_store = StopsStore([{"url": "https://datahub.bbnavi.de/export/rideshare_points.geojson", "vicinity": 250}]) + stops_store.load_stop_sources() + trips_store = TripStore(stops_store) + trips_store.put_carpool(cp) + assert len(trips_store.trips) == 1 + + +class TestTripConverter: + + def setup_method(self, method): + self.stops_store = StopsStore([{"url": "https://datahub.bbnavi.de/export/rideshare_points.geojson", "vicinity": 50}]) + self.trips_store = TripStore(self.stops_store) + + def test_as_one_time_trip_as_delete_update(self): + cp = Carpool(**data1) + self.trips_store.put_carpool(cp) + trip = next(iter(self.trips_store.trips.values())) + + converter = GtfsRtProducer(self.trips_store) + json = converter._as_delete_updates(trip, datetime(2022,4,11)) + + assert json == [{ + 'trip': { + 'tripId': 'mfdz:Eins', + 'startTime': '23:59:00', + 'startDate': '20220530', + 'scheduleRelationship': 'CANCELED', + 'routeId': 'mfdz:Eins' + } + }] + + def test_as_one_time_trip_as_added_update(self): + cp = Carpool(**data1) + self.trips_store.put_carpool(cp) + trip = next(iter(self.trips_store.trips.values())) + + converter = GtfsRtProducer(self.trips_store) + json = converter._as_added_updates(trip, datetime(2022,4,11)) + assert json == [{ + 'trip': { + 'tripId': 'mfdz:Eins', + 'startTime': '23:59:00', + 'startDate': '20220530', + 'scheduleRelationship': 'ADDED', + 'routeId': 'mfdz:Eins', + '[transit_realtime.trip_descriptor]': { + 'routeUrl' : 'https://mfdz.de/trip/123', + 'agencyId' : 'mfdz', + 'route_long_name' : 'abc nach xyz', + 'route_type': 1551 + } + }, + 'stopTimeUpdate': [{ + 'stopSequence': 1, + 'arrival': { + 'time': time.mktime(datetime(2022,5,30,23,59,0).timetuple()), + 'uncertainty': 600 + }, + 'departure': { + 'time': time.mktime(datetime(2022,5,30,23,59,0).timetuple()), + 'uncertainty': 600 + }, + 'stopId': 'mfdz:12073:001', + 'scheduleRelationship': 'SCHEDULED', + 'stop_time_properties': { + '[transit_realtime.stop_time_properties]': { + 'dropoffType': 'NONE', + 'pickupType': 'COORDINATE_WITH_DRIVER' + } + } + }, + { + 'stopSequence': 2, + 'arrival': { + 'time': time.mktime(datetime(2022,5,31,0,16,45,0).timetuple()), + 'uncertainty': 600 + }, + 'departure': { + 'time': time.mktime(datetime(2022,5,31,0,16,45,0).timetuple()), + 'uncertainty': 600 + }, + + 'stopId': 'de:12073:900340137::3', + 'scheduleRelationship': 'SCHEDULED', + 'stop_time_properties': { + '[transit_realtime.stop_time_properties]': { + 'dropoffType': 'COORDINATE_WITH_DRIVER', + 'pickupType': 'NONE' + } + } + }] + }] + + def test_as_periodic_trip_as_delete_update(self): + cp = Carpool(**carpool_repeating_json) + self.trips_store.put_carpool(cp) + trip = next(iter(self.trips_store.trips.values())) + + converter = GtfsRtProducer(self.trips_store) + json = converter._as_delete_updates(trip, datetime(2022,4,11)) + + assert json == [{ + 'trip': { + 'tripId': 'mfdz:Zwei', + 'startTime': '15:00:00', + 'startDate': '20220411', + 'scheduleRelationship': 'CANCELED', + 'routeId': 'mfdz:Zwei' + } + }, + { + 'trip': { + 'tripId': 'mfdz:Zwei', + 'startTime': '15:00:00', + 'startDate': '20220418', + 'scheduleRelationship': 'CANCELED', + 'routeId': 'mfdz:Zwei' + } + } + ] \ No newline at end of file diff --git a/amarillo/plugins/enhancer/tests/test_stops_store.py b/amarillo/plugins/enhancer/tests/test_stops_store.py new file mode 100644 index 0000000..95cd3a2 --- /dev/null +++ b/amarillo/plugins/enhancer/tests/test_stops_store.py @@ -0,0 +1,24 @@ +from amarillo.plugins.enhancer.services import stops +from amarillo.models.Carpool import StopTime + +def test_load_stops_from_file(): + store = stops.StopsStore([{"url": "amarillo/plugins/enhancer/tests/stops.csv", "vicinity": 50}]) + store.load_stop_sources() + assert len(store.stopsDataFrames[0]['stops']) > 0 + +def test_load_csv_stops_from_web_(): + store = stops.StopsStore([{"url": "https://data.mfdz.de/mfdz/stops/custom.csv", "vicinity": 50}]) + store.load_stop_sources() + assert len(store.stopsDataFrames[0]['stops']) > 0 + +def test_load_geojson_stops_from_web_(): + store = stops.StopsStore([{"url": "https://datahub.bbnavi.de/export/rideshare_points.geojson", "vicinity": 50}]) + store.load_stop_sources() + assert len(store.stopsDataFrames[0]['stops']) > 0 + +def test_find_closest_stop(): + store = stops.StopsStore([{"url": "amarillo/plugins/enhancer/tests/stops.csv", "vicinity": 50}]) + store.load_stop_sources() + carpool_stop = StopTime(name="start", lat=53.1191, lon=14.01577) + stop = store.find_closest_stop(carpool_stop, 1000) + assert stop.name=='Mitfahrbank Biesenbrow' diff --git a/amarillo/plugins/enhancer/tests/test_trip_store.py b/amarillo/plugins/enhancer/tests/test_trip_store.py new file mode 100644 index 0000000..6447668 --- /dev/null +++ b/amarillo/plugins/enhancer/tests/test_trip_store.py @@ -0,0 +1,23 @@ +from amarillo.tests.sampledata import cp1, carpool_repeating +from amarillo.plugins.enhancer.services.trips import TripStore +from amarillo.plugins.enhancer.services.stops import StopsStore + + +import logging +logger = logging.getLogger(__name__) + +def test_trip_store_put_one_time_carpool(): + trip_store = TripStore(StopsStore()) + + t = trip_store.put_carpool(cp1) + assert t != None + assert len(t.stop_times) >= 2 + assert t.stop_times[0].stop_id == 'mfdz:12073:001' + assert t.stop_times[-1].stop_id == 'de:12073:900340137::3' + +def test_trip_store_put_repeating_carpool(): + trip_store = TripStore(StopsStore()) + + t = trip_store.put_carpool(carpool_repeating) + assert t != None + assert len(t.stop_times) >= 2 diff --git a/config b/config new file mode 100644 index 0000000..fadf873 --- /dev/null +++ b/config @@ -0,0 +1,5 @@ +# Bounding-Box Germany +ride2go_query_data = '{ "southWestCoordinates": { "lat": 47.3, "lon": 5.98 }, "northEastCoordinates": { "lat": 54.99, "lon": 15.02 }, "lastModifiedSinceDays": 180 }' +env = 'PROD' +graphhopper_base_url = 'https://api.mfdz.de/gh' +stop_sources_file = 'conf/stop_sources.json' \ No newline at end of file diff --git a/logging.conf b/logging.conf new file mode 100644 index 0000000..429da8e --- /dev/null +++ b/logging.conf @@ -0,0 +1,22 @@ +[loggers] +keys=root + +[handlers] +keys=consoleHandler + +[formatters] +keys=simpleFormatter + +[logger_root] +level=INFO +handlers=consoleHandler +propagate=yes + +[handler_consoleHandler] +class=StreamHandler +level=DEBUG +formatter=simpleFormatter +args=(sys.stdout,) + +[formatter_simpleFormatter] +format=%(asctime)s - %(name)s - %(levelname)s - %(message)s \ No newline at end of file From 6a5c9788caa94f06c24f5c6984b6b1d4852f38b4 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Mon, 5 Feb 2024 15:33:09 +0100 Subject: [PATCH 17/33] fix import paths --- amarillo/plugins/enhancer/configuration.py | 2 +- amarillo/plugins/enhancer/services/gtfs.py | 2 +- amarillo/plugins/enhancer/services/gtfs_export.py | 2 +- amarillo/plugins/enhancer/services/trips.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/amarillo/plugins/enhancer/configuration.py b/amarillo/plugins/enhancer/configuration.py index dec2e88..b361d5d 100644 --- a/amarillo/plugins/enhancer/configuration.py +++ b/amarillo/plugins/enhancer/configuration.py @@ -5,7 +5,7 @@ import logging from glob import glob from amarillo.models.Carpool import Carpool -from amarillo.services import stops +from amarillo.plugins.enhancer.services import stops from amarillo.plugins.enhancer.services import trips from amarillo.plugins.enhancer.services.carpools import CarpoolService from amarillo.plugins.enhancer.services import gtfs_generator diff --git a/amarillo/plugins/enhancer/services/gtfs.py b/amarillo/plugins/enhancer/services/gtfs.py index c45a626..daafa98 100644 --- a/amarillo/plugins/enhancer/services/gtfs.py +++ b/amarillo/plugins/enhancer/services/gtfs.py @@ -1,6 +1,6 @@ import amarillo.services.gtfsrt.gtfs_realtime_pb2 as gtfs_realtime_pb2 import amarillo.services.gtfsrt.realtime_extension_pb2 as mfdzrte -from amarillo.services.gtfs_constants import * +from amarillo.plugins.enhancer.services.gtfs_constants import * from google.protobuf.json_format import MessageToDict from google.protobuf.json_format import ParseDict from datetime import datetime, timedelta diff --git a/amarillo/plugins/enhancer/services/gtfs_export.py b/amarillo/plugins/enhancer/services/gtfs_export.py index 9c0c26e..0866cf3 100644 --- a/amarillo/plugins/enhancer/services/gtfs_export.py +++ b/amarillo/plugins/enhancer/services/gtfs_export.py @@ -9,7 +9,7 @@ import re from amarillo.utils.utils import assert_folder_exists from amarillo.models.gtfs import GtfsTimeDelta, GtfsFeedInfo, GtfsAgency, GtfsRoute, GtfsStop, GtfsStopTime, GtfsTrip, GtfsCalendar, GtfsCalendarDate, GtfsShape -from amarillo.services.stops import is_carpooling_stop +from amarillo.plugins.enhancer.services.stops import is_carpooling_stop from amarillo.plugins.enhancer.services.gtfs_constants import * diff --git a/amarillo/plugins/enhancer/services/trips.py b/amarillo/plugins/enhancer/services/trips.py index 6a59f1d..ce09c85 100644 --- a/amarillo/plugins/enhancer/services/trips.py +++ b/amarillo/plugins/enhancer/services/trips.py @@ -2,7 +2,7 @@ from amarillo.models.gtfs import GtfsTimeDelta, GtfsStopTime from amarillo.models.Carpool import MAX_STOPS_PER_TRIP, Carpool, Weekday, StopTime, PickupDropoffType from amarillo.plugins.enhancer.services.gtfs_constants import * from amarillo.services.routing import RoutingService, RoutingException -from amarillo.services.stops import is_carpooling_stop +from amarillo.plugins.enhancer.services.stops import is_carpooling_stop from amarillo.utils.utils import assert_folder_exists, is_older_than_days, yesterday, geodesic_distance_in_m from shapely.geometry import Point, LineString, box from geojson_pydantic.geometries import LineString as GeoJSONLineString From 0a00a1061656f698e492502461fda6a190fc4c9e Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Tue, 6 Feb 2024 14:30:38 +0100 Subject: [PATCH 18/33] added routing.py --- amarillo/plugins/enhancer/services/routing.py | 47 +++++++++++++++++++ amarillo/plugins/enhancer/services/trips.py | 2 +- 2 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 amarillo/plugins/enhancer/services/routing.py diff --git a/amarillo/plugins/enhancer/services/routing.py b/amarillo/plugins/enhancer/services/routing.py new file mode 100644 index 0000000..fbf8e02 --- /dev/null +++ b/amarillo/plugins/enhancer/services/routing.py @@ -0,0 +1,47 @@ +import requests +import logging + +logger = logging.getLogger(__name__) + +class RoutingException(Exception): + def __init__(self, message): + # Call Exception.__init__(message) + # to use the same Message header as the parent class + super().__init__(message) + +class RoutingService(): + def __init__(self, gh_url = 'https://api.mfdz.de/gh'): + self.gh_service_url = gh_url + + def path_for_stops(self, points): + # Retrieve graphhopper route traversing given points + directions = self._get_directions(points) + if directions and len(directions.get("paths"))>0: + return directions.get("paths")[0] + else: + return {} + + def _get_directions(self, points): + req_url = self._create_url(points, True, True) + logger.debug("Get directions via: {}".format(req_url)) + response = requests.get(req_url) + status = response.status_code + if status == 200: + # Found route between points + return response.json() + else: + try: + message = response.json().get('message') + except: + raise RoutingException("Get directions failed with status code {}".format(status)) + else: + raise RoutingException(message) + + def _create_url(self, points, calc_points = False, instructions = False): + """ Creates GH request URL """ + locations = "" + for point in points: + locations += "point={0}%2C{1}&".format(point.y, point.x) + + return "{0}/route?{1}instructions={2}&calc_points={3}&points_encoded=false".format( + self.gh_service_url, locations, instructions, calc_points) diff --git a/amarillo/plugins/enhancer/services/trips.py b/amarillo/plugins/enhancer/services/trips.py index ce09c85..e488fa4 100644 --- a/amarillo/plugins/enhancer/services/trips.py +++ b/amarillo/plugins/enhancer/services/trips.py @@ -1,7 +1,7 @@ from amarillo.models.gtfs import GtfsTimeDelta, GtfsStopTime from amarillo.models.Carpool import MAX_STOPS_PER_TRIP, Carpool, Weekday, StopTime, PickupDropoffType from amarillo.plugins.enhancer.services.gtfs_constants import * -from amarillo.services.routing import RoutingService, RoutingException +from amarillo.plugins.enhancer.services.routing import RoutingService, RoutingException from amarillo.plugins.enhancer.services.stops import is_carpooling_stop from amarillo.utils.utils import assert_folder_exists, is_older_than_days, yesterday, geodesic_distance_in_m from shapely.geometry import Point, LineString, box From b71029bb91e1b04f72ec9984fa724d409d220884 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Tue, 6 Feb 2024 15:30:38 +0100 Subject: [PATCH 19/33] added services/gtfsrt --- amarillo/plugins/enhancer/services/gtfs.py | 14 ++-- .../enhancer/services/gtfsrt/__init__.py | 0 .../services/gtfsrt/gtfs_realtime_pb2.py | 80 +++++++++++++++++++ .../services/gtfsrt/realtime_extension_pb2.py | 33 ++++++++ 4 files changed, 120 insertions(+), 7 deletions(-) create mode 100644 amarillo/plugins/enhancer/services/gtfsrt/__init__.py create mode 100644 amarillo/plugins/enhancer/services/gtfsrt/gtfs_realtime_pb2.py create mode 100644 amarillo/plugins/enhancer/services/gtfsrt/realtime_extension_pb2.py diff --git a/amarillo/plugins/enhancer/services/gtfs.py b/amarillo/plugins/enhancer/services/gtfs.py index daafa98..d84b863 100644 --- a/amarillo/plugins/enhancer/services/gtfs.py +++ b/amarillo/plugins/enhancer/services/gtfs.py @@ -1,5 +1,5 @@ -import amarillo.services.gtfsrt.gtfs_realtime_pb2 as gtfs_realtime_pb2 -import amarillo.services.gtfsrt.realtime_extension_pb2 as mfdzrte +import amarillo.plugins.enhancer.services.gtfsrt.gtfs_realtime_pb2 as gtfs_realtime_pb2 +import amarillo.plugins.enhancer.services.gtfsrt.realtime_extension_pb2 as mfdzrte from amarillo.plugins.enhancer.services.gtfs_constants import * from google.protobuf.json_format import MessageToDict from google.protobuf.json_format import ParseDict @@ -49,11 +49,11 @@ class GtfsRtProducer(): trips.extend(self._get_deleted(bbox)) trip_updates = [] for num, trip in enumerate(trips): - trip_updates.append( { - 'id': f'carpool-update-{num}', - 'tripUpdate': trip - } - ) + trip_updates.append( { + 'id': f'carpool-update-{num}', + 'tripUpdate': trip + } + ) return trip_updates def _get_deleted(self, bbox = None): diff --git a/amarillo/plugins/enhancer/services/gtfsrt/__init__.py b/amarillo/plugins/enhancer/services/gtfsrt/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/amarillo/plugins/enhancer/services/gtfsrt/gtfs_realtime_pb2.py b/amarillo/plugins/enhancer/services/gtfsrt/gtfs_realtime_pb2.py new file mode 100644 index 0000000..4e10463 --- /dev/null +++ b/amarillo/plugins/enhancer/services/gtfsrt/gtfs_realtime_pb2.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: gtfs-realtime.proto +"""Generated protocol buffer code.""" +from google.protobuf.internal import builder as _builder +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x13gtfs-realtime.proto\x12\x10transit_realtime\"y\n\x0b\x46\x65\x65\x64Message\x12,\n\x06header\x18\x01 \x02(\x0b\x32\x1c.transit_realtime.FeedHeader\x12,\n\x06\x65ntity\x18\x02 \x03(\x0b\x32\x1c.transit_realtime.FeedEntity*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\xd7\x01\n\nFeedHeader\x12\x1d\n\x15gtfs_realtime_version\x18\x01 \x02(\t\x12Q\n\x0eincrementality\x18\x02 \x01(\x0e\x32+.transit_realtime.FeedHeader.Incrementality:\x0c\x46ULL_DATASET\x12\x11\n\ttimestamp\x18\x03 \x01(\x04\"4\n\x0eIncrementality\x12\x10\n\x0c\x46ULL_DATASET\x10\x00\x12\x10\n\x0c\x44IFFERENTIAL\x10\x01*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\xd2\x01\n\nFeedEntity\x12\n\n\x02id\x18\x01 \x02(\t\x12\x19\n\nis_deleted\x18\x02 \x01(\x08:\x05\x66\x61lse\x12\x31\n\x0btrip_update\x18\x03 \x01(\x0b\x32\x1c.transit_realtime.TripUpdate\x12\x32\n\x07vehicle\x18\x04 \x01(\x0b\x32!.transit_realtime.VehiclePosition\x12&\n\x05\x61lert\x18\x05 \x01(\x0b\x32\x17.transit_realtime.Alert*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\x82\x08\n\nTripUpdate\x12.\n\x04trip\x18\x01 \x02(\x0b\x32 .transit_realtime.TripDescriptor\x12\x34\n\x07vehicle\x18\x03 \x01(\x0b\x32#.transit_realtime.VehicleDescriptor\x12\x45\n\x10stop_time_update\x18\x02 \x03(\x0b\x32+.transit_realtime.TripUpdate.StopTimeUpdate\x12\x11\n\ttimestamp\x18\x04 \x01(\x04\x12\r\n\x05\x64\x65lay\x18\x05 \x01(\x05\x12\x44\n\x0ftrip_properties\x18\x06 \x01(\x0b\x32+.transit_realtime.TripUpdate.TripProperties\x1aQ\n\rStopTimeEvent\x12\r\n\x05\x64\x65lay\x18\x01 \x01(\x05\x12\x0c\n\x04time\x18\x02 \x01(\x03\x12\x13\n\x0buncertainty\x18\x03 \x01(\x05*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\x1a\xa0\x04\n\x0eStopTimeUpdate\x12\x15\n\rstop_sequence\x18\x01 \x01(\r\x12\x0f\n\x07stop_id\x18\x04 \x01(\t\x12;\n\x07\x61rrival\x18\x02 \x01(\x0b\x32*.transit_realtime.TripUpdate.StopTimeEvent\x12=\n\tdeparture\x18\x03 \x01(\x0b\x32*.transit_realtime.TripUpdate.StopTimeEvent\x12j\n\x15schedule_relationship\x18\x05 \x01(\x0e\x32@.transit_realtime.TripUpdate.StopTimeUpdate.ScheduleRelationship:\tSCHEDULED\x12\\\n\x14stop_time_properties\x18\x06 \x01(\x0b\x32>.transit_realtime.TripUpdate.StopTimeUpdate.StopTimeProperties\x1a>\n\x12StopTimeProperties\x12\x18\n\x10\x61ssigned_stop_id\x18\x01 \x01(\t*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"P\n\x14ScheduleRelationship\x12\r\n\tSCHEDULED\x10\x00\x12\x0b\n\x07SKIPPED\x10\x01\x12\x0b\n\x07NO_DATA\x10\x02\x12\x0f\n\x0bUNSCHEDULED\x10\x03*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\x1aY\n\x0eTripProperties\x12\x0f\n\x07trip_id\x18\x01 \x01(\t\x12\x12\n\nstart_date\x18\x02 \x01(\t\x12\x12\n\nstart_time\x18\x03 \x01(\t*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\xdf\t\n\x0fVehiclePosition\x12.\n\x04trip\x18\x01 \x01(\x0b\x32 .transit_realtime.TripDescriptor\x12\x34\n\x07vehicle\x18\x08 \x01(\x0b\x32#.transit_realtime.VehicleDescriptor\x12,\n\x08position\x18\x02 \x01(\x0b\x32\x1a.transit_realtime.Position\x12\x1d\n\x15\x63urrent_stop_sequence\x18\x03 \x01(\r\x12\x0f\n\x07stop_id\x18\x07 \x01(\t\x12Z\n\x0e\x63urrent_status\x18\x04 \x01(\x0e\x32\x33.transit_realtime.VehiclePosition.VehicleStopStatus:\rIN_TRANSIT_TO\x12\x11\n\ttimestamp\x18\x05 \x01(\x04\x12K\n\x10\x63ongestion_level\x18\x06 \x01(\x0e\x32\x31.transit_realtime.VehiclePosition.CongestionLevel\x12K\n\x10occupancy_status\x18\t \x01(\x0e\x32\x31.transit_realtime.VehiclePosition.OccupancyStatus\x12\x1c\n\x14occupancy_percentage\x18\n \x01(\r\x12Q\n\x16multi_carriage_details\x18\x0b \x03(\x0b\x32\x31.transit_realtime.VehiclePosition.CarriageDetails\x1a\xd9\x01\n\x0f\x43\x61rriageDetails\x12\n\n\x02id\x18\x01 \x01(\t\x12\r\n\x05label\x18\x02 \x01(\t\x12^\n\x10occupancy_status\x18\x03 \x01(\x0e\x32\x31.transit_realtime.VehiclePosition.OccupancyStatus:\x11NO_DATA_AVAILABLE\x12 \n\x14occupancy_percentage\x18\x04 \x01(\x05:\x02-1\x12\x19\n\x11\x63\x61rriage_sequence\x18\x05 \x01(\r*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"G\n\x11VehicleStopStatus\x12\x0f\n\x0bINCOMING_AT\x10\x00\x12\x0e\n\nSTOPPED_AT\x10\x01\x12\x11\n\rIN_TRANSIT_TO\x10\x02\"}\n\x0f\x43ongestionLevel\x12\x1c\n\x18UNKNOWN_CONGESTION_LEVEL\x10\x00\x12\x14\n\x10RUNNING_SMOOTHLY\x10\x01\x12\x0f\n\x0bSTOP_AND_GO\x10\x02\x12\x0e\n\nCONGESTION\x10\x03\x12\x15\n\x11SEVERE_CONGESTION\x10\x04\"\xd9\x01\n\x0fOccupancyStatus\x12\t\n\x05\x45MPTY\x10\x00\x12\x18\n\x14MANY_SEATS_AVAILABLE\x10\x01\x12\x17\n\x13\x46\x45W_SEATS_AVAILABLE\x10\x02\x12\x16\n\x12STANDING_ROOM_ONLY\x10\x03\x12\x1e\n\x1a\x43RUSHED_STANDING_ROOM_ONLY\x10\x04\x12\x08\n\x04\x46ULL\x10\x05\x12\x1c\n\x18NOT_ACCEPTING_PASSENGERS\x10\x06\x12\x15\n\x11NO_DATA_AVAILABLE\x10\x07\x12\x11\n\rNOT_BOARDABLE\x10\x08*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\x80\t\n\x05\x41lert\x12\x32\n\ractive_period\x18\x01 \x03(\x0b\x32\x1b.transit_realtime.TimeRange\x12\x39\n\x0finformed_entity\x18\x05 \x03(\x0b\x32 .transit_realtime.EntitySelector\x12;\n\x05\x63\x61use\x18\x06 \x01(\x0e\x32\x1d.transit_realtime.Alert.Cause:\rUNKNOWN_CAUSE\x12>\n\x06\x65\x66\x66\x65\x63t\x18\x07 \x01(\x0e\x32\x1e.transit_realtime.Alert.Effect:\x0eUNKNOWN_EFFECT\x12/\n\x03url\x18\x08 \x01(\x0b\x32\".transit_realtime.TranslatedString\x12\x37\n\x0bheader_text\x18\n \x01(\x0b\x32\".transit_realtime.TranslatedString\x12<\n\x10\x64\x65scription_text\x18\x0b \x01(\x0b\x32\".transit_realtime.TranslatedString\x12;\n\x0ftts_header_text\x18\x0c \x01(\x0b\x32\".transit_realtime.TranslatedString\x12@\n\x14tts_description_text\x18\r \x01(\x0b\x32\".transit_realtime.TranslatedString\x12O\n\x0eseverity_level\x18\x0e \x01(\x0e\x32%.transit_realtime.Alert.SeverityLevel:\x10UNKNOWN_SEVERITY\"\xd8\x01\n\x05\x43\x61use\x12\x11\n\rUNKNOWN_CAUSE\x10\x01\x12\x0f\n\x0bOTHER_CAUSE\x10\x02\x12\x15\n\x11TECHNICAL_PROBLEM\x10\x03\x12\n\n\x06STRIKE\x10\x04\x12\x11\n\rDEMONSTRATION\x10\x05\x12\x0c\n\x08\x41\x43\x43IDENT\x10\x06\x12\x0b\n\x07HOLIDAY\x10\x07\x12\x0b\n\x07WEATHER\x10\x08\x12\x0f\n\x0bMAINTENANCE\x10\t\x12\x10\n\x0c\x43ONSTRUCTION\x10\n\x12\x13\n\x0fPOLICE_ACTIVITY\x10\x0b\x12\x15\n\x11MEDICAL_EMERGENCY\x10\x0c\"\xdd\x01\n\x06\x45\x66\x66\x65\x63t\x12\x0e\n\nNO_SERVICE\x10\x01\x12\x13\n\x0fREDUCED_SERVICE\x10\x02\x12\x16\n\x12SIGNIFICANT_DELAYS\x10\x03\x12\n\n\x06\x44\x45TOUR\x10\x04\x12\x16\n\x12\x41\x44\x44ITIONAL_SERVICE\x10\x05\x12\x14\n\x10MODIFIED_SERVICE\x10\x06\x12\x10\n\x0cOTHER_EFFECT\x10\x07\x12\x12\n\x0eUNKNOWN_EFFECT\x10\x08\x12\x0e\n\nSTOP_MOVED\x10\t\x12\r\n\tNO_EFFECT\x10\n\x12\x17\n\x13\x41\x43\x43\x45SSIBILITY_ISSUE\x10\x0b\"H\n\rSeverityLevel\x12\x14\n\x10UNKNOWN_SEVERITY\x10\x01\x12\x08\n\x04INFO\x10\x02\x12\x0b\n\x07WARNING\x10\x03\x12\n\n\x06SEVERE\x10\x04*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"7\n\tTimeRange\x12\r\n\x05start\x18\x01 \x01(\x04\x12\x0b\n\x03\x65nd\x18\x02 \x01(\x04*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"q\n\x08Position\x12\x10\n\x08latitude\x18\x01 \x02(\x02\x12\x11\n\tlongitude\x18\x02 \x02(\x02\x12\x0f\n\x07\x62\x65\x61ring\x18\x03 \x01(\x02\x12\x10\n\x08odometer\x18\x04 \x01(\x01\x12\r\n\x05speed\x18\x05 \x01(\x02*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\xcd\x02\n\x0eTripDescriptor\x12\x0f\n\x07trip_id\x18\x01 \x01(\t\x12\x10\n\x08route_id\x18\x05 \x01(\t\x12\x14\n\x0c\x64irection_id\x18\x06 \x01(\r\x12\x12\n\nstart_time\x18\x02 \x01(\t\x12\x12\n\nstart_date\x18\x03 \x01(\t\x12T\n\x15schedule_relationship\x18\x04 \x01(\x0e\x32\x35.transit_realtime.TripDescriptor.ScheduleRelationship\"t\n\x14ScheduleRelationship\x12\r\n\tSCHEDULED\x10\x00\x12\t\n\x05\x41\x44\x44\x45\x44\x10\x01\x12\x0f\n\x0bUNSCHEDULED\x10\x02\x12\x0c\n\x08\x43\x41NCELED\x10\x03\x12\x13\n\x0bREPLACEMENT\x10\x05\x1a\x02\x08\x01\x12\x0e\n\nDUPLICATED\x10\x06*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"U\n\x11VehicleDescriptor\x12\n\n\x02id\x18\x01 \x01(\t\x12\r\n\x05label\x18\x02 \x01(\t\x12\x15\n\rlicense_plate\x18\x03 \x01(\t*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\xb0\x01\n\x0e\x45ntitySelector\x12\x11\n\tagency_id\x18\x01 \x01(\t\x12\x10\n\x08route_id\x18\x02 \x01(\t\x12\x12\n\nroute_type\x18\x03 \x01(\x05\x12.\n\x04trip\x18\x04 \x01(\x0b\x32 .transit_realtime.TripDescriptor\x12\x0f\n\x07stop_id\x18\x05 \x01(\t\x12\x14\n\x0c\x64irection_id\x18\x06 \x01(\r*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\xa6\x01\n\x10TranslatedString\x12\x43\n\x0btranslation\x18\x01 \x03(\x0b\x32..transit_realtime.TranslatedString.Translation\x1a=\n\x0bTranslation\x12\x0c\n\x04text\x18\x01 \x02(\t\x12\x10\n\x08language\x18\x02 \x01(\t*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90NB\x1d\n\x1b\x63om.google.transit.realtime') + +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'gtfs_realtime_pb2', globals()) +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'\n\033com.google.transit.realtime' + _TRIPDESCRIPTOR_SCHEDULERELATIONSHIP.values_by_name["REPLACEMENT"]._options = None + _TRIPDESCRIPTOR_SCHEDULERELATIONSHIP.values_by_name["REPLACEMENT"]._serialized_options = b'\010\001' + _FEEDMESSAGE._serialized_start=41 + _FEEDMESSAGE._serialized_end=162 + _FEEDHEADER._serialized_start=165 + _FEEDHEADER._serialized_end=380 + _FEEDHEADER_INCREMENTALITY._serialized_start=312 + _FEEDHEADER_INCREMENTALITY._serialized_end=364 + _FEEDENTITY._serialized_start=383 + _FEEDENTITY._serialized_end=593 + _TRIPUPDATE._serialized_start=596 + _TRIPUPDATE._serialized_end=1622 + _TRIPUPDATE_STOPTIMEEVENT._serialized_start=887 + _TRIPUPDATE_STOPTIMEEVENT._serialized_end=968 + _TRIPUPDATE_STOPTIMEUPDATE._serialized_start=971 + _TRIPUPDATE_STOPTIMEUPDATE._serialized_end=1515 + _TRIPUPDATE_STOPTIMEUPDATE_STOPTIMEPROPERTIES._serialized_start=1355 + _TRIPUPDATE_STOPTIMEUPDATE_STOPTIMEPROPERTIES._serialized_end=1417 + _TRIPUPDATE_STOPTIMEUPDATE_SCHEDULERELATIONSHIP._serialized_start=1419 + _TRIPUPDATE_STOPTIMEUPDATE_SCHEDULERELATIONSHIP._serialized_end=1499 + _TRIPUPDATE_TRIPPROPERTIES._serialized_start=1517 + _TRIPUPDATE_TRIPPROPERTIES._serialized_end=1606 + _VEHICLEPOSITION._serialized_start=1625 + _VEHICLEPOSITION._serialized_end=2872 + _VEHICLEPOSITION_CARRIAGEDETAILS._serialized_start=2219 + _VEHICLEPOSITION_CARRIAGEDETAILS._serialized_end=2436 + _VEHICLEPOSITION_VEHICLESTOPSTATUS._serialized_start=2438 + _VEHICLEPOSITION_VEHICLESTOPSTATUS._serialized_end=2509 + _VEHICLEPOSITION_CONGESTIONLEVEL._serialized_start=2511 + _VEHICLEPOSITION_CONGESTIONLEVEL._serialized_end=2636 + _VEHICLEPOSITION_OCCUPANCYSTATUS._serialized_start=2639 + _VEHICLEPOSITION_OCCUPANCYSTATUS._serialized_end=2856 + _ALERT._serialized_start=2875 + _ALERT._serialized_end=4027 + _ALERT_CAUSE._serialized_start=3497 + _ALERT_CAUSE._serialized_end=3713 + _ALERT_EFFECT._serialized_start=3716 + _ALERT_EFFECT._serialized_end=3937 + _ALERT_SEVERITYLEVEL._serialized_start=3939 + _ALERT_SEVERITYLEVEL._serialized_end=4011 + _TIMERANGE._serialized_start=4029 + _TIMERANGE._serialized_end=4084 + _POSITION._serialized_start=4086 + _POSITION._serialized_end=4199 + _TRIPDESCRIPTOR._serialized_start=4202 + _TRIPDESCRIPTOR._serialized_end=4535 + _TRIPDESCRIPTOR_SCHEDULERELATIONSHIP._serialized_start=4403 + _TRIPDESCRIPTOR_SCHEDULERELATIONSHIP._serialized_end=4519 + _VEHICLEDESCRIPTOR._serialized_start=4537 + _VEHICLEDESCRIPTOR._serialized_end=4622 + _ENTITYSELECTOR._serialized_start=4625 + _ENTITYSELECTOR._serialized_end=4801 + _TRANSLATEDSTRING._serialized_start=4804 + _TRANSLATEDSTRING._serialized_end=4970 + _TRANSLATEDSTRING_TRANSLATION._serialized_start=4893 + _TRANSLATEDSTRING_TRANSLATION._serialized_end=4954 +# @@protoc_insertion_point(module_scope) diff --git a/amarillo/plugins/enhancer/services/gtfsrt/realtime_extension_pb2.py b/amarillo/plugins/enhancer/services/gtfsrt/realtime_extension_pb2.py new file mode 100644 index 0000000..c2bbd7b --- /dev/null +++ b/amarillo/plugins/enhancer/services/gtfsrt/realtime_extension_pb2.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: realtime_extension.proto +"""Generated protocol buffer code.""" +from google.protobuf.internal import builder as _builder +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +import amarillo.plugins.enhancer.services.gtfsrt.gtfs_realtime_pb2 as gtfs__realtime__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18realtime_extension.proto\x12\x10transit_realtime\x1a\x13gtfs-realtime.proto\"p\n\x1bMfdzTripDescriptorExtension\x12\x11\n\troute_url\x18\x01 \x01(\t\x12\x11\n\tagency_id\x18\x02 \x01(\t\x12\x17\n\x0froute_long_name\x18\x03 \x01(\t\x12\x12\n\nroute_type\x18\x04 \x01(\r\"\xb0\x02\n\x1fMfdzStopTimePropertiesExtension\x12X\n\x0bpickup_type\x18\x01 \x01(\x0e\x32\x43.transit_realtime.MfdzStopTimePropertiesExtension.DropOffPickupType\x12Y\n\x0c\x64ropoff_type\x18\x02 \x01(\x0e\x32\x43.transit_realtime.MfdzStopTimePropertiesExtension.DropOffPickupType\"X\n\x11\x44ropOffPickupType\x12\x0b\n\x07REGULAR\x10\x00\x12\x08\n\x04NONE\x10\x01\x12\x10\n\x0cPHONE_AGENCY\x10\x02\x12\x1a\n\x16\x43OORDINATE_WITH_DRIVER\x10\x03:i\n\x0ftrip_descriptor\x12 .transit_realtime.TripDescriptor\x18\xf5\x07 \x01(\x0b\x32-.transit_realtime.MfdzTripDescriptorExtension:\x90\x01\n\x14stop_time_properties\x12>.transit_realtime.TripUpdate.StopTimeUpdate.StopTimeProperties\x18\xf5\x07 \x01(\x0b\x32\x31.transit_realtime.MfdzStopTimePropertiesExtensionB\t\n\x07\x64\x65.mfdz') + +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'realtime_extension_pb2', globals()) +if _descriptor._USE_C_DESCRIPTORS == False: + gtfs__realtime__pb2.TripDescriptor.RegisterExtension(trip_descriptor) + gtfs__realtime__pb2.TripUpdate.StopTimeUpdate.StopTimeProperties.RegisterExtension(stop_time_properties) + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'\n\007de.mfdz' + _MFDZTRIPDESCRIPTOREXTENSION._serialized_start=67 + _MFDZTRIPDESCRIPTOREXTENSION._serialized_end=179 + _MFDZSTOPTIMEPROPERTIESEXTENSION._serialized_start=182 + _MFDZSTOPTIMEPROPERTIESEXTENSION._serialized_end=486 + _MFDZSTOPTIMEPROPERTIESEXTENSION_DROPOFFPICKUPTYPE._serialized_start=398 + _MFDZSTOPTIMEPROPERTIESEXTENSION_DROPOFFPICKUPTYPE._serialized_end=486 +# @@protoc_insertion_point(module_scope) From f848acc11a13f603e935dbbdd134a9613e0291fc Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Thu, 8 Feb 2024 11:46:17 +0100 Subject: [PATCH 20/33] added models/gtfs.py --- amarillo/plugins/enhancer/models/gtfs.py | 30 +++++++++++++++++++ .../plugins/enhancer/services/gtfs_export.py | 2 +- amarillo/plugins/enhancer/services/trips.py | 2 +- 3 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 amarillo/plugins/enhancer/models/gtfs.py diff --git a/amarillo/plugins/enhancer/models/gtfs.py b/amarillo/plugins/enhancer/models/gtfs.py new file mode 100644 index 0000000..33d38a0 --- /dev/null +++ b/amarillo/plugins/enhancer/models/gtfs.py @@ -0,0 +1,30 @@ +# TODO: move to enhancer +from collections import namedtuple +from datetime import timedelta + +GtfsFeedInfo = namedtuple('GtfsFeedInfo', 'feed_id feed_publisher_name feed_publisher_url feed_lang feed_version') +GtfsAgency = namedtuple('GtfsAgency', 'agency_id agency_name agency_url agency_timezone agency_lang agency_email') +GtfsRoute = namedtuple('GtfsRoute', 'agency_id route_id route_long_name route_type route_url route_short_name') +GtfsStop = namedtuple('GtfsStop', 'stop_id stop_lat stop_lon stop_name') +GtfsStopTime = namedtuple('GtfsStopTime', 'trip_id departure_time arrival_time stop_id stop_sequence pickup_type drop_off_type timepoint') +GtfsTrip = namedtuple('GtfsTrip', 'route_id trip_id service_id shape_id trip_headsign bikes_allowed') +GtfsCalendar = namedtuple('GtfsCalendar', 'service_id start_date end_date monday tuesday wednesday thursday friday saturday sunday') +GtfsCalendarDate = namedtuple('GtfsCalendarDate', 'service_id date exception_type') +GtfsShape = namedtuple('GtfsShape','shape_id shape_pt_lon shape_pt_lat shape_pt_sequence') + +# TODO Move to utils +class GtfsTimeDelta(timedelta): + def __str__(self): + seconds = self.total_seconds() + hours = seconds // 3600 + minutes = (seconds % 3600) // 60 + seconds = seconds % 60 + str = '{:02d}:{:02d}:{:02d}'.format(int(hours), int(minutes), int(seconds)) + return (str) + + def __add__(self, other): + if isinstance(other, timedelta): + return self.__class__(self.days + other.days, + self.seconds + other.seconds, + self.microseconds + other.microseconds) + return NotImplemented \ No newline at end of file diff --git a/amarillo/plugins/enhancer/services/gtfs_export.py b/amarillo/plugins/enhancer/services/gtfs_export.py index 0866cf3..0fafa19 100644 --- a/amarillo/plugins/enhancer/services/gtfs_export.py +++ b/amarillo/plugins/enhancer/services/gtfs_export.py @@ -8,7 +8,7 @@ import logging import re from amarillo.utils.utils import assert_folder_exists -from amarillo.models.gtfs import GtfsTimeDelta, GtfsFeedInfo, GtfsAgency, GtfsRoute, GtfsStop, GtfsStopTime, GtfsTrip, GtfsCalendar, GtfsCalendarDate, GtfsShape +from amarillo.plugins.enhancer.models.gtfs import GtfsTimeDelta, GtfsFeedInfo, GtfsAgency, GtfsRoute, GtfsStop, GtfsStopTime, GtfsTrip, GtfsCalendar, GtfsCalendarDate, GtfsShape from amarillo.plugins.enhancer.services.stops import is_carpooling_stop from amarillo.plugins.enhancer.services.gtfs_constants import * diff --git a/amarillo/plugins/enhancer/services/trips.py b/amarillo/plugins/enhancer/services/trips.py index e488fa4..348c296 100644 --- a/amarillo/plugins/enhancer/services/trips.py +++ b/amarillo/plugins/enhancer/services/trips.py @@ -1,4 +1,4 @@ -from amarillo.models.gtfs import GtfsTimeDelta, GtfsStopTime +from amarillo.plugins.enhancer.models.gtfs import GtfsTimeDelta, GtfsStopTime from amarillo.models.Carpool import MAX_STOPS_PER_TRIP, Carpool, Weekday, StopTime, PickupDropoffType from amarillo.plugins.enhancer.services.gtfs_constants import * from amarillo.plugins.enhancer.services.routing import RoutingService, RoutingException From 69bdc4a262ec71553822fd8a803130c3813af2b5 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Thu, 8 Feb 2024 12:04:02 +0100 Subject: [PATCH 21/33] added MANIFEST.in --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..b2a83a0 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +recursive-include amarillo/plugins/enhancer/tests/ * \ No newline at end of file From 0c6e86c20b11746922da2a773fa9c713fbf0361b Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Thu, 8 Feb 2024 12:04:26 +0100 Subject: [PATCH 22/33] updated pyproject.toml --- pyproject.toml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2e958c2..5b14be0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,8 +1,11 @@ [project] name = "amarillo-enhancer" -version = "0.0.4" +version = "0.0.5" dependencies = [ "amarillo", "watchdog", "schedule==1.2.1", -] \ No newline at end of file +] + +[tool.setuptools.packages] +find = {} \ No newline at end of file From fe5de9f7c15eeac539d77c5981e0162493b1bc76 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Thu, 8 Feb 2024 13:31:21 +0100 Subject: [PATCH 23/33] make stops sources configurable --- amarillo/plugins/enhancer/configuration.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/amarillo/plugins/enhancer/configuration.py b/amarillo/plugins/enhancer/configuration.py index b361d5d..e94c368 100644 --- a/amarillo/plugins/enhancer/configuration.py +++ b/amarillo/plugins/enhancer/configuration.py @@ -9,28 +9,26 @@ from amarillo.plugins.enhancer.services import stops from amarillo.plugins.enhancer.services import trips from amarillo.plugins.enhancer.services.carpools import CarpoolService from amarillo.plugins.enhancer.services import gtfs_generator - +from amarillo.services.config import config from amarillo.configuration import configure_services logger = logging.getLogger(__name__) - -def configure_enhancer_services(): - configure_services() - +def configure_container(): logger.info("Load stops...") - stop_sources = [ - {"url": "https://datahub.bbnavi.de/export/rideshare_points.geojson", "vicinity": 50}, - {"url": "https://data.mfdz.de/mfdz/stops/stops_zhv.csv", "vicinity": 50}, - {"url": "https://data.mfdz.de/mfdz/stops/parkings_osm.csv", "vicinity": 500}, - ] - stop_store = stops.StopsStore(stop_sources) + with open(config.stop_sources_file) as stop_sources_file: + stop_sources = json.load(stop_sources_file) + stop_store = stops.StopsStore(stop_sources) stop_store.load_stop_sources() container['stops_store'] = stop_store container['trips_store'] = trips.TripStore(stop_store) container['carpools'] = CarpoolService(container['trips_store']) +def configure_enhancer_services(): + configure_services() + configure_container() + logger.info("Restore carpools...") for agency_id in container['agencies'].agencies: From 8cb1302360d035802a688e8f1d15e2a6eb3c8977 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Thu, 8 Feb 2024 14:20:53 +0100 Subject: [PATCH 24/33] make graphhopper url configurable --- amarillo/plugins/enhancer/services/trips.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/amarillo/plugins/enhancer/services/trips.py b/amarillo/plugins/enhancer/services/trips.py index 348c296..3a93c53 100644 --- a/amarillo/plugins/enhancer/services/trips.py +++ b/amarillo/plugins/enhancer/services/trips.py @@ -1,5 +1,6 @@ from amarillo.plugins.enhancer.models.gtfs import GtfsTimeDelta, GtfsStopTime from amarillo.models.Carpool import MAX_STOPS_PER_TRIP, Carpool, Weekday, StopTime, PickupDropoffType +from amarillo.services.config import config from amarillo.plugins.enhancer.services.gtfs_constants import * from amarillo.plugins.enhancer.services.routing import RoutingService, RoutingException from amarillo.plugins.enhancer.services.stops import is_carpooling_stop @@ -198,7 +199,7 @@ class TripTransformer: REPLACEMENT_STOPS_SERACH_RADIUS_IN_M = 1000 SIMPLIFY_TOLERANCE = 0.0001 - router = RoutingService() + router = RoutingService(config.graphhopper_base_url) def __init__(self, stops_store): self.stops_store = stops_store From 38a8447ba1f5574e98912dee4fe9467dedaea624 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Thu, 8 Feb 2024 14:24:08 +0100 Subject: [PATCH 25/33] run in daemon mode --- amarillo/plugins/enhancer/enhancer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/amarillo/plugins/enhancer/enhancer.py b/amarillo/plugins/enhancer/enhancer.py index 30b44e6..d459585 100644 --- a/amarillo/plugins/enhancer/enhancer.py +++ b/amarillo/plugins/enhancer/enhancer.py @@ -70,7 +70,7 @@ def run_enhancer(): logger.info("Goodbye Enhancer") def setup(app): - thread = Thread(target=run_enhancer) + thread = Thread(target=run_enhancer, daemon=True) thread.start() From a6fe91267379c28467b31686d96858461a026b79 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Thu, 8 Feb 2024 14:32:54 +0100 Subject: [PATCH 26/33] moved gtfs generation to export plugin --- amarillo/plugins/enhancer/configuration.py | 10 +- amarillo/plugins/enhancer/services/gtfs.py | 137 ----------- .../plugins/enhancer/services/gtfs_export.py | 229 ------------------ .../enhancer/services/gtfs_generator.py | 71 ------ 4 files changed, 6 insertions(+), 441 deletions(-) delete mode 100644 amarillo/plugins/enhancer/services/gtfs.py delete mode 100644 amarillo/plugins/enhancer/services/gtfs_export.py delete mode 100644 amarillo/plugins/enhancer/services/gtfs_generator.py diff --git a/amarillo/plugins/enhancer/configuration.py b/amarillo/plugins/enhancer/configuration.py index e94c368..8bc5ff5 100644 --- a/amarillo/plugins/enhancer/configuration.py +++ b/amarillo/plugins/enhancer/configuration.py @@ -27,7 +27,11 @@ def configure_container(): def configure_enhancer_services(): configure_services() - configure_container() + + # configure_container may have been called already by an exporter, no need to run it again + if 'stops_store' not in container: + configure_container() + else: logger.info("Container already configured") logger.info("Restore carpools...") @@ -46,6 +50,4 @@ def configure_enhancer_services(): carpool = Carpool(**(json.load(carpool_file))) container['carpools'].delete(carpool.agency, carpool.id) - logger.info("Restored carpools: %s", container['carpools'].get_all_ids()) - logger.info("Starting scheduler") - gtfs_generator.start_schedule() \ No newline at end of file + logger.info("Restored carpools: %s", container['carpools'].get_all_ids()) \ No newline at end of file diff --git a/amarillo/plugins/enhancer/services/gtfs.py b/amarillo/plugins/enhancer/services/gtfs.py deleted file mode 100644 index d84b863..0000000 --- a/amarillo/plugins/enhancer/services/gtfs.py +++ /dev/null @@ -1,137 +0,0 @@ -import amarillo.plugins.enhancer.services.gtfsrt.gtfs_realtime_pb2 as gtfs_realtime_pb2 -import amarillo.plugins.enhancer.services.gtfsrt.realtime_extension_pb2 as mfdzrte -from amarillo.plugins.enhancer.services.gtfs_constants import * -from google.protobuf.json_format import MessageToDict -from google.protobuf.json_format import ParseDict -from datetime import datetime, timedelta -import json -import re -import time - -class GtfsRtProducer(): - - def __init__(self, trip_store): - self.trip_store = trip_store - - def generate_feed(self, time, format='protobuf', bbox=None): - # See https://developers.google.com/transit/gtfs-realtime/reference - # https://github.com/mfdz/carpool-gtfs-rt/blob/master/src/main/java/de/mfdz/resource/CarpoolResource.java - gtfsrt_dict = { - 'header': { - 'gtfsRealtimeVersion': '1.0', - 'timestamp': int(time) - }, - 'entity': self._get_trip_updates(bbox) - } - feed = gtfs_realtime_pb2.FeedMessage() - ParseDict(gtfsrt_dict, feed) - - if "message" == format.lower(): - return feed - elif "json" == format.lower(): - return MessageToDict(feed) - else: - return feed.SerializeToString() - - def export_feed(self, timestamp, file_path, bbox=None): - """ - Exports gtfs-rt feed as .json and .pbf file to file_path - """ - feed = self.generate_feed(timestamp, "message", bbox) - with open(f"{file_path}.pbf", "wb") as f: - f.write(feed.SerializeToString()) - with open(f"{file_path}.json", "w") as f: - json.dump(MessageToDict(feed), f) - - def _get_trip_updates(self, bbox = None): - trips = [] - trips.extend(self._get_added(bbox)) - trips.extend(self._get_deleted(bbox)) - trip_updates = [] - for num, trip in enumerate(trips): - trip_updates.append( { - 'id': f'carpool-update-{num}', - 'tripUpdate': trip - } - ) - return trip_updates - - def _get_deleted(self, bbox = None): - return self._get_updates( - self.trip_store.recently_deleted_trips(), - self._as_delete_updates, - bbox) - - def _get_added(self, bbox = None): - return self._get_updates( - self.trip_store.recently_added_trips(), - self._as_added_updates, - bbox) - - def _get_updates(self, trips, update_func, bbox = None): - updates = [] - today = datetime.today() - for t in trips: - if bbox == None or t.intersects(bbox): - updates.extend(update_func(t, today)) - return updates - - def _as_delete_updates(self, trip, fromdate): - return [{ - 'trip': { - 'tripId': trip.trip_id, - 'startTime': trip.start_time_str(), - 'startDate': trip_date, - 'scheduleRelationship': 'CANCELED', - 'routeId': trip.trip_id - } - } for trip_date in trip.next_trip_dates(fromdate)] - - def _to_seconds(self, fromdate, stop_time): - startdate = datetime.strptime(fromdate, '%Y%m%d') - m = re.search(r'(\d+):(\d+):(\d+)', stop_time) - delta = timedelta( - hours=int(m.group(1)), - minutes=int(m.group(2)), - seconds=int(m.group(3))) - return time.mktime((startdate + delta).timetuple()) - - def _to_stop_times(self, trip, fromdate): - return [{ - 'stopSequence': stoptime.stop_sequence, - 'arrival': { - 'time': self._to_seconds(fromdate, stoptime.arrival_time), - 'uncertainty': MFDZ_DEFAULT_UNCERTAINITY - }, - 'departure': { - 'time': self._to_seconds(fromdate, stoptime.departure_time), - 'uncertainty': MFDZ_DEFAULT_UNCERTAINITY - }, - 'stopId': stoptime.stop_id, - 'scheduleRelationship': 'SCHEDULED', - 'stop_time_properties': { - '[transit_realtime.stop_time_properties]': { - 'dropoffType': 'COORDINATE_WITH_DRIVER' if stoptime.drop_off_type == STOP_TIMES_STOP_TYPE_COORDINATE_DRIVER else 'NONE', - 'pickupType': 'COORDINATE_WITH_DRIVER' if stoptime.pickup_type == STOP_TIMES_STOP_TYPE_COORDINATE_DRIVER else 'NONE' - } - } - } - for stoptime in trip.stop_times] - - def _as_added_updates(self, trip, fromdate): - return [{ - 'trip': { - 'tripId': trip.trip_id, - 'startTime': trip.start_time_str(), - 'startDate': trip_date, - 'scheduleRelationship': 'ADDED', - 'routeId': trip.trip_id, - '[transit_realtime.trip_descriptor]': { - 'routeUrl' : trip.url, - 'agencyId' : trip.agency, - 'route_long_name' : trip.route_long_name(), - 'route_type': RIDESHARING_ROUTE_TYPE - } - }, - 'stopTimeUpdate': self._to_stop_times(trip, trip_date) - } for trip_date in trip.next_trip_dates(fromdate)] diff --git a/amarillo/plugins/enhancer/services/gtfs_export.py b/amarillo/plugins/enhancer/services/gtfs_export.py deleted file mode 100644 index 0fafa19..0000000 --- a/amarillo/plugins/enhancer/services/gtfs_export.py +++ /dev/null @@ -1,229 +0,0 @@ - -from collections.abc import Iterable -from datetime import datetime, timedelta -from zipfile import ZipFile -import csv -import gettext -import logging -import re - -from amarillo.utils.utils import assert_folder_exists -from amarillo.plugins.enhancer.models.gtfs import GtfsTimeDelta, GtfsFeedInfo, GtfsAgency, GtfsRoute, GtfsStop, GtfsStopTime, GtfsTrip, GtfsCalendar, GtfsCalendarDate, GtfsShape -from amarillo.plugins.enhancer.services.stops import is_carpooling_stop -from amarillo.plugins.enhancer.services.gtfs_constants import * - - -logger = logging.getLogger(__name__) - -class GtfsExport: - - stops_counter = 0 - trips_counter = 0 - routes_counter = 0 - - stored_stops = {} - - def __init__(self, agencies, feed_info, ridestore, stopstore, bbox = None): - self.stops = {} - self.routes = [] - self.calendar_dates = [] - self.calendar = [] - self.trips = [] - self.stop_times = [] - self.calendar = [] - self.shapes = [] - self.agencies = agencies - self.feed_info = feed_info - self.localized_to = " nach " - self.localized_short_name = "Mitfahrgelegenheit" - self.stopstore = stopstore - self.ridestore = ridestore - self.bbox = bbox - - def export(self, gtfszip_filename, gtfsfolder): - assert_folder_exists(gtfsfolder) - self._prepare_gtfs_feed(self.ridestore, self.stopstore) - self._write_csvfile(gtfsfolder, 'agency.txt', self.agencies) - self._write_csvfile(gtfsfolder, 'feed_info.txt', self.feed_info) - self._write_csvfile(gtfsfolder, 'routes.txt', self.routes) - self._write_csvfile(gtfsfolder, 'trips.txt', self.trips) - self._write_csvfile(gtfsfolder, 'calendar.txt', self.calendar) - self._write_csvfile(gtfsfolder, 'calendar_dates.txt', self.calendar_dates) - self._write_csvfile(gtfsfolder, 'stops.txt', self.stops.values()) - self._write_csvfile(gtfsfolder, 'stop_times.txt', self.stop_times) - self._write_csvfile(gtfsfolder, 'shapes.txt', self.shapes) - self._zip_files(gtfszip_filename, gtfsfolder) - - def _zip_files(self, gtfszip_filename, gtfsfolder): - gtfsfiles = ['agency.txt', 'feed_info.txt', 'routes.txt', 'trips.txt', - 'calendar.txt', 'calendar_dates.txt', 'stops.txt', 'stop_times.txt', 'shapes.txt'] - with ZipFile(gtfszip_filename, 'w') as gtfszip: - for gtfsfile in gtfsfiles: - gtfszip.write(gtfsfolder+'/'+gtfsfile, gtfsfile) - - def _prepare_gtfs_feed(self, ridestore, stopstore): - """ - Prepares all gtfs objects in memory before they are written - to their respective streams. - - For all wellknown stops a GTFS stop is created and - afterwards all ride offers are transformed into their - gtfs equivalents. - """ - for stopSet in stopstore.stopsDataFrames: - for stop in stopSet["stops"].itertuples(): - self._load_stored_stop(stop) - cloned_trips = dict(ridestore.trips) - for url, trip in cloned_trips.items(): - if self.bbox is None or trip.intersects(self.bbox): - self._convert_trip(trip) - - def _convert_trip(self, trip): - self.routes_counter += 1 - self.routes.append(self._create_route(trip)) - self.calendar.append(self._create_calendar(trip)) - if not trip.runs_regularly: - self.calendar_dates.append(self._create_calendar_date(trip)) - self.trips.append(self._create_trip(trip, self.routes_counter)) - self._append_stops_and_stop_times(trip) - self._append_shapes(trip, self.routes_counter) - - def _trip_headsign(self, destination): - destination = destination.replace('(Deutschland)', '') - destination = destination.replace(', Deutschland', '') - appendix = '' - if 'Schweiz' in destination or 'Switzerland' in destination: - appendix = ', Schweiz' - destination = destination.replace('(Schweiz)', '') - destination = destination.replace(', Schweiz', '') - destination = destination.replace('(Switzerland)', '') - - try: - matches = re.match(r"(.*,)? ?(\d{4,5})? ?(.*)", destination) - - match = matches.group(3).strip() if matches != None else destination.strip() - if match[-1]==')' and not '(' in match: - match = match[0:-1] - - return match + appendix - except Exception as ex: - logger.error("error for "+destination ) - logger.exception(ex) - return destination - - def _create_route(self, trip): - return GtfsRoute(trip.agency, trip.trip_id, trip.route_long_name(), RIDESHARING_ROUTE_TYPE, trip.url, "") - - def _create_calendar(self, trip): - # TODO currently, calendar is not provided by Fahrgemeinschaft.de interface. - # We could apply some heuristics like requesting multiple days and extrapolate - # if multiple trips are found, but better would be to have these provided by the - # offical interface. Then validity periods should be provided as well (not - # sure if these are available) - # For fahrgemeinschaft.de, regurlar trips are recognizable via their url - # which contains "regelmaessig". However, we don't know on which days of the week, - # nor until when. As a first guess, if datetime is a mo-fr, we assume each workday, - # if it's sa/su, only this... - - feed_start_date = datetime.today() - stop_date = self._convert_stop_date(feed_start_date) - return GtfsCalendar(trip.trip_id, stop_date, self._convert_stop_date(feed_start_date + timedelta(days=31)), *(trip.weekdays)) - - def _create_calendar_date(self, trip): - return GtfsCalendarDate(trip.trip_id, self._convert_stop_date(trip.start), CALENDAR_DATES_EXCEPTION_TYPE_ADDED) - - def _create_trip(self, trip, shape_id): - return GtfsTrip(trip.trip_id, trip.trip_id, trip.trip_id, shape_id, trip.trip_headsign, NO_BIKES_ALLOWED) - - def _convert_stop(self, stop): - """ - Converts a stop represented as pandas row to a gtfs stop. - Expected attributes of stop: id, stop_name, x, y (in wgs84) - """ - if stop.id: - id = stop.id - else: - self.stops_counter += 1 - id = "tmp-{}".format(self.stops_counter) - - stop_name = "k.A." if stop.stop_name is None else stop.stop_name - return GtfsStop(id, stop.y, stop.x, stop_name) - - def _append_stops_and_stop_times(self, trip): - # Assumptions: - # arrival_time = departure_time - # pickup_type, drop_off_type for origin: = coordinate/none - # pickup_type, drop_off_type for destination: = none/coordinate - # timepoint = approximate for origin and destination (not sure what consequences this might have for trip planners) - for stop_time in trip.stop_times: - # retrieve stop from stored_stops and mark it to be exported - wkn_stop = self.stored_stops.get(stop_time.stop_id) - if not wkn_stop: - logger.warning("No stop found in stop_store for %s. Will skip stop_time %s of trip %s", stop_time.stop_id, stop_time.stop_sequence, trip.trip_id) - else: - self.stops[stop_time.stop_id] = wkn_stop - # Append stop_time - self.stop_times.append(stop_time) - - def _append_shapes(self, trip, shape_id): - counter = 0 - for point in trip.path.coordinates: - counter += 1 - self.shapes.append(GtfsShape(shape_id, point[0], point[1], counter)) - - def _stop_hash(self, stop): - return "{}#{}#{}".format(stop.stop_name,stop.x,stop.y) - - def _should_always_export(self, stop): - """ - Returns true, if the given stop shall be exported to GTFS, - regardless, if it's part of a trip or not. - - This is necessary, as potential stops are required - to be part of the GTFS to be referenced later on - by dynamicly added trips. - """ - if self.bbox: - return (self.bbox[0] <= stop.stop_lon <= self.bbox[2] and - self.bbox[1] <= stop.stop_lat <= self.bbox[3]) - else: - return is_carpooling_stop(stop.stop_id, stop.stop_name) - - def _load_stored_stop(self, stop): - gtfsstop = self._convert_stop(stop) - stop_hash = self._stop_hash(stop) - self.stored_stops[gtfsstop.stop_id] = gtfsstop - if self._should_always_export(gtfsstop): - self.stops[gtfsstop.stop_id] = gtfsstop - - def _get_stop_by_hash(self, stop_hash): - return self.stops.get(stop_hash, self.stored_stops.get(stop_hash)) - - def _get_or_create_stop(self, stop): - stop_hash = self._stop_hash(stop) - gtfsstop = self.stops.get(stop_hash) - if gtfsstop is None: - gtfsstop = self.stored_stops.get(stop_hash, self._convert_stop(stop)) - self.stops[stop_hash] = gtfsstop - return gtfsstop - - def _convert_stop_date(self, date_time): - return date_time.strftime("%Y%m%d") - - def _write_csvfile(self, gtfsfolder, filename, content): - with open(gtfsfolder+"/"+filename, 'w', newline="\n", encoding="utf-8") as csvfile: - self._write_csv(csvfile, content) - - def _write_csv(self, csvfile, content): - if hasattr(content, '_fields'): - writer = csv.DictWriter(csvfile, content._fields) - writer.writeheader() - writer.writerow(content._asdict()) - else: - if content: - writer = csv.DictWriter(csvfile, next(iter(content))._fields) - writer.writeheader() - for record in content: - writer.writerow(record._asdict()) - - \ No newline at end of file diff --git a/amarillo/plugins/enhancer/services/gtfs_generator.py b/amarillo/plugins/enhancer/services/gtfs_generator.py deleted file mode 100644 index 60f024b..0000000 --- a/amarillo/plugins/enhancer/services/gtfs_generator.py +++ /dev/null @@ -1,71 +0,0 @@ -from amarillo.models.Carpool import Region -from amarillo.plugins.enhancer.services.gtfs_export import GtfsExport, GtfsFeedInfo, GtfsAgency -from amarillo.plugins.enhancer.services.gtfs import GtfsRtProducer -from amarillo.utils.container import container -from glob import glob -import json -import schedule -import threading -import time -import logging -from datetime import date, timedelta - -logger = logging.getLogger(__name__) - -regions = {} -for region_file_name in glob('conf/region/*.json'): - with open(region_file_name) as region_file: - dict = json.load(region_file) - region = Region(**dict) - region_id = region.id - regions[region_id] = region - -agencies = [] -for agency_file_name in glob('conf/agency/*.json'): - with open(agency_file_name) as agency_file: - dict = json.load(agency_file) - agency = GtfsAgency(dict["id"], dict["name"], dict["url"], dict["timezone"], dict["lang"], dict["email"]) - agency_id = agency.agency_id - agencies.append(agency) - -def run_schedule(): - while 1: - try: - schedule.run_pending() - except Exception as e: - logger.exception(e) - time.sleep(1) - -def midnight(): - container['stops_store'].load_stop_sources() - container['trips_store'].unflag_unrecent_updates() - container['carpools'].purge_outdated_offers() - generate_gtfs() - -def generate_gtfs(): - logger.info("Generate GTFS") - - for region in regions.values(): - # TODO make feed producer infos configurable - feed_info = GtfsFeedInfo('mfdz', 'MITFAHR|DE|ZENTRALE', 'http://www.mitfahrdezentrale.de', 'de', 1) - exporter = GtfsExport( - agencies, - feed_info, - container['trips_store'], - container['stops_store'], - region.bbox) - exporter.export(f"data/gtfs/amarillo.{region.id}.gtfs.zip", "data/tmp/") - -def generate_gtfs_rt(): - logger.info("Generate GTFS-RT") - producer = GtfsRtProducer(container['trips_store']) - for region in regions.values(): - rt = producer.export_feed(time.time(), f"data/gtfs/amarillo.{region.id}.gtfsrt", bbox=region.bbox) - -def start_schedule(): - schedule.every().day.at("00:00").do(midnight) - schedule.every(60).seconds.do(generate_gtfs_rt) - # Create all feeds once at startup - schedule.run_all() - job_thread = threading.Thread(target=run_schedule, daemon=True) - job_thread.start() \ No newline at end of file From 3587ab03321cff2b023284ae9cad5f9586206c30 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Thu, 8 Feb 2024 14:59:36 +0100 Subject: [PATCH 27/33] add car profile to gh request --- amarillo/plugins/enhancer/services/routing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/amarillo/plugins/enhancer/services/routing.py b/amarillo/plugins/enhancer/services/routing.py index fbf8e02..96f6229 100644 --- a/amarillo/plugins/enhancer/services/routing.py +++ b/amarillo/plugins/enhancer/services/routing.py @@ -43,5 +43,5 @@ class RoutingService(): for point in points: locations += "point={0}%2C{1}&".format(point.y, point.x) - return "{0}/route?{1}instructions={2}&calc_points={3}&points_encoded=false".format( + return "{0}/route?{1}instructions={2}&calc_points={3}&points_encoded=false&profile=car".format( self.gh_service_url, locations, instructions, calc_points) From 6eb9151e6f7d6e595dede9202e416a10f5a708ec Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Fri, 9 Feb 2024 12:02:05 +0100 Subject: [PATCH 28/33] revert configure_container --- amarillo/plugins/enhancer/configuration.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/amarillo/plugins/enhancer/configuration.py b/amarillo/plugins/enhancer/configuration.py index 8bc5ff5..53e4948 100644 --- a/amarillo/plugins/enhancer/configuration.py +++ b/amarillo/plugins/enhancer/configuration.py @@ -14,7 +14,9 @@ from amarillo.configuration import configure_services logger = logging.getLogger(__name__) -def configure_container(): +def configure_enhancer_services(): + configure_services() + logger.info("Load stops...") with open(config.stop_sources_file) as stop_sources_file: stop_sources = json.load(stop_sources_file) @@ -25,14 +27,6 @@ def configure_container(): container['trips_store'] = trips.TripStore(stop_store) container['carpools'] = CarpoolService(container['trips_store']) -def configure_enhancer_services(): - configure_services() - - # configure_container may have been called already by an exporter, no need to run it again - if 'stops_store' not in container: - configure_container() - else: logger.info("Container already configured") - logger.info("Restore carpools...") for agency_id in container['agencies'].agencies: From e21f67621ca80f28715bf28bc070c9220f27fe91 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Fri, 9 Feb 2024 13:17:55 +0100 Subject: [PATCH 29/33] make sure configuration only happens once --- amarillo/plugins/enhancer/configuration.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/amarillo/plugins/enhancer/configuration.py b/amarillo/plugins/enhancer/configuration.py index 53e4948..e70fe99 100644 --- a/amarillo/plugins/enhancer/configuration.py +++ b/amarillo/plugins/enhancer/configuration.py @@ -14,7 +14,15 @@ from amarillo.configuration import configure_services logger = logging.getLogger(__name__) +enhancer_configured = False + def configure_enhancer_services(): + #Make sure configuration only happens once + global enhancer_configured + if enhancer_configured: + logger.info("Enhancer is already configured") + return + configure_services() logger.info("Load stops...") @@ -44,4 +52,6 @@ def configure_enhancer_services(): carpool = Carpool(**(json.load(carpool_file))) container['carpools'].delete(carpool.agency, carpool.id) - logger.info("Restored carpools: %s", container['carpools'].get_all_ids()) \ No newline at end of file + logger.info("Restored carpools: %s", container['carpools'].get_all_ids()) + + enhancer_configured = True From b2036f0b7d8e14cc8cb8a3ce782e13bb7a055f56 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Fri, 9 Feb 2024 14:08:24 +0100 Subject: [PATCH 30/33] moved test_gtfs to export plugin --- amarillo/plugins/enhancer/tests/test_gtfs.py | 142 ------------------- 1 file changed, 142 deletions(-) delete mode 100644 amarillo/plugins/enhancer/tests/test_gtfs.py diff --git a/amarillo/plugins/enhancer/tests/test_gtfs.py b/amarillo/plugins/enhancer/tests/test_gtfs.py deleted file mode 100644 index 3fbe97c..0000000 --- a/amarillo/plugins/enhancer/tests/test_gtfs.py +++ /dev/null @@ -1,142 +0,0 @@ -from amarillo.tests.sampledata import carpool_1234, data1, carpool_repeating_json, stop_issue -from amarillo.plugins.enhancer.services.gtfs_export import GtfsExport -from amarillo.plugins.enhancer.services.gtfs import GtfsRtProducer -from amarillo.plugins.enhancer.services.stops import StopsStore -from amarillo.plugins.enhancer.services.trips import TripStore -from amarillo.models.Carpool import Carpool -from datetime import datetime -import time -import pytest - - -def test_gtfs_generation(): - cp = Carpool(**data1) - stops_store = StopsStore() - trips_store = TripStore(stops_store) - trips_store.put_carpool(cp) - - exporter = GtfsExport(None, None, trips_store, stops_store) - exporter.export('target/tests/test_gtfs_generation/test.gtfs.zip', "target/tests/test_gtfs_generation") - -def test_correct_stops(): - cp = Carpool(**stop_issue) - stops_store = StopsStore([{"url": "https://datahub.bbnavi.de/export/rideshare_points.geojson", "vicinity": 250}]) - stops_store.load_stop_sources() - trips_store = TripStore(stops_store) - trips_store.put_carpool(cp) - assert len(trips_store.trips) == 1 - - -class TestTripConverter: - - def setup_method(self, method): - self.stops_store = StopsStore([{"url": "https://datahub.bbnavi.de/export/rideshare_points.geojson", "vicinity": 50}]) - self.trips_store = TripStore(self.stops_store) - - def test_as_one_time_trip_as_delete_update(self): - cp = Carpool(**data1) - self.trips_store.put_carpool(cp) - trip = next(iter(self.trips_store.trips.values())) - - converter = GtfsRtProducer(self.trips_store) - json = converter._as_delete_updates(trip, datetime(2022,4,11)) - - assert json == [{ - 'trip': { - 'tripId': 'mfdz:Eins', - 'startTime': '23:59:00', - 'startDate': '20220530', - 'scheduleRelationship': 'CANCELED', - 'routeId': 'mfdz:Eins' - } - }] - - def test_as_one_time_trip_as_added_update(self): - cp = Carpool(**data1) - self.trips_store.put_carpool(cp) - trip = next(iter(self.trips_store.trips.values())) - - converter = GtfsRtProducer(self.trips_store) - json = converter._as_added_updates(trip, datetime(2022,4,11)) - assert json == [{ - 'trip': { - 'tripId': 'mfdz:Eins', - 'startTime': '23:59:00', - 'startDate': '20220530', - 'scheduleRelationship': 'ADDED', - 'routeId': 'mfdz:Eins', - '[transit_realtime.trip_descriptor]': { - 'routeUrl' : 'https://mfdz.de/trip/123', - 'agencyId' : 'mfdz', - 'route_long_name' : 'abc nach xyz', - 'route_type': 1551 - } - }, - 'stopTimeUpdate': [{ - 'stopSequence': 1, - 'arrival': { - 'time': time.mktime(datetime(2022,5,30,23,59,0).timetuple()), - 'uncertainty': 600 - }, - 'departure': { - 'time': time.mktime(datetime(2022,5,30,23,59,0).timetuple()), - 'uncertainty': 600 - }, - 'stopId': 'mfdz:12073:001', - 'scheduleRelationship': 'SCHEDULED', - 'stop_time_properties': { - '[transit_realtime.stop_time_properties]': { - 'dropoffType': 'NONE', - 'pickupType': 'COORDINATE_WITH_DRIVER' - } - } - }, - { - 'stopSequence': 2, - 'arrival': { - 'time': time.mktime(datetime(2022,5,31,0,16,45,0).timetuple()), - 'uncertainty': 600 - }, - 'departure': { - 'time': time.mktime(datetime(2022,5,31,0,16,45,0).timetuple()), - 'uncertainty': 600 - }, - - 'stopId': 'de:12073:900340137::3', - 'scheduleRelationship': 'SCHEDULED', - 'stop_time_properties': { - '[transit_realtime.stop_time_properties]': { - 'dropoffType': 'COORDINATE_WITH_DRIVER', - 'pickupType': 'NONE' - } - } - }] - }] - - def test_as_periodic_trip_as_delete_update(self): - cp = Carpool(**carpool_repeating_json) - self.trips_store.put_carpool(cp) - trip = next(iter(self.trips_store.trips.values())) - - converter = GtfsRtProducer(self.trips_store) - json = converter._as_delete_updates(trip, datetime(2022,4,11)) - - assert json == [{ - 'trip': { - 'tripId': 'mfdz:Zwei', - 'startTime': '15:00:00', - 'startDate': '20220411', - 'scheduleRelationship': 'CANCELED', - 'routeId': 'mfdz:Zwei' - } - }, - { - 'trip': { - 'tripId': 'mfdz:Zwei', - 'startTime': '15:00:00', - 'startDate': '20220418', - 'scheduleRelationship': 'CANCELED', - 'routeId': 'mfdz:Zwei' - } - } - ] \ No newline at end of file From 18b3d2a7bb4129dc2f420fc76a600ec9bee4f432 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Tue, 13 Feb 2024 11:17:14 +0100 Subject: [PATCH 31/33] optional GRFS attributes --- amarillo/plugins/enhancer/services/trips.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/amarillo/plugins/enhancer/services/trips.py b/amarillo/plugins/enhancer/services/trips.py index 3a93c53..6faa749 100644 --- a/amarillo/plugins/enhancer/services/trips.py +++ b/amarillo/plugins/enhancer/services/trips.py @@ -1,5 +1,5 @@ from amarillo.plugins.enhancer.models.gtfs import GtfsTimeDelta, GtfsStopTime -from amarillo.models.Carpool import MAX_STOPS_PER_TRIP, Carpool, Weekday, StopTime, PickupDropoffType +from amarillo.models.Carpool import MAX_STOPS_PER_TRIP, Carpool, Weekday, StopTime, PickupDropoffType, Driver, RidesharingInfo from amarillo.services.config import config from amarillo.plugins.enhancer.services.gtfs_constants import * from amarillo.plugins.enhancer.services.routing import RoutingService, RoutingException @@ -17,8 +17,7 @@ logger = logging.getLogger(__name__) class Trip: - # TODO: add driver attributes, additional ridesharing info - def __init__(self, trip_id, route_name, headsign, url, calendar, departureTime, path, agency, lastUpdated, stop_times, bbox): + def __init__(self, trip_id, route_name, headsign, url, calendar, departureTime, path, agency, lastUpdated, stop_times, driver: Driver, additional_ridesharing_info: RidesharingInfo, bbox): if isinstance(calendar, set): self.runs_regularly = True self.weekdays = [ @@ -44,6 +43,8 @@ class Trip: self.stops = [] self.lastUpdated = lastUpdated self.stop_times = stop_times + self.driver = driver + self.additional_ridesharing_info = additional_ridesharing_info self.bbox = bbox self.route_name = route_name self.trip_headsign = headsign @@ -204,7 +205,7 @@ class TripTransformer: def __init__(self, stops_store): self.stops_store = stops_store - def transform_to_trip(self, carpool): + def transform_to_trip(self, carpool : Carpool): stop_times = self._convert_stop_times(carpool) route_name = carpool.stops[0].name + " nach " + carpool.stops[-1].name headsign= carpool.stops[-1].name @@ -216,8 +217,7 @@ class TripTransformer: max([pt[0] for pt in path.coordinates]), max([pt[1] for pt in path.coordinates])) - # TODO: pass driver and ridesharing info object to the Trip constructor - trip = Trip(trip_id, route_name, headsign, str(carpool.deeplink), carpool.departureDate, carpool.departureTime, carpool.path, carpool.agency, carpool.lastUpdated, stop_times, bbox) + trip = Trip(trip_id, route_name, headsign, str(carpool.deeplink), carpool.departureDate, carpool.departureTime, carpool.path, carpool.agency, carpool.lastUpdated, stop_times, carpool.driver, carpool.additional_ridesharing_info, bbox) return trip From 0a4975caabd9b333e79952b59fb7b55dfc325941 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Fri, 8 Mar 2024 13:02:20 +0100 Subject: [PATCH 32/33] Moved GTFS-RT definitions to exporter --- .../enhancer/services/gtfsrt/__init__.py | 0 .../services/gtfsrt/gtfs_realtime_pb2.py | 80 ------------------- .../services/gtfsrt/realtime_extension_pb2.py | 33 -------- 3 files changed, 113 deletions(-) delete mode 100644 amarillo/plugins/enhancer/services/gtfsrt/__init__.py delete mode 100644 amarillo/plugins/enhancer/services/gtfsrt/gtfs_realtime_pb2.py delete mode 100644 amarillo/plugins/enhancer/services/gtfsrt/realtime_extension_pb2.py diff --git a/amarillo/plugins/enhancer/services/gtfsrt/__init__.py b/amarillo/plugins/enhancer/services/gtfsrt/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/amarillo/plugins/enhancer/services/gtfsrt/gtfs_realtime_pb2.py b/amarillo/plugins/enhancer/services/gtfsrt/gtfs_realtime_pb2.py deleted file mode 100644 index 4e10463..0000000 --- a/amarillo/plugins/enhancer/services/gtfsrt/gtfs_realtime_pb2.py +++ /dev/null @@ -1,80 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: gtfs-realtime.proto -"""Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder -from google.protobuf import descriptor as _descriptor -from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import symbol_database as _symbol_database -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x13gtfs-realtime.proto\x12\x10transit_realtime\"y\n\x0b\x46\x65\x65\x64Message\x12,\n\x06header\x18\x01 \x02(\x0b\x32\x1c.transit_realtime.FeedHeader\x12,\n\x06\x65ntity\x18\x02 \x03(\x0b\x32\x1c.transit_realtime.FeedEntity*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\xd7\x01\n\nFeedHeader\x12\x1d\n\x15gtfs_realtime_version\x18\x01 \x02(\t\x12Q\n\x0eincrementality\x18\x02 \x01(\x0e\x32+.transit_realtime.FeedHeader.Incrementality:\x0c\x46ULL_DATASET\x12\x11\n\ttimestamp\x18\x03 \x01(\x04\"4\n\x0eIncrementality\x12\x10\n\x0c\x46ULL_DATASET\x10\x00\x12\x10\n\x0c\x44IFFERENTIAL\x10\x01*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\xd2\x01\n\nFeedEntity\x12\n\n\x02id\x18\x01 \x02(\t\x12\x19\n\nis_deleted\x18\x02 \x01(\x08:\x05\x66\x61lse\x12\x31\n\x0btrip_update\x18\x03 \x01(\x0b\x32\x1c.transit_realtime.TripUpdate\x12\x32\n\x07vehicle\x18\x04 \x01(\x0b\x32!.transit_realtime.VehiclePosition\x12&\n\x05\x61lert\x18\x05 \x01(\x0b\x32\x17.transit_realtime.Alert*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\x82\x08\n\nTripUpdate\x12.\n\x04trip\x18\x01 \x02(\x0b\x32 .transit_realtime.TripDescriptor\x12\x34\n\x07vehicle\x18\x03 \x01(\x0b\x32#.transit_realtime.VehicleDescriptor\x12\x45\n\x10stop_time_update\x18\x02 \x03(\x0b\x32+.transit_realtime.TripUpdate.StopTimeUpdate\x12\x11\n\ttimestamp\x18\x04 \x01(\x04\x12\r\n\x05\x64\x65lay\x18\x05 \x01(\x05\x12\x44\n\x0ftrip_properties\x18\x06 \x01(\x0b\x32+.transit_realtime.TripUpdate.TripProperties\x1aQ\n\rStopTimeEvent\x12\r\n\x05\x64\x65lay\x18\x01 \x01(\x05\x12\x0c\n\x04time\x18\x02 \x01(\x03\x12\x13\n\x0buncertainty\x18\x03 \x01(\x05*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\x1a\xa0\x04\n\x0eStopTimeUpdate\x12\x15\n\rstop_sequence\x18\x01 \x01(\r\x12\x0f\n\x07stop_id\x18\x04 \x01(\t\x12;\n\x07\x61rrival\x18\x02 \x01(\x0b\x32*.transit_realtime.TripUpdate.StopTimeEvent\x12=\n\tdeparture\x18\x03 \x01(\x0b\x32*.transit_realtime.TripUpdate.StopTimeEvent\x12j\n\x15schedule_relationship\x18\x05 \x01(\x0e\x32@.transit_realtime.TripUpdate.StopTimeUpdate.ScheduleRelationship:\tSCHEDULED\x12\\\n\x14stop_time_properties\x18\x06 \x01(\x0b\x32>.transit_realtime.TripUpdate.StopTimeUpdate.StopTimeProperties\x1a>\n\x12StopTimeProperties\x12\x18\n\x10\x61ssigned_stop_id\x18\x01 \x01(\t*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"P\n\x14ScheduleRelationship\x12\r\n\tSCHEDULED\x10\x00\x12\x0b\n\x07SKIPPED\x10\x01\x12\x0b\n\x07NO_DATA\x10\x02\x12\x0f\n\x0bUNSCHEDULED\x10\x03*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\x1aY\n\x0eTripProperties\x12\x0f\n\x07trip_id\x18\x01 \x01(\t\x12\x12\n\nstart_date\x18\x02 \x01(\t\x12\x12\n\nstart_time\x18\x03 \x01(\t*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\xdf\t\n\x0fVehiclePosition\x12.\n\x04trip\x18\x01 \x01(\x0b\x32 .transit_realtime.TripDescriptor\x12\x34\n\x07vehicle\x18\x08 \x01(\x0b\x32#.transit_realtime.VehicleDescriptor\x12,\n\x08position\x18\x02 \x01(\x0b\x32\x1a.transit_realtime.Position\x12\x1d\n\x15\x63urrent_stop_sequence\x18\x03 \x01(\r\x12\x0f\n\x07stop_id\x18\x07 \x01(\t\x12Z\n\x0e\x63urrent_status\x18\x04 \x01(\x0e\x32\x33.transit_realtime.VehiclePosition.VehicleStopStatus:\rIN_TRANSIT_TO\x12\x11\n\ttimestamp\x18\x05 \x01(\x04\x12K\n\x10\x63ongestion_level\x18\x06 \x01(\x0e\x32\x31.transit_realtime.VehiclePosition.CongestionLevel\x12K\n\x10occupancy_status\x18\t \x01(\x0e\x32\x31.transit_realtime.VehiclePosition.OccupancyStatus\x12\x1c\n\x14occupancy_percentage\x18\n \x01(\r\x12Q\n\x16multi_carriage_details\x18\x0b \x03(\x0b\x32\x31.transit_realtime.VehiclePosition.CarriageDetails\x1a\xd9\x01\n\x0f\x43\x61rriageDetails\x12\n\n\x02id\x18\x01 \x01(\t\x12\r\n\x05label\x18\x02 \x01(\t\x12^\n\x10occupancy_status\x18\x03 \x01(\x0e\x32\x31.transit_realtime.VehiclePosition.OccupancyStatus:\x11NO_DATA_AVAILABLE\x12 \n\x14occupancy_percentage\x18\x04 \x01(\x05:\x02-1\x12\x19\n\x11\x63\x61rriage_sequence\x18\x05 \x01(\r*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"G\n\x11VehicleStopStatus\x12\x0f\n\x0bINCOMING_AT\x10\x00\x12\x0e\n\nSTOPPED_AT\x10\x01\x12\x11\n\rIN_TRANSIT_TO\x10\x02\"}\n\x0f\x43ongestionLevel\x12\x1c\n\x18UNKNOWN_CONGESTION_LEVEL\x10\x00\x12\x14\n\x10RUNNING_SMOOTHLY\x10\x01\x12\x0f\n\x0bSTOP_AND_GO\x10\x02\x12\x0e\n\nCONGESTION\x10\x03\x12\x15\n\x11SEVERE_CONGESTION\x10\x04\"\xd9\x01\n\x0fOccupancyStatus\x12\t\n\x05\x45MPTY\x10\x00\x12\x18\n\x14MANY_SEATS_AVAILABLE\x10\x01\x12\x17\n\x13\x46\x45W_SEATS_AVAILABLE\x10\x02\x12\x16\n\x12STANDING_ROOM_ONLY\x10\x03\x12\x1e\n\x1a\x43RUSHED_STANDING_ROOM_ONLY\x10\x04\x12\x08\n\x04\x46ULL\x10\x05\x12\x1c\n\x18NOT_ACCEPTING_PASSENGERS\x10\x06\x12\x15\n\x11NO_DATA_AVAILABLE\x10\x07\x12\x11\n\rNOT_BOARDABLE\x10\x08*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\x80\t\n\x05\x41lert\x12\x32\n\ractive_period\x18\x01 \x03(\x0b\x32\x1b.transit_realtime.TimeRange\x12\x39\n\x0finformed_entity\x18\x05 \x03(\x0b\x32 .transit_realtime.EntitySelector\x12;\n\x05\x63\x61use\x18\x06 \x01(\x0e\x32\x1d.transit_realtime.Alert.Cause:\rUNKNOWN_CAUSE\x12>\n\x06\x65\x66\x66\x65\x63t\x18\x07 \x01(\x0e\x32\x1e.transit_realtime.Alert.Effect:\x0eUNKNOWN_EFFECT\x12/\n\x03url\x18\x08 \x01(\x0b\x32\".transit_realtime.TranslatedString\x12\x37\n\x0bheader_text\x18\n \x01(\x0b\x32\".transit_realtime.TranslatedString\x12<\n\x10\x64\x65scription_text\x18\x0b \x01(\x0b\x32\".transit_realtime.TranslatedString\x12;\n\x0ftts_header_text\x18\x0c \x01(\x0b\x32\".transit_realtime.TranslatedString\x12@\n\x14tts_description_text\x18\r \x01(\x0b\x32\".transit_realtime.TranslatedString\x12O\n\x0eseverity_level\x18\x0e \x01(\x0e\x32%.transit_realtime.Alert.SeverityLevel:\x10UNKNOWN_SEVERITY\"\xd8\x01\n\x05\x43\x61use\x12\x11\n\rUNKNOWN_CAUSE\x10\x01\x12\x0f\n\x0bOTHER_CAUSE\x10\x02\x12\x15\n\x11TECHNICAL_PROBLEM\x10\x03\x12\n\n\x06STRIKE\x10\x04\x12\x11\n\rDEMONSTRATION\x10\x05\x12\x0c\n\x08\x41\x43\x43IDENT\x10\x06\x12\x0b\n\x07HOLIDAY\x10\x07\x12\x0b\n\x07WEATHER\x10\x08\x12\x0f\n\x0bMAINTENANCE\x10\t\x12\x10\n\x0c\x43ONSTRUCTION\x10\n\x12\x13\n\x0fPOLICE_ACTIVITY\x10\x0b\x12\x15\n\x11MEDICAL_EMERGENCY\x10\x0c\"\xdd\x01\n\x06\x45\x66\x66\x65\x63t\x12\x0e\n\nNO_SERVICE\x10\x01\x12\x13\n\x0fREDUCED_SERVICE\x10\x02\x12\x16\n\x12SIGNIFICANT_DELAYS\x10\x03\x12\n\n\x06\x44\x45TOUR\x10\x04\x12\x16\n\x12\x41\x44\x44ITIONAL_SERVICE\x10\x05\x12\x14\n\x10MODIFIED_SERVICE\x10\x06\x12\x10\n\x0cOTHER_EFFECT\x10\x07\x12\x12\n\x0eUNKNOWN_EFFECT\x10\x08\x12\x0e\n\nSTOP_MOVED\x10\t\x12\r\n\tNO_EFFECT\x10\n\x12\x17\n\x13\x41\x43\x43\x45SSIBILITY_ISSUE\x10\x0b\"H\n\rSeverityLevel\x12\x14\n\x10UNKNOWN_SEVERITY\x10\x01\x12\x08\n\x04INFO\x10\x02\x12\x0b\n\x07WARNING\x10\x03\x12\n\n\x06SEVERE\x10\x04*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"7\n\tTimeRange\x12\r\n\x05start\x18\x01 \x01(\x04\x12\x0b\n\x03\x65nd\x18\x02 \x01(\x04*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"q\n\x08Position\x12\x10\n\x08latitude\x18\x01 \x02(\x02\x12\x11\n\tlongitude\x18\x02 \x02(\x02\x12\x0f\n\x07\x62\x65\x61ring\x18\x03 \x01(\x02\x12\x10\n\x08odometer\x18\x04 \x01(\x01\x12\r\n\x05speed\x18\x05 \x01(\x02*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\xcd\x02\n\x0eTripDescriptor\x12\x0f\n\x07trip_id\x18\x01 \x01(\t\x12\x10\n\x08route_id\x18\x05 \x01(\t\x12\x14\n\x0c\x64irection_id\x18\x06 \x01(\r\x12\x12\n\nstart_time\x18\x02 \x01(\t\x12\x12\n\nstart_date\x18\x03 \x01(\t\x12T\n\x15schedule_relationship\x18\x04 \x01(\x0e\x32\x35.transit_realtime.TripDescriptor.ScheduleRelationship\"t\n\x14ScheduleRelationship\x12\r\n\tSCHEDULED\x10\x00\x12\t\n\x05\x41\x44\x44\x45\x44\x10\x01\x12\x0f\n\x0bUNSCHEDULED\x10\x02\x12\x0c\n\x08\x43\x41NCELED\x10\x03\x12\x13\n\x0bREPLACEMENT\x10\x05\x1a\x02\x08\x01\x12\x0e\n\nDUPLICATED\x10\x06*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"U\n\x11VehicleDescriptor\x12\n\n\x02id\x18\x01 \x01(\t\x12\r\n\x05label\x18\x02 \x01(\t\x12\x15\n\rlicense_plate\x18\x03 \x01(\t*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\xb0\x01\n\x0e\x45ntitySelector\x12\x11\n\tagency_id\x18\x01 \x01(\t\x12\x10\n\x08route_id\x18\x02 \x01(\t\x12\x12\n\nroute_type\x18\x03 \x01(\x05\x12.\n\x04trip\x18\x04 \x01(\x0b\x32 .transit_realtime.TripDescriptor\x12\x0f\n\x07stop_id\x18\x05 \x01(\t\x12\x14\n\x0c\x64irection_id\x18\x06 \x01(\r*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\xa6\x01\n\x10TranslatedString\x12\x43\n\x0btranslation\x18\x01 \x03(\x0b\x32..transit_realtime.TranslatedString.Translation\x1a=\n\x0bTranslation\x12\x0c\n\x04text\x18\x01 \x02(\t\x12\x10\n\x08language\x18\x02 \x01(\t*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90NB\x1d\n\x1b\x63om.google.transit.realtime') - -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'gtfs_realtime_pb2', globals()) -if _descriptor._USE_C_DESCRIPTORS == False: - - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'\n\033com.google.transit.realtime' - _TRIPDESCRIPTOR_SCHEDULERELATIONSHIP.values_by_name["REPLACEMENT"]._options = None - _TRIPDESCRIPTOR_SCHEDULERELATIONSHIP.values_by_name["REPLACEMENT"]._serialized_options = b'\010\001' - _FEEDMESSAGE._serialized_start=41 - _FEEDMESSAGE._serialized_end=162 - _FEEDHEADER._serialized_start=165 - _FEEDHEADER._serialized_end=380 - _FEEDHEADER_INCREMENTALITY._serialized_start=312 - _FEEDHEADER_INCREMENTALITY._serialized_end=364 - _FEEDENTITY._serialized_start=383 - _FEEDENTITY._serialized_end=593 - _TRIPUPDATE._serialized_start=596 - _TRIPUPDATE._serialized_end=1622 - _TRIPUPDATE_STOPTIMEEVENT._serialized_start=887 - _TRIPUPDATE_STOPTIMEEVENT._serialized_end=968 - _TRIPUPDATE_STOPTIMEUPDATE._serialized_start=971 - _TRIPUPDATE_STOPTIMEUPDATE._serialized_end=1515 - _TRIPUPDATE_STOPTIMEUPDATE_STOPTIMEPROPERTIES._serialized_start=1355 - _TRIPUPDATE_STOPTIMEUPDATE_STOPTIMEPROPERTIES._serialized_end=1417 - _TRIPUPDATE_STOPTIMEUPDATE_SCHEDULERELATIONSHIP._serialized_start=1419 - _TRIPUPDATE_STOPTIMEUPDATE_SCHEDULERELATIONSHIP._serialized_end=1499 - _TRIPUPDATE_TRIPPROPERTIES._serialized_start=1517 - _TRIPUPDATE_TRIPPROPERTIES._serialized_end=1606 - _VEHICLEPOSITION._serialized_start=1625 - _VEHICLEPOSITION._serialized_end=2872 - _VEHICLEPOSITION_CARRIAGEDETAILS._serialized_start=2219 - _VEHICLEPOSITION_CARRIAGEDETAILS._serialized_end=2436 - _VEHICLEPOSITION_VEHICLESTOPSTATUS._serialized_start=2438 - _VEHICLEPOSITION_VEHICLESTOPSTATUS._serialized_end=2509 - _VEHICLEPOSITION_CONGESTIONLEVEL._serialized_start=2511 - _VEHICLEPOSITION_CONGESTIONLEVEL._serialized_end=2636 - _VEHICLEPOSITION_OCCUPANCYSTATUS._serialized_start=2639 - _VEHICLEPOSITION_OCCUPANCYSTATUS._serialized_end=2856 - _ALERT._serialized_start=2875 - _ALERT._serialized_end=4027 - _ALERT_CAUSE._serialized_start=3497 - _ALERT_CAUSE._serialized_end=3713 - _ALERT_EFFECT._serialized_start=3716 - _ALERT_EFFECT._serialized_end=3937 - _ALERT_SEVERITYLEVEL._serialized_start=3939 - _ALERT_SEVERITYLEVEL._serialized_end=4011 - _TIMERANGE._serialized_start=4029 - _TIMERANGE._serialized_end=4084 - _POSITION._serialized_start=4086 - _POSITION._serialized_end=4199 - _TRIPDESCRIPTOR._serialized_start=4202 - _TRIPDESCRIPTOR._serialized_end=4535 - _TRIPDESCRIPTOR_SCHEDULERELATIONSHIP._serialized_start=4403 - _TRIPDESCRIPTOR_SCHEDULERELATIONSHIP._serialized_end=4519 - _VEHICLEDESCRIPTOR._serialized_start=4537 - _VEHICLEDESCRIPTOR._serialized_end=4622 - _ENTITYSELECTOR._serialized_start=4625 - _ENTITYSELECTOR._serialized_end=4801 - _TRANSLATEDSTRING._serialized_start=4804 - _TRANSLATEDSTRING._serialized_end=4970 - _TRANSLATEDSTRING_TRANSLATION._serialized_start=4893 - _TRANSLATEDSTRING_TRANSLATION._serialized_end=4954 -# @@protoc_insertion_point(module_scope) diff --git a/amarillo/plugins/enhancer/services/gtfsrt/realtime_extension_pb2.py b/amarillo/plugins/enhancer/services/gtfsrt/realtime_extension_pb2.py deleted file mode 100644 index c2bbd7b..0000000 --- a/amarillo/plugins/enhancer/services/gtfsrt/realtime_extension_pb2.py +++ /dev/null @@ -1,33 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: realtime_extension.proto -"""Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder -from google.protobuf import descriptor as _descriptor -from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import symbol_database as _symbol_database -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -import amarillo.plugins.enhancer.services.gtfsrt.gtfs_realtime_pb2 as gtfs__realtime__pb2 - - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18realtime_extension.proto\x12\x10transit_realtime\x1a\x13gtfs-realtime.proto\"p\n\x1bMfdzTripDescriptorExtension\x12\x11\n\troute_url\x18\x01 \x01(\t\x12\x11\n\tagency_id\x18\x02 \x01(\t\x12\x17\n\x0froute_long_name\x18\x03 \x01(\t\x12\x12\n\nroute_type\x18\x04 \x01(\r\"\xb0\x02\n\x1fMfdzStopTimePropertiesExtension\x12X\n\x0bpickup_type\x18\x01 \x01(\x0e\x32\x43.transit_realtime.MfdzStopTimePropertiesExtension.DropOffPickupType\x12Y\n\x0c\x64ropoff_type\x18\x02 \x01(\x0e\x32\x43.transit_realtime.MfdzStopTimePropertiesExtension.DropOffPickupType\"X\n\x11\x44ropOffPickupType\x12\x0b\n\x07REGULAR\x10\x00\x12\x08\n\x04NONE\x10\x01\x12\x10\n\x0cPHONE_AGENCY\x10\x02\x12\x1a\n\x16\x43OORDINATE_WITH_DRIVER\x10\x03:i\n\x0ftrip_descriptor\x12 .transit_realtime.TripDescriptor\x18\xf5\x07 \x01(\x0b\x32-.transit_realtime.MfdzTripDescriptorExtension:\x90\x01\n\x14stop_time_properties\x12>.transit_realtime.TripUpdate.StopTimeUpdate.StopTimeProperties\x18\xf5\x07 \x01(\x0b\x32\x31.transit_realtime.MfdzStopTimePropertiesExtensionB\t\n\x07\x64\x65.mfdz') - -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'realtime_extension_pb2', globals()) -if _descriptor._USE_C_DESCRIPTORS == False: - gtfs__realtime__pb2.TripDescriptor.RegisterExtension(trip_descriptor) - gtfs__realtime__pb2.TripUpdate.StopTimeUpdate.StopTimeProperties.RegisterExtension(stop_time_properties) - - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'\n\007de.mfdz' - _MFDZTRIPDESCRIPTOREXTENSION._serialized_start=67 - _MFDZTRIPDESCRIPTOREXTENSION._serialized_end=179 - _MFDZSTOPTIMEPROPERTIESEXTENSION._serialized_start=182 - _MFDZSTOPTIMEPROPERTIESEXTENSION._serialized_end=486 - _MFDZSTOPTIMEPROPERTIESEXTENSION_DROPOFFPICKUPTYPE._serialized_start=398 - _MFDZSTOPTIMEPROPERTIESEXTENSION_DROPOFFPICKUPTYPE._serialized_end=486 -# @@protoc_insertion_point(module_scope) From c3e62c404e20701199d460538d10243aa1f93b7c Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Fri, 8 Mar 2024 13:15:18 +0100 Subject: [PATCH 33/33] Removed config and logging.conf --- config | 5 ----- logging.conf | 22 ---------------------- 2 files changed, 27 deletions(-) delete mode 100644 config delete mode 100644 logging.conf diff --git a/config b/config deleted file mode 100644 index fadf873..0000000 --- a/config +++ /dev/null @@ -1,5 +0,0 @@ -# Bounding-Box Germany -ride2go_query_data = '{ "southWestCoordinates": { "lat": 47.3, "lon": 5.98 }, "northEastCoordinates": { "lat": 54.99, "lon": 15.02 }, "lastModifiedSinceDays": 180 }' -env = 'PROD' -graphhopper_base_url = 'https://api.mfdz.de/gh' -stop_sources_file = 'conf/stop_sources.json' \ No newline at end of file diff --git a/logging.conf b/logging.conf deleted file mode 100644 index 429da8e..0000000 --- a/logging.conf +++ /dev/null @@ -1,22 +0,0 @@ -[loggers] -keys=root - -[handlers] -keys=consoleHandler - -[formatters] -keys=simpleFormatter - -[logger_root] -level=INFO -handlers=consoleHandler -propagate=yes - -[handler_consoleHandler] -class=StreamHandler -level=DEBUG -formatter=simpleFormatter -args=(sys.stdout,) - -[formatter_simpleFormatter] -format=%(asctime)s - %(name)s - %(levelname)s - %(message)s \ No newline at end of file