diff --git a/amarillo/plugins/enhancer/services/gtfs.py b/amarillo/plugins/enhancer/services/gtfs.py deleted file mode 100644 index d84b863..0000000 --- a/amarillo/plugins/enhancer/services/gtfs.py +++ /dev/null @@ -1,137 +0,0 @@ -import amarillo.plugins.enhancer.services.gtfsrt.gtfs_realtime_pb2 as gtfs_realtime_pb2 -import amarillo.plugins.enhancer.services.gtfsrt.realtime_extension_pb2 as mfdzrte -from amarillo.plugins.enhancer.services.gtfs_constants import * -from google.protobuf.json_format import MessageToDict -from google.protobuf.json_format import ParseDict -from datetime import datetime, timedelta -import json -import re -import time - -class GtfsRtProducer(): - - def __init__(self, trip_store): - self.trip_store = trip_store - - def generate_feed(self, time, format='protobuf', bbox=None): - # See https://developers.google.com/transit/gtfs-realtime/reference - # https://github.com/mfdz/carpool-gtfs-rt/blob/master/src/main/java/de/mfdz/resource/CarpoolResource.java - gtfsrt_dict = { - 'header': { - 'gtfsRealtimeVersion': '1.0', - 'timestamp': int(time) - }, - 'entity': self._get_trip_updates(bbox) - } - feed = gtfs_realtime_pb2.FeedMessage() - ParseDict(gtfsrt_dict, feed) - - if "message" == format.lower(): - return feed - elif "json" == format.lower(): - return MessageToDict(feed) - else: - return feed.SerializeToString() - - def export_feed(self, timestamp, file_path, bbox=None): - """ - Exports gtfs-rt feed as .json and .pbf file to file_path - """ - feed = self.generate_feed(timestamp, "message", bbox) - with open(f"{file_path}.pbf", "wb") as f: - f.write(feed.SerializeToString()) - with open(f"{file_path}.json", "w") as f: - json.dump(MessageToDict(feed), f) - - def _get_trip_updates(self, bbox = None): - trips = [] - trips.extend(self._get_added(bbox)) - trips.extend(self._get_deleted(bbox)) - trip_updates = [] - for num, trip in enumerate(trips): - trip_updates.append( { - 'id': f'carpool-update-{num}', - 'tripUpdate': trip - } - ) - return trip_updates - - def _get_deleted(self, bbox = None): - return self._get_updates( - self.trip_store.recently_deleted_trips(), - self._as_delete_updates, - bbox) - - def _get_added(self, bbox = None): - return self._get_updates( - self.trip_store.recently_added_trips(), - self._as_added_updates, - bbox) - - def _get_updates(self, trips, update_func, bbox = None): - updates = [] - today = datetime.today() - for t in trips: - if bbox == None or t.intersects(bbox): - updates.extend(update_func(t, today)) - return updates - - def _as_delete_updates(self, trip, fromdate): - return [{ - 'trip': { - 'tripId': trip.trip_id, - 'startTime': trip.start_time_str(), - 'startDate': trip_date, - 'scheduleRelationship': 'CANCELED', - 'routeId': trip.trip_id - } - } for trip_date in trip.next_trip_dates(fromdate)] - - def _to_seconds(self, fromdate, stop_time): - startdate = datetime.strptime(fromdate, '%Y%m%d') - m = re.search(r'(\d+):(\d+):(\d+)', stop_time) - delta = timedelta( - hours=int(m.group(1)), - minutes=int(m.group(2)), - seconds=int(m.group(3))) - return time.mktime((startdate + delta).timetuple()) - - def _to_stop_times(self, trip, fromdate): - return [{ - 'stopSequence': stoptime.stop_sequence, - 'arrival': { - 'time': self._to_seconds(fromdate, stoptime.arrival_time), - 'uncertainty': MFDZ_DEFAULT_UNCERTAINITY - }, - 'departure': { - 'time': self._to_seconds(fromdate, stoptime.departure_time), - 'uncertainty': MFDZ_DEFAULT_UNCERTAINITY - }, - 'stopId': stoptime.stop_id, - 'scheduleRelationship': 'SCHEDULED', - 'stop_time_properties': { - '[transit_realtime.stop_time_properties]': { - 'dropoffType': 'COORDINATE_WITH_DRIVER' if stoptime.drop_off_type == STOP_TIMES_STOP_TYPE_COORDINATE_DRIVER else 'NONE', - 'pickupType': 'COORDINATE_WITH_DRIVER' if stoptime.pickup_type == STOP_TIMES_STOP_TYPE_COORDINATE_DRIVER else 'NONE' - } - } - } - for stoptime in trip.stop_times] - - def _as_added_updates(self, trip, fromdate): - return [{ - 'trip': { - 'tripId': trip.trip_id, - 'startTime': trip.start_time_str(), - 'startDate': trip_date, - 'scheduleRelationship': 'ADDED', - 'routeId': trip.trip_id, - '[transit_realtime.trip_descriptor]': { - 'routeUrl' : trip.url, - 'agencyId' : trip.agency, - 'route_long_name' : trip.route_long_name(), - 'route_type': RIDESHARING_ROUTE_TYPE - } - }, - 'stopTimeUpdate': self._to_stop_times(trip, trip_date) - } for trip_date in trip.next_trip_dates(fromdate)] diff --git a/amarillo/plugins/enhancer/services/gtfs_export.py b/amarillo/plugins/enhancer/services/gtfs_export.py deleted file mode 100644 index 0fafa19..0000000 --- a/amarillo/plugins/enhancer/services/gtfs_export.py +++ /dev/null @@ -1,229 +0,0 @@ - -from collections.abc import Iterable -from datetime import datetime, timedelta -from zipfile import ZipFile -import csv -import gettext -import logging -import re - -from amarillo.utils.utils import assert_folder_exists -from amarillo.plugins.enhancer.models.gtfs import GtfsTimeDelta, GtfsFeedInfo, GtfsAgency, GtfsRoute, GtfsStop, GtfsStopTime, GtfsTrip, GtfsCalendar, GtfsCalendarDate, GtfsShape -from amarillo.plugins.enhancer.services.stops import is_carpooling_stop -from amarillo.plugins.enhancer.services.gtfs_constants import * - - -logger = logging.getLogger(__name__) - -class GtfsExport: - - stops_counter = 0 - trips_counter = 0 - routes_counter = 0 - - stored_stops = {} - - def __init__(self, agencies, feed_info, ridestore, stopstore, bbox = None): - self.stops = {} - self.routes = [] - self.calendar_dates = [] - self.calendar = [] - self.trips = [] - self.stop_times = [] - self.calendar = [] - self.shapes = [] - self.agencies = agencies - self.feed_info = feed_info - self.localized_to = " nach " - self.localized_short_name = "Mitfahrgelegenheit" - self.stopstore = stopstore - self.ridestore = ridestore - self.bbox = bbox - - def export(self, gtfszip_filename, gtfsfolder): - assert_folder_exists(gtfsfolder) - self._prepare_gtfs_feed(self.ridestore, self.stopstore) - self._write_csvfile(gtfsfolder, 'agency.txt', self.agencies) - self._write_csvfile(gtfsfolder, 'feed_info.txt', self.feed_info) - self._write_csvfile(gtfsfolder, 'routes.txt', self.routes) - self._write_csvfile(gtfsfolder, 'trips.txt', self.trips) - self._write_csvfile(gtfsfolder, 'calendar.txt', self.calendar) - self._write_csvfile(gtfsfolder, 'calendar_dates.txt', self.calendar_dates) - self._write_csvfile(gtfsfolder, 'stops.txt', self.stops.values()) - self._write_csvfile(gtfsfolder, 'stop_times.txt', self.stop_times) - self._write_csvfile(gtfsfolder, 'shapes.txt', self.shapes) - self._zip_files(gtfszip_filename, gtfsfolder) - - def _zip_files(self, gtfszip_filename, gtfsfolder): - gtfsfiles = ['agency.txt', 'feed_info.txt', 'routes.txt', 'trips.txt', - 'calendar.txt', 'calendar_dates.txt', 'stops.txt', 'stop_times.txt', 'shapes.txt'] - with ZipFile(gtfszip_filename, 'w') as gtfszip: - for gtfsfile in gtfsfiles: - gtfszip.write(gtfsfolder+'/'+gtfsfile, gtfsfile) - - def _prepare_gtfs_feed(self, ridestore, stopstore): - """ - Prepares all gtfs objects in memory before they are written - to their respective streams. - - For all wellknown stops a GTFS stop is created and - afterwards all ride offers are transformed into their - gtfs equivalents. - """ - for stopSet in stopstore.stopsDataFrames: - for stop in stopSet["stops"].itertuples(): - self._load_stored_stop(stop) - cloned_trips = dict(ridestore.trips) - for url, trip in cloned_trips.items(): - if self.bbox is None or trip.intersects(self.bbox): - self._convert_trip(trip) - - def _convert_trip(self, trip): - self.routes_counter += 1 - self.routes.append(self._create_route(trip)) - self.calendar.append(self._create_calendar(trip)) - if not trip.runs_regularly: - self.calendar_dates.append(self._create_calendar_date(trip)) - self.trips.append(self._create_trip(trip, self.routes_counter)) - self._append_stops_and_stop_times(trip) - self._append_shapes(trip, self.routes_counter) - - def _trip_headsign(self, destination): - destination = destination.replace('(Deutschland)', '') - destination = destination.replace(', Deutschland', '') - appendix = '' - if 'Schweiz' in destination or 'Switzerland' in destination: - appendix = ', Schweiz' - destination = destination.replace('(Schweiz)', '') - destination = destination.replace(', Schweiz', '') - destination = destination.replace('(Switzerland)', '') - - try: - matches = re.match(r"(.*,)? ?(\d{4,5})? ?(.*)", destination) - - match = matches.group(3).strip() if matches != None else destination.strip() - if match[-1]==')' and not '(' in match: - match = match[0:-1] - - return match + appendix - except Exception as ex: - logger.error("error for "+destination ) - logger.exception(ex) - return destination - - def _create_route(self, trip): - return GtfsRoute(trip.agency, trip.trip_id, trip.route_long_name(), RIDESHARING_ROUTE_TYPE, trip.url, "") - - def _create_calendar(self, trip): - # TODO currently, calendar is not provided by Fahrgemeinschaft.de interface. - # We could apply some heuristics like requesting multiple days and extrapolate - # if multiple trips are found, but better would be to have these provided by the - # offical interface. Then validity periods should be provided as well (not - # sure if these are available) - # For fahrgemeinschaft.de, regurlar trips are recognizable via their url - # which contains "regelmaessig". However, we don't know on which days of the week, - # nor until when. As a first guess, if datetime is a mo-fr, we assume each workday, - # if it's sa/su, only this... - - feed_start_date = datetime.today() - stop_date = self._convert_stop_date(feed_start_date) - return GtfsCalendar(trip.trip_id, stop_date, self._convert_stop_date(feed_start_date + timedelta(days=31)), *(trip.weekdays)) - - def _create_calendar_date(self, trip): - return GtfsCalendarDate(trip.trip_id, self._convert_stop_date(trip.start), CALENDAR_DATES_EXCEPTION_TYPE_ADDED) - - def _create_trip(self, trip, shape_id): - return GtfsTrip(trip.trip_id, trip.trip_id, trip.trip_id, shape_id, trip.trip_headsign, NO_BIKES_ALLOWED) - - def _convert_stop(self, stop): - """ - Converts a stop represented as pandas row to a gtfs stop. - Expected attributes of stop: id, stop_name, x, y (in wgs84) - """ - if stop.id: - id = stop.id - else: - self.stops_counter += 1 - id = "tmp-{}".format(self.stops_counter) - - stop_name = "k.A." if stop.stop_name is None else stop.stop_name - return GtfsStop(id, stop.y, stop.x, stop_name) - - def _append_stops_and_stop_times(self, trip): - # Assumptions: - # arrival_time = departure_time - # pickup_type, drop_off_type for origin: = coordinate/none - # pickup_type, drop_off_type for destination: = none/coordinate - # timepoint = approximate for origin and destination (not sure what consequences this might have for trip planners) - for stop_time in trip.stop_times: - # retrieve stop from stored_stops and mark it to be exported - wkn_stop = self.stored_stops.get(stop_time.stop_id) - if not wkn_stop: - logger.warning("No stop found in stop_store for %s. Will skip stop_time %s of trip %s", stop_time.stop_id, stop_time.stop_sequence, trip.trip_id) - else: - self.stops[stop_time.stop_id] = wkn_stop - # Append stop_time - self.stop_times.append(stop_time) - - def _append_shapes(self, trip, shape_id): - counter = 0 - for point in trip.path.coordinates: - counter += 1 - self.shapes.append(GtfsShape(shape_id, point[0], point[1], counter)) - - def _stop_hash(self, stop): - return "{}#{}#{}".format(stop.stop_name,stop.x,stop.y) - - def _should_always_export(self, stop): - """ - Returns true, if the given stop shall be exported to GTFS, - regardless, if it's part of a trip or not. - - This is necessary, as potential stops are required - to be part of the GTFS to be referenced later on - by dynamicly added trips. - """ - if self.bbox: - return (self.bbox[0] <= stop.stop_lon <= self.bbox[2] and - self.bbox[1] <= stop.stop_lat <= self.bbox[3]) - else: - return is_carpooling_stop(stop.stop_id, stop.stop_name) - - def _load_stored_stop(self, stop): - gtfsstop = self._convert_stop(stop) - stop_hash = self._stop_hash(stop) - self.stored_stops[gtfsstop.stop_id] = gtfsstop - if self._should_always_export(gtfsstop): - self.stops[gtfsstop.stop_id] = gtfsstop - - def _get_stop_by_hash(self, stop_hash): - return self.stops.get(stop_hash, self.stored_stops.get(stop_hash)) - - def _get_or_create_stop(self, stop): - stop_hash = self._stop_hash(stop) - gtfsstop = self.stops.get(stop_hash) - if gtfsstop is None: - gtfsstop = self.stored_stops.get(stop_hash, self._convert_stop(stop)) - self.stops[stop_hash] = gtfsstop - return gtfsstop - - def _convert_stop_date(self, date_time): - return date_time.strftime("%Y%m%d") - - def _write_csvfile(self, gtfsfolder, filename, content): - with open(gtfsfolder+"/"+filename, 'w', newline="\n", encoding="utf-8") as csvfile: - self._write_csv(csvfile, content) - - def _write_csv(self, csvfile, content): - if hasattr(content, '_fields'): - writer = csv.DictWriter(csvfile, content._fields) - writer.writeheader() - writer.writerow(content._asdict()) - else: - if content: - writer = csv.DictWriter(csvfile, next(iter(content))._fields) - writer.writeheader() - for record in content: - writer.writerow(record._asdict()) - - \ No newline at end of file diff --git a/amarillo/plugins/enhancer/services/gtfs_generator.py b/amarillo/plugins/enhancer/services/gtfs_generator.py deleted file mode 100644 index 60f024b..0000000 --- a/amarillo/plugins/enhancer/services/gtfs_generator.py +++ /dev/null @@ -1,71 +0,0 @@ -from amarillo.models.Carpool import Region -from amarillo.plugins.enhancer.services.gtfs_export import GtfsExport, GtfsFeedInfo, GtfsAgency -from amarillo.plugins.enhancer.services.gtfs import GtfsRtProducer -from amarillo.utils.container import container -from glob import glob -import json -import schedule -import threading -import time -import logging -from datetime import date, timedelta - -logger = logging.getLogger(__name__) - -regions = {} -for region_file_name in glob('conf/region/*.json'): - with open(region_file_name) as region_file: - dict = json.load(region_file) - region = Region(**dict) - region_id = region.id - regions[region_id] = region - -agencies = [] -for agency_file_name in glob('conf/agency/*.json'): - with open(agency_file_name) as agency_file: - dict = json.load(agency_file) - agency = GtfsAgency(dict["id"], dict["name"], dict["url"], dict["timezone"], dict["lang"], dict["email"]) - agency_id = agency.agency_id - agencies.append(agency) - -def run_schedule(): - while 1: - try: - schedule.run_pending() - except Exception as e: - logger.exception(e) - time.sleep(1) - -def midnight(): - container['stops_store'].load_stop_sources() - container['trips_store'].unflag_unrecent_updates() - container['carpools'].purge_outdated_offers() - generate_gtfs() - -def generate_gtfs(): - logger.info("Generate GTFS") - - for region in regions.values(): - # TODO make feed producer infos configurable - feed_info = GtfsFeedInfo('mfdz', 'MITFAHR|DE|ZENTRALE', 'http://www.mitfahrdezentrale.de', 'de', 1) - exporter = GtfsExport( - agencies, - feed_info, - container['trips_store'], - container['stops_store'], - region.bbox) - exporter.export(f"data/gtfs/amarillo.{region.id}.gtfs.zip", "data/tmp/") - -def generate_gtfs_rt(): - logger.info("Generate GTFS-RT") - producer = GtfsRtProducer(container['trips_store']) - for region in regions.values(): - rt = producer.export_feed(time.time(), f"data/gtfs/amarillo.{region.id}.gtfsrt", bbox=region.bbox) - -def start_schedule(): - schedule.every().day.at("00:00").do(midnight) - schedule.every(60).seconds.do(generate_gtfs_rt) - # Create all feeds once at startup - schedule.run_all() - job_thread = threading.Thread(target=run_schedule, daemon=True) - job_thread.start() \ No newline at end of file diff --git a/amarillo/plugins/enhancer/services/gtfsrt/__init__.py b/amarillo/plugins/enhancer/services/gtfsrt/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/amarillo/plugins/enhancer/services/gtfsrt/gtfs_realtime_pb2.py b/amarillo/plugins/enhancer/services/gtfsrt/gtfs_realtime_pb2.py deleted file mode 100644 index 4e10463..0000000 --- a/amarillo/plugins/enhancer/services/gtfsrt/gtfs_realtime_pb2.py +++ /dev/null @@ -1,80 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: gtfs-realtime.proto -"""Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder -from google.protobuf import descriptor as _descriptor -from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import symbol_database as _symbol_database -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x13gtfs-realtime.proto\x12\x10transit_realtime\"y\n\x0b\x46\x65\x65\x64Message\x12,\n\x06header\x18\x01 \x02(\x0b\x32\x1c.transit_realtime.FeedHeader\x12,\n\x06\x65ntity\x18\x02 \x03(\x0b\x32\x1c.transit_realtime.FeedEntity*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\xd7\x01\n\nFeedHeader\x12\x1d\n\x15gtfs_realtime_version\x18\x01 \x02(\t\x12Q\n\x0eincrementality\x18\x02 \x01(\x0e\x32+.transit_realtime.FeedHeader.Incrementality:\x0c\x46ULL_DATASET\x12\x11\n\ttimestamp\x18\x03 \x01(\x04\"4\n\x0eIncrementality\x12\x10\n\x0c\x46ULL_DATASET\x10\x00\x12\x10\n\x0c\x44IFFERENTIAL\x10\x01*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\xd2\x01\n\nFeedEntity\x12\n\n\x02id\x18\x01 \x02(\t\x12\x19\n\nis_deleted\x18\x02 \x01(\x08:\x05\x66\x61lse\x12\x31\n\x0btrip_update\x18\x03 \x01(\x0b\x32\x1c.transit_realtime.TripUpdate\x12\x32\n\x07vehicle\x18\x04 \x01(\x0b\x32!.transit_realtime.VehiclePosition\x12&\n\x05\x61lert\x18\x05 \x01(\x0b\x32\x17.transit_realtime.Alert*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\x82\x08\n\nTripUpdate\x12.\n\x04trip\x18\x01 \x02(\x0b\x32 .transit_realtime.TripDescriptor\x12\x34\n\x07vehicle\x18\x03 \x01(\x0b\x32#.transit_realtime.VehicleDescriptor\x12\x45\n\x10stop_time_update\x18\x02 \x03(\x0b\x32+.transit_realtime.TripUpdate.StopTimeUpdate\x12\x11\n\ttimestamp\x18\x04 \x01(\x04\x12\r\n\x05\x64\x65lay\x18\x05 \x01(\x05\x12\x44\n\x0ftrip_properties\x18\x06 \x01(\x0b\x32+.transit_realtime.TripUpdate.TripProperties\x1aQ\n\rStopTimeEvent\x12\r\n\x05\x64\x65lay\x18\x01 \x01(\x05\x12\x0c\n\x04time\x18\x02 \x01(\x03\x12\x13\n\x0buncertainty\x18\x03 \x01(\x05*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\x1a\xa0\x04\n\x0eStopTimeUpdate\x12\x15\n\rstop_sequence\x18\x01 \x01(\r\x12\x0f\n\x07stop_id\x18\x04 \x01(\t\x12;\n\x07\x61rrival\x18\x02 \x01(\x0b\x32*.transit_realtime.TripUpdate.StopTimeEvent\x12=\n\tdeparture\x18\x03 \x01(\x0b\x32*.transit_realtime.TripUpdate.StopTimeEvent\x12j\n\x15schedule_relationship\x18\x05 \x01(\x0e\x32@.transit_realtime.TripUpdate.StopTimeUpdate.ScheduleRelationship:\tSCHEDULED\x12\\\n\x14stop_time_properties\x18\x06 \x01(\x0b\x32>.transit_realtime.TripUpdate.StopTimeUpdate.StopTimeProperties\x1a>\n\x12StopTimeProperties\x12\x18\n\x10\x61ssigned_stop_id\x18\x01 \x01(\t*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"P\n\x14ScheduleRelationship\x12\r\n\tSCHEDULED\x10\x00\x12\x0b\n\x07SKIPPED\x10\x01\x12\x0b\n\x07NO_DATA\x10\x02\x12\x0f\n\x0bUNSCHEDULED\x10\x03*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\x1aY\n\x0eTripProperties\x12\x0f\n\x07trip_id\x18\x01 \x01(\t\x12\x12\n\nstart_date\x18\x02 \x01(\t\x12\x12\n\nstart_time\x18\x03 \x01(\t*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\xdf\t\n\x0fVehiclePosition\x12.\n\x04trip\x18\x01 \x01(\x0b\x32 .transit_realtime.TripDescriptor\x12\x34\n\x07vehicle\x18\x08 \x01(\x0b\x32#.transit_realtime.VehicleDescriptor\x12,\n\x08position\x18\x02 \x01(\x0b\x32\x1a.transit_realtime.Position\x12\x1d\n\x15\x63urrent_stop_sequence\x18\x03 \x01(\r\x12\x0f\n\x07stop_id\x18\x07 \x01(\t\x12Z\n\x0e\x63urrent_status\x18\x04 \x01(\x0e\x32\x33.transit_realtime.VehiclePosition.VehicleStopStatus:\rIN_TRANSIT_TO\x12\x11\n\ttimestamp\x18\x05 \x01(\x04\x12K\n\x10\x63ongestion_level\x18\x06 \x01(\x0e\x32\x31.transit_realtime.VehiclePosition.CongestionLevel\x12K\n\x10occupancy_status\x18\t \x01(\x0e\x32\x31.transit_realtime.VehiclePosition.OccupancyStatus\x12\x1c\n\x14occupancy_percentage\x18\n \x01(\r\x12Q\n\x16multi_carriage_details\x18\x0b \x03(\x0b\x32\x31.transit_realtime.VehiclePosition.CarriageDetails\x1a\xd9\x01\n\x0f\x43\x61rriageDetails\x12\n\n\x02id\x18\x01 \x01(\t\x12\r\n\x05label\x18\x02 \x01(\t\x12^\n\x10occupancy_status\x18\x03 \x01(\x0e\x32\x31.transit_realtime.VehiclePosition.OccupancyStatus:\x11NO_DATA_AVAILABLE\x12 \n\x14occupancy_percentage\x18\x04 \x01(\x05:\x02-1\x12\x19\n\x11\x63\x61rriage_sequence\x18\x05 \x01(\r*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"G\n\x11VehicleStopStatus\x12\x0f\n\x0bINCOMING_AT\x10\x00\x12\x0e\n\nSTOPPED_AT\x10\x01\x12\x11\n\rIN_TRANSIT_TO\x10\x02\"}\n\x0f\x43ongestionLevel\x12\x1c\n\x18UNKNOWN_CONGESTION_LEVEL\x10\x00\x12\x14\n\x10RUNNING_SMOOTHLY\x10\x01\x12\x0f\n\x0bSTOP_AND_GO\x10\x02\x12\x0e\n\nCONGESTION\x10\x03\x12\x15\n\x11SEVERE_CONGESTION\x10\x04\"\xd9\x01\n\x0fOccupancyStatus\x12\t\n\x05\x45MPTY\x10\x00\x12\x18\n\x14MANY_SEATS_AVAILABLE\x10\x01\x12\x17\n\x13\x46\x45W_SEATS_AVAILABLE\x10\x02\x12\x16\n\x12STANDING_ROOM_ONLY\x10\x03\x12\x1e\n\x1a\x43RUSHED_STANDING_ROOM_ONLY\x10\x04\x12\x08\n\x04\x46ULL\x10\x05\x12\x1c\n\x18NOT_ACCEPTING_PASSENGERS\x10\x06\x12\x15\n\x11NO_DATA_AVAILABLE\x10\x07\x12\x11\n\rNOT_BOARDABLE\x10\x08*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\x80\t\n\x05\x41lert\x12\x32\n\ractive_period\x18\x01 \x03(\x0b\x32\x1b.transit_realtime.TimeRange\x12\x39\n\x0finformed_entity\x18\x05 \x03(\x0b\x32 .transit_realtime.EntitySelector\x12;\n\x05\x63\x61use\x18\x06 \x01(\x0e\x32\x1d.transit_realtime.Alert.Cause:\rUNKNOWN_CAUSE\x12>\n\x06\x65\x66\x66\x65\x63t\x18\x07 \x01(\x0e\x32\x1e.transit_realtime.Alert.Effect:\x0eUNKNOWN_EFFECT\x12/\n\x03url\x18\x08 \x01(\x0b\x32\".transit_realtime.TranslatedString\x12\x37\n\x0bheader_text\x18\n \x01(\x0b\x32\".transit_realtime.TranslatedString\x12<\n\x10\x64\x65scription_text\x18\x0b \x01(\x0b\x32\".transit_realtime.TranslatedString\x12;\n\x0ftts_header_text\x18\x0c \x01(\x0b\x32\".transit_realtime.TranslatedString\x12@\n\x14tts_description_text\x18\r \x01(\x0b\x32\".transit_realtime.TranslatedString\x12O\n\x0eseverity_level\x18\x0e \x01(\x0e\x32%.transit_realtime.Alert.SeverityLevel:\x10UNKNOWN_SEVERITY\"\xd8\x01\n\x05\x43\x61use\x12\x11\n\rUNKNOWN_CAUSE\x10\x01\x12\x0f\n\x0bOTHER_CAUSE\x10\x02\x12\x15\n\x11TECHNICAL_PROBLEM\x10\x03\x12\n\n\x06STRIKE\x10\x04\x12\x11\n\rDEMONSTRATION\x10\x05\x12\x0c\n\x08\x41\x43\x43IDENT\x10\x06\x12\x0b\n\x07HOLIDAY\x10\x07\x12\x0b\n\x07WEATHER\x10\x08\x12\x0f\n\x0bMAINTENANCE\x10\t\x12\x10\n\x0c\x43ONSTRUCTION\x10\n\x12\x13\n\x0fPOLICE_ACTIVITY\x10\x0b\x12\x15\n\x11MEDICAL_EMERGENCY\x10\x0c\"\xdd\x01\n\x06\x45\x66\x66\x65\x63t\x12\x0e\n\nNO_SERVICE\x10\x01\x12\x13\n\x0fREDUCED_SERVICE\x10\x02\x12\x16\n\x12SIGNIFICANT_DELAYS\x10\x03\x12\n\n\x06\x44\x45TOUR\x10\x04\x12\x16\n\x12\x41\x44\x44ITIONAL_SERVICE\x10\x05\x12\x14\n\x10MODIFIED_SERVICE\x10\x06\x12\x10\n\x0cOTHER_EFFECT\x10\x07\x12\x12\n\x0eUNKNOWN_EFFECT\x10\x08\x12\x0e\n\nSTOP_MOVED\x10\t\x12\r\n\tNO_EFFECT\x10\n\x12\x17\n\x13\x41\x43\x43\x45SSIBILITY_ISSUE\x10\x0b\"H\n\rSeverityLevel\x12\x14\n\x10UNKNOWN_SEVERITY\x10\x01\x12\x08\n\x04INFO\x10\x02\x12\x0b\n\x07WARNING\x10\x03\x12\n\n\x06SEVERE\x10\x04*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"7\n\tTimeRange\x12\r\n\x05start\x18\x01 \x01(\x04\x12\x0b\n\x03\x65nd\x18\x02 \x01(\x04*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"q\n\x08Position\x12\x10\n\x08latitude\x18\x01 \x02(\x02\x12\x11\n\tlongitude\x18\x02 \x02(\x02\x12\x0f\n\x07\x62\x65\x61ring\x18\x03 \x01(\x02\x12\x10\n\x08odometer\x18\x04 \x01(\x01\x12\r\n\x05speed\x18\x05 \x01(\x02*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\xcd\x02\n\x0eTripDescriptor\x12\x0f\n\x07trip_id\x18\x01 \x01(\t\x12\x10\n\x08route_id\x18\x05 \x01(\t\x12\x14\n\x0c\x64irection_id\x18\x06 \x01(\r\x12\x12\n\nstart_time\x18\x02 \x01(\t\x12\x12\n\nstart_date\x18\x03 \x01(\t\x12T\n\x15schedule_relationship\x18\x04 \x01(\x0e\x32\x35.transit_realtime.TripDescriptor.ScheduleRelationship\"t\n\x14ScheduleRelationship\x12\r\n\tSCHEDULED\x10\x00\x12\t\n\x05\x41\x44\x44\x45\x44\x10\x01\x12\x0f\n\x0bUNSCHEDULED\x10\x02\x12\x0c\n\x08\x43\x41NCELED\x10\x03\x12\x13\n\x0bREPLACEMENT\x10\x05\x1a\x02\x08\x01\x12\x0e\n\nDUPLICATED\x10\x06*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"U\n\x11VehicleDescriptor\x12\n\n\x02id\x18\x01 \x01(\t\x12\r\n\x05label\x18\x02 \x01(\t\x12\x15\n\rlicense_plate\x18\x03 \x01(\t*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\xb0\x01\n\x0e\x45ntitySelector\x12\x11\n\tagency_id\x18\x01 \x01(\t\x12\x10\n\x08route_id\x18\x02 \x01(\t\x12\x12\n\nroute_type\x18\x03 \x01(\x05\x12.\n\x04trip\x18\x04 \x01(\x0b\x32 .transit_realtime.TripDescriptor\x12\x0f\n\x07stop_id\x18\x05 \x01(\t\x12\x14\n\x0c\x64irection_id\x18\x06 \x01(\r*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\xa6\x01\n\x10TranslatedString\x12\x43\n\x0btranslation\x18\x01 \x03(\x0b\x32..transit_realtime.TranslatedString.Translation\x1a=\n\x0bTranslation\x12\x0c\n\x04text\x18\x01 \x02(\t\x12\x10\n\x08language\x18\x02 \x01(\t*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90NB\x1d\n\x1b\x63om.google.transit.realtime') - -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'gtfs_realtime_pb2', globals()) -if _descriptor._USE_C_DESCRIPTORS == False: - - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'\n\033com.google.transit.realtime' - _TRIPDESCRIPTOR_SCHEDULERELATIONSHIP.values_by_name["REPLACEMENT"]._options = None - _TRIPDESCRIPTOR_SCHEDULERELATIONSHIP.values_by_name["REPLACEMENT"]._serialized_options = b'\010\001' - _FEEDMESSAGE._serialized_start=41 - _FEEDMESSAGE._serialized_end=162 - _FEEDHEADER._serialized_start=165 - _FEEDHEADER._serialized_end=380 - _FEEDHEADER_INCREMENTALITY._serialized_start=312 - _FEEDHEADER_INCREMENTALITY._serialized_end=364 - _FEEDENTITY._serialized_start=383 - _FEEDENTITY._serialized_end=593 - _TRIPUPDATE._serialized_start=596 - _TRIPUPDATE._serialized_end=1622 - _TRIPUPDATE_STOPTIMEEVENT._serialized_start=887 - _TRIPUPDATE_STOPTIMEEVENT._serialized_end=968 - _TRIPUPDATE_STOPTIMEUPDATE._serialized_start=971 - _TRIPUPDATE_STOPTIMEUPDATE._serialized_end=1515 - _TRIPUPDATE_STOPTIMEUPDATE_STOPTIMEPROPERTIES._serialized_start=1355 - _TRIPUPDATE_STOPTIMEUPDATE_STOPTIMEPROPERTIES._serialized_end=1417 - _TRIPUPDATE_STOPTIMEUPDATE_SCHEDULERELATIONSHIP._serialized_start=1419 - _TRIPUPDATE_STOPTIMEUPDATE_SCHEDULERELATIONSHIP._serialized_end=1499 - _TRIPUPDATE_TRIPPROPERTIES._serialized_start=1517 - _TRIPUPDATE_TRIPPROPERTIES._serialized_end=1606 - _VEHICLEPOSITION._serialized_start=1625 - _VEHICLEPOSITION._serialized_end=2872 - _VEHICLEPOSITION_CARRIAGEDETAILS._serialized_start=2219 - _VEHICLEPOSITION_CARRIAGEDETAILS._serialized_end=2436 - _VEHICLEPOSITION_VEHICLESTOPSTATUS._serialized_start=2438 - _VEHICLEPOSITION_VEHICLESTOPSTATUS._serialized_end=2509 - _VEHICLEPOSITION_CONGESTIONLEVEL._serialized_start=2511 - _VEHICLEPOSITION_CONGESTIONLEVEL._serialized_end=2636 - _VEHICLEPOSITION_OCCUPANCYSTATUS._serialized_start=2639 - _VEHICLEPOSITION_OCCUPANCYSTATUS._serialized_end=2856 - _ALERT._serialized_start=2875 - _ALERT._serialized_end=4027 - _ALERT_CAUSE._serialized_start=3497 - _ALERT_CAUSE._serialized_end=3713 - _ALERT_EFFECT._serialized_start=3716 - _ALERT_EFFECT._serialized_end=3937 - _ALERT_SEVERITYLEVEL._serialized_start=3939 - _ALERT_SEVERITYLEVEL._serialized_end=4011 - _TIMERANGE._serialized_start=4029 - _TIMERANGE._serialized_end=4084 - _POSITION._serialized_start=4086 - _POSITION._serialized_end=4199 - _TRIPDESCRIPTOR._serialized_start=4202 - _TRIPDESCRIPTOR._serialized_end=4535 - _TRIPDESCRIPTOR_SCHEDULERELATIONSHIP._serialized_start=4403 - _TRIPDESCRIPTOR_SCHEDULERELATIONSHIP._serialized_end=4519 - _VEHICLEDESCRIPTOR._serialized_start=4537 - _VEHICLEDESCRIPTOR._serialized_end=4622 - _ENTITYSELECTOR._serialized_start=4625 - _ENTITYSELECTOR._serialized_end=4801 - _TRANSLATEDSTRING._serialized_start=4804 - _TRANSLATEDSTRING._serialized_end=4970 - _TRANSLATEDSTRING_TRANSLATION._serialized_start=4893 - _TRANSLATEDSTRING_TRANSLATION._serialized_end=4954 -# @@protoc_insertion_point(module_scope) diff --git a/amarillo/plugins/enhancer/services/gtfsrt/realtime_extension_pb2.py b/amarillo/plugins/enhancer/services/gtfsrt/realtime_extension_pb2.py deleted file mode 100644 index c2bbd7b..0000000 --- a/amarillo/plugins/enhancer/services/gtfsrt/realtime_extension_pb2.py +++ /dev/null @@ -1,33 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: realtime_extension.proto -"""Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder -from google.protobuf import descriptor as _descriptor -from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import symbol_database as _symbol_database -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -import amarillo.plugins.enhancer.services.gtfsrt.gtfs_realtime_pb2 as gtfs__realtime__pb2 - - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18realtime_extension.proto\x12\x10transit_realtime\x1a\x13gtfs-realtime.proto\"p\n\x1bMfdzTripDescriptorExtension\x12\x11\n\troute_url\x18\x01 \x01(\t\x12\x11\n\tagency_id\x18\x02 \x01(\t\x12\x17\n\x0froute_long_name\x18\x03 \x01(\t\x12\x12\n\nroute_type\x18\x04 \x01(\r\"\xb0\x02\n\x1fMfdzStopTimePropertiesExtension\x12X\n\x0bpickup_type\x18\x01 \x01(\x0e\x32\x43.transit_realtime.MfdzStopTimePropertiesExtension.DropOffPickupType\x12Y\n\x0c\x64ropoff_type\x18\x02 \x01(\x0e\x32\x43.transit_realtime.MfdzStopTimePropertiesExtension.DropOffPickupType\"X\n\x11\x44ropOffPickupType\x12\x0b\n\x07REGULAR\x10\x00\x12\x08\n\x04NONE\x10\x01\x12\x10\n\x0cPHONE_AGENCY\x10\x02\x12\x1a\n\x16\x43OORDINATE_WITH_DRIVER\x10\x03:i\n\x0ftrip_descriptor\x12 .transit_realtime.TripDescriptor\x18\xf5\x07 \x01(\x0b\x32-.transit_realtime.MfdzTripDescriptorExtension:\x90\x01\n\x14stop_time_properties\x12>.transit_realtime.TripUpdate.StopTimeUpdate.StopTimeProperties\x18\xf5\x07 \x01(\x0b\x32\x31.transit_realtime.MfdzStopTimePropertiesExtensionB\t\n\x07\x64\x65.mfdz') - -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'realtime_extension_pb2', globals()) -if _descriptor._USE_C_DESCRIPTORS == False: - gtfs__realtime__pb2.TripDescriptor.RegisterExtension(trip_descriptor) - gtfs__realtime__pb2.TripUpdate.StopTimeUpdate.StopTimeProperties.RegisterExtension(stop_time_properties) - - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'\n\007de.mfdz' - _MFDZTRIPDESCRIPTOREXTENSION._serialized_start=67 - _MFDZTRIPDESCRIPTOREXTENSION._serialized_end=179 - _MFDZSTOPTIMEPROPERTIESEXTENSION._serialized_start=182 - _MFDZSTOPTIMEPROPERTIESEXTENSION._serialized_end=486 - _MFDZSTOPTIMEPROPERTIESEXTENSION_DROPOFFPICKUPTYPE._serialized_start=398 - _MFDZSTOPTIMEPROPERTIESEXTENSION_DROPOFFPICKUPTYPE._serialized_end=486 -# @@protoc_insertion_point(module_scope) diff --git a/amarillo/plugins/enhancer/tests/test_gtfs.py b/amarillo/plugins/enhancer/tests/test_gtfs.py deleted file mode 100644 index 3fbe97c..0000000 --- a/amarillo/plugins/enhancer/tests/test_gtfs.py +++ /dev/null @@ -1,142 +0,0 @@ -from amarillo.tests.sampledata import carpool_1234, data1, carpool_repeating_json, stop_issue -from amarillo.plugins.enhancer.services.gtfs_export import GtfsExport -from amarillo.plugins.enhancer.services.gtfs import GtfsRtProducer -from amarillo.plugins.enhancer.services.stops import StopsStore -from amarillo.plugins.enhancer.services.trips import TripStore -from amarillo.models.Carpool import Carpool -from datetime import datetime -import time -import pytest - - -def test_gtfs_generation(): - cp = Carpool(**data1) - stops_store = StopsStore() - trips_store = TripStore(stops_store) - trips_store.put_carpool(cp) - - exporter = GtfsExport(None, None, trips_store, stops_store) - exporter.export('target/tests/test_gtfs_generation/test.gtfs.zip', "target/tests/test_gtfs_generation") - -def test_correct_stops(): - cp = Carpool(**stop_issue) - stops_store = StopsStore([{"url": "https://datahub.bbnavi.de/export/rideshare_points.geojson", "vicinity": 250}]) - stops_store.load_stop_sources() - trips_store = TripStore(stops_store) - trips_store.put_carpool(cp) - assert len(trips_store.trips) == 1 - - -class TestTripConverter: - - def setup_method(self, method): - self.stops_store = StopsStore([{"url": "https://datahub.bbnavi.de/export/rideshare_points.geojson", "vicinity": 50}]) - self.trips_store = TripStore(self.stops_store) - - def test_as_one_time_trip_as_delete_update(self): - cp = Carpool(**data1) - self.trips_store.put_carpool(cp) - trip = next(iter(self.trips_store.trips.values())) - - converter = GtfsRtProducer(self.trips_store) - json = converter._as_delete_updates(trip, datetime(2022,4,11)) - - assert json == [{ - 'trip': { - 'tripId': 'mfdz:Eins', - 'startTime': '23:59:00', - 'startDate': '20220530', - 'scheduleRelationship': 'CANCELED', - 'routeId': 'mfdz:Eins' - } - }] - - def test_as_one_time_trip_as_added_update(self): - cp = Carpool(**data1) - self.trips_store.put_carpool(cp) - trip = next(iter(self.trips_store.trips.values())) - - converter = GtfsRtProducer(self.trips_store) - json = converter._as_added_updates(trip, datetime(2022,4,11)) - assert json == [{ - 'trip': { - 'tripId': 'mfdz:Eins', - 'startTime': '23:59:00', - 'startDate': '20220530', - 'scheduleRelationship': 'ADDED', - 'routeId': 'mfdz:Eins', - '[transit_realtime.trip_descriptor]': { - 'routeUrl' : 'https://mfdz.de/trip/123', - 'agencyId' : 'mfdz', - 'route_long_name' : 'abc nach xyz', - 'route_type': 1551 - } - }, - 'stopTimeUpdate': [{ - 'stopSequence': 1, - 'arrival': { - 'time': time.mktime(datetime(2022,5,30,23,59,0).timetuple()), - 'uncertainty': 600 - }, - 'departure': { - 'time': time.mktime(datetime(2022,5,30,23,59,0).timetuple()), - 'uncertainty': 600 - }, - 'stopId': 'mfdz:12073:001', - 'scheduleRelationship': 'SCHEDULED', - 'stop_time_properties': { - '[transit_realtime.stop_time_properties]': { - 'dropoffType': 'NONE', - 'pickupType': 'COORDINATE_WITH_DRIVER' - } - } - }, - { - 'stopSequence': 2, - 'arrival': { - 'time': time.mktime(datetime(2022,5,31,0,16,45,0).timetuple()), - 'uncertainty': 600 - }, - 'departure': { - 'time': time.mktime(datetime(2022,5,31,0,16,45,0).timetuple()), - 'uncertainty': 600 - }, - - 'stopId': 'de:12073:900340137::3', - 'scheduleRelationship': 'SCHEDULED', - 'stop_time_properties': { - '[transit_realtime.stop_time_properties]': { - 'dropoffType': 'COORDINATE_WITH_DRIVER', - 'pickupType': 'NONE' - } - } - }] - }] - - def test_as_periodic_trip_as_delete_update(self): - cp = Carpool(**carpool_repeating_json) - self.trips_store.put_carpool(cp) - trip = next(iter(self.trips_store.trips.values())) - - converter = GtfsRtProducer(self.trips_store) - json = converter._as_delete_updates(trip, datetime(2022,4,11)) - - assert json == [{ - 'trip': { - 'tripId': 'mfdz:Zwei', - 'startTime': '15:00:00', - 'startDate': '20220411', - 'scheduleRelationship': 'CANCELED', - 'routeId': 'mfdz:Zwei' - } - }, - { - 'trip': { - 'tripId': 'mfdz:Zwei', - 'startTime': '15:00:00', - 'startDate': '20220418', - 'scheduleRelationship': 'CANCELED', - 'routeId': 'mfdz:Zwei' - } - } - ] \ No newline at end of file diff --git a/config b/config deleted file mode 100644 index fadf873..0000000 --- a/config +++ /dev/null @@ -1,5 +0,0 @@ -# Bounding-Box Germany -ride2go_query_data = '{ "southWestCoordinates": { "lat": 47.3, "lon": 5.98 }, "northEastCoordinates": { "lat": 54.99, "lon": 15.02 }, "lastModifiedSinceDays": 180 }' -env = 'PROD' -graphhopper_base_url = 'https://api.mfdz.de/gh' -stop_sources_file = 'conf/stop_sources.json' \ No newline at end of file diff --git a/logging.conf b/logging.conf deleted file mode 100644 index 429da8e..0000000 --- a/logging.conf +++ /dev/null @@ -1,22 +0,0 @@ -[loggers] -keys=root - -[handlers] -keys=consoleHandler - -[formatters] -keys=simpleFormatter - -[logger_root] -level=INFO -handlers=consoleHandler -propagate=yes - -[handler_consoleHandler] -class=StreamHandler -level=DEBUG -formatter=simpleFormatter -args=(sys.stdout,) - -[formatter_simpleFormatter] -format=%(asctime)s - %(name)s - %(levelname)s - %(message)s \ No newline at end of file