Use carpool events for metrics

Dockerfile

@@ -18,18 +18,13 @@ RUN \
 ENV ADMIN_TOKEN=''
 ENV RIDE2GO_TOKEN=''
 ENV SECRET_KEY=''
-ENV METRICS_USER=''
+
-ENV METRICS_PASSWORD=''
+ENV MODULE_NAME=amarillo.main
 
 EXPOSE 80
 
-ARG PACKAGE_REGISTRY_URL
-ARG PLUGINS
-
 COPY requirements.txt /app/requirements.txt
 RUN pip install --no-cache-dir --upgrade -r /app/requirements.txt
-RUN --mount=type=secret,id=AMARILLO_REGISTRY_CREDENTIALS \
-	pip install --no-cache-dir --upgrade --extra-index-url https://$(cat /run/secrets/AMARILLO_REGISTRY_CREDENTIALS)@${PACKAGE_REGISTRY_URL} ${PLUGINS}
 
 COPY ./amarillo /app/amarillo
 COPY ./amarillo/plugins /app/amarillo/plugins

Jenkinsfile

@@ -1,16 +1,17 @@
 pipeline {
-    agent any
+    agent { label 'builtin' }
     environment {
         GITEA_CREDS = credentials('AMARILLO-JENKINS-GITEA-USER')
         TWINE_REPO_URL = "https://git.gerhardt.io/api/packages/amarillo/pypi"
         PLUGINS_REPO_URL = "git.gerhardt.io/api/packages/amarillo/pypi/simple"
-        DOCKER_REGISTRY_URL = 'https://git.gerhardt.io'
+        DOCKER_REGISTRY = 'git.gerhardt.io'
+        DERIVED_DOCKERFILE = 'standard.Dockerfile'
         OWNER = 'amarillo'
+        BASE_IMAGE_NAME = 'amarillo-base'
         IMAGE_NAME = 'amarillo'
-        AMARILLO_DISTRIBUTION = '0.2'
+        AMARILLO_DISTRIBUTION = '0.3'
-        TAG = "${AMARILLO_DISTRIBUTION}.${BUILD_NUMBER}"
+        TAG = "${AMARILLO_DISTRIBUTION}.${BUILD_NUMBER}${env.BRANCH_NAME == 'main' ? '' : '-dev'}"
-        PLUGINS = 'amarillo-metrics amarillo-enhancer amarillo-grfs-export'
+        DEPLOY_WEBHOOK_URL = "${env.BRANCH_NAME == 'main' ? 'http://amarillo.mfdz.de:8888/mitanand' : 'http://amarillo.mfdz.de:8888/dev'}" 
-        DEPLOY_WEBHOOK_URL = 'http://amarillo.mfdz.de:8888/mitanand'
         DEPLOY_SECRET = credentials('AMARILLO-JENKINS-DEPLOY-SECRET')
     }
     stages {
@@ -41,27 +42,58 @@ pipeline {
                 sh 'python3 -m twine upload --skip-existing --verbose --repository-url $TWINE_REPO_URL --username $GITEA_CREDS_USR --password $GITEA_CREDS_PSW ./dist/*'              
             }
         }
-        stage('Build docker image') {
+        stage('Build base docker image') {
             when {
-                branch 'main'
+                anyOf { branch 'main'; branch 'dev' }
             }
             steps {
                 echo 'Building image'
                 script {
-                    docker.build("${OWNER}/${IMAGE_NAME}:${TAG}",
+                    docker.build("${OWNER}/${BASE_IMAGE_NAME}:${TAG}")
-                    //--no-cache to make sure plugins are updated
+                    // "--no-cache --build-arg='PACKAGE_REGISTRY_URL=${PLUGINS_REPO_URL}' --build-arg='PLUGINS=${PLUGINS}' --secret id=AMARILLO_REGISTRY_CREDENTIALS,env=GITEA_CREDS .")
-                     "--no-cache --build-arg='PACKAGE_REGISTRY_URL=${PLUGINS_REPO_URL}' --build-arg='PLUGINS=${PLUGINS}' --secret id=AMARILLO_REGISTRY_CREDENTIALS,env=GITEA_CREDS .")
                 }
             }
         }
-        stage('Push image to container registry') {
+        stage('Push base image to container registry') {
             when {
-                branch 'main'
+                anyOf { branch 'main'; branch 'dev' }
             }
             steps {
                 echo 'Pushing image to registry'
                 script {
-                    docker.withRegistry(DOCKER_REGISTRY_URL, 'AMARILLO-JENKINS-GITEA-USER'){
+                    docker.withRegistry("https://${DOCKER_REGISTRY}", 'AMARILLO-JENKINS-GITEA-USER'){
+                        def image = docker.image("${OWNER}/${BASE_IMAGE_NAME}:${TAG}")
+                        image.push()
+                        image.push('latest')
+                    }
+                }
+            }
+        }
+        stage('Build derived docker image') {
+            when {
+                anyOf { branch 'main'; branch 'dev' }
+            }
+            steps {
+                echo 'Building image'
+                script {
+                    docker.withRegistry("https://${DOCKER_REGISTRY}", 'AMARILLO-JENKINS-GITEA-USER'){
+                        docker.build("${OWNER}/${IMAGE_NAME}:${TAG}",
+                        //--no-cache to make sure plugins are updated
+                        "-f ${DERIVED_DOCKERFILE} --no-cache --build-arg='PACKAGE_REGISTRY_URL=${PLUGINS_REPO_URL}' --build-arg='DOCKER_REGISTRY=${DOCKER_REGISTRY}' --secret id=AMARILLO_REGISTRY_CREDENTIALS,env=GITEA_CREDS .")
+                        // "--no-cache --build-arg='PACKAGE_REGISTRY_URL=${PLUGINS_REPO_URL}' --build-arg='PLUGINS=${PLUGINS}' --secret id=AMARILLO_REGISTRY_CREDENTIALS,env=GITEA_CREDS .")
+                    }
+
+                }
+            }
+        }
+        stage('Push derived image to container registry') {
+            when {
+                anyOf { branch 'main'; branch 'dev' }
+            }
+            steps {
+                echo 'Pushing image to registry'
+                script {
+                    docker.withRegistry("https://${DOCKER_REGISTRY}", 'AMARILLO-JENKINS-GITEA-USER'){
                         def image = docker.image("${OWNER}/${IMAGE_NAME}:${TAG}")
                         image.push()
                         image.push('latest')
@@ -71,7 +103,7 @@ pipeline {
         }
         stage('Notify CD script') {
             when {
-                branch 'main'
+                anyOf { branch 'main'; branch 'dev' }
             }
             steps {
                 echo 'Triggering deploy webhook'

amarillo/routers/carpool.py

@@ -5,7 +5,8 @@ import os.path
 import re
 from glob import glob
 
-from fastapi import APIRouter, Body, HTTPException, status, Depends
+from fastapi import APIRouter, Body, HTTPException, status, Depends, BackgroundTasks
+import requests
 from datetime import datetime
 
 from amarillo.models.Carpool import Carpool
@@ -13,6 +14,8 @@ from amarillo.models.User import User
 from amarillo.services.oauth2 import get_current_user, verify_permission
 from amarillo.tests.sampledata import examples
 
+from amarillo.services.config import config
+from amarillo.utils.utils import assert_folder_exists
 
 logger = logging.getLogger(__name__)
 
@@ -21,6 +24,20 @@ router = APIRouter(
     tags=["carpool"]
 )
 
+#TODO: housekeeping for outdated trips
+
+def enhance_trip(carpool: Carpool):
+    response = requests.post(f"{config.enhancer_url}", carpool.model_dump_json())
+    enhanced_carpool = Carpool(**json.loads(response.content))
+
+    #TODO: use data/enhanced directory
+    folder = f'data/enhanced/{carpool.agency}'
+    filename = f'{folder}/{carpool.id}.json'
+
+    assert_folder_exists(folder)
+    with open(filename, 'w', encoding='utf-8') as f:
+        f.write(enhanced_carpool.model_dump_json())
+
 @router.post("/",
              operation_id="addcarpool",
              summary="Add a new or update existing carpool",
@@ -32,7 +49,7 @@ router = APIRouter(
                      "description": "Agency does not exist"},
                  
                 })
-async def post_carpool(carpool: Carpool = Body(..., examples=examples),
+async def post_carpool(background_tasks: BackgroundTasks, carpool: Carpool = Body(..., examples=examples),
                        requesting_user: User = Depends(get_current_user)) -> Carpool:
     verify_permission(f"{carpool.agency}:write", requesting_user)
 
@@ -41,6 +58,8 @@ async def post_carpool(carpool: Carpool = Body(..., examples=examples),
 
     await store_carpool(carpool)
 
+    background_tasks.add_task(enhance_trip, carpool)
+
     return carpool
     
 # TODO 403
@@ -91,12 +110,10 @@ async def _delete_carpool(agency_id: str, carpool_id: str):
     # load and store, to receive pyinotify events and have file timestamp updated
     await save_carpool(cp, 'data/trash')
     logger.info(f"Saved carpool {agency_id}:{carpool_id} in trash.")
-    os.remove(f"data/carpool/{agency_id}/{carpool_id}.json")
-
     try:
-        from amarillo.plugins.metrics import trips_deleted_counter
+        os.remove(f"data/carpool/{agency_id}/{carpool_id}.json")
-        trips_deleted_counter.inc()
+        os.remove(f"data/enhanced/{agency_id}/{carpool_id}.json", )
-    except ImportError:
+    except FileNotFoundError:
         pass
     
 
@@ -106,17 +123,6 @@ async def store_carpool(carpool: Carpool) -> Carpool:
     await set_lastUpdated_if_unset(carpool)
     await save_carpool(carpool)
 
-    try:
-        from amarillo.plugins.metrics import trips_created_counter, trips_updated_counter
-        if(carpool_exists):
-            # logger.info("Incrementing trips updated")
-            trips_updated_counter.inc()
-        else:
-            # logger.info("Incrementing trips created")
-            trips_created_counter.inc()
-    except ImportError:
-        pass
-
     return carpool
 
 async def set_lastUpdated_if_unset(carpool):

amarillo/services/carpools.py

@@ -37,11 +37,6 @@ class CarpoolService():
             if cp and self.is_outdated(cp):
                 logger.info("Purge outdated offer %s", key)
                 self.delete(cp.agency, cp.id)
-                try:
-                    from amarillo.plugins.metrics import trips_deleted_counter
-                    trips_deleted_counter.inc()
-                except ImportError:
-                    pass
 
     def get(self, agency_id: str, carpool_id: str):
         return self.carpools.get(f"{agency_id}:{carpool_id}")

amarillo/services/config.py

@@ -8,5 +8,6 @@ class Config(BaseSettings):
     env: str = 'DEV'
     graphhopper_base_url: str = 'https://api.mfdz.de/gh'
     stop_sources_file: str = 'data/stop_sources.json'
+    enhancer_url: str = 'http://localhost:8001'
 
 config = Config(_env_file='config', _env_file_encoding='utf-8')

amarillo/services/users.py

@@ -65,6 +65,7 @@ class UserService:
         logger.error(message)
         raise HTTPException(status_code=400, detail=message)
 
+    #TODO: fix duplicate None api key
     def add(self, user_conf: User):
 
         user_id = user_conf.user_id

amarillo/static/logging.conf

@@ -22,7 +22,7 @@ args=(sys.stdout,)
 class=handlers.RotatingFileHandler
 level=ERROR
 formatter=simpleFormatter
-args=('error.log', 'a', 1000000, 3)  # Filename, mode, maxBytes, backupCount
+args=('data/error.log', 'a', 1000000, 3)  # Filename, mode, maxBytes, backupCount
 
 [formatter_simpleFormatter]
 format=%(asctime)s - %(name)s - %(levelname)s - %(message)s

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "amarillo"
-version = "0.0.15a4"
+version = "0.0.16a1"
 description = "Aggregates and enhances carpooling-offers and publishes them as GTFS(-RT)"
 readme = "README.md"
 license = {file = "LICENSE"}

standard.Dockerfile

@@ -0,0 +1,19 @@
+ARG DOCKER_REGISTRY
+FROM ${DOCKER_REGISTRY}/amarillo/amarillo-base
+
+ARG PLUGINS=\
+"amarillo-metrics "\
+"amarillo-gtfs-exporter "
+
+ARG PACKAGE_REGISTRY_URL
+
+ENV METRICS_USER=''
+ENV METRICS_PASSWORD=''
+
+# RUN pip install $PLUGINS
+
+RUN --mount=type=secret,id=AMARILLO_REGISTRY_CREDENTIALS \
+pip install --no-cache-dir --upgrade --extra-index-url https://$(cat /run/secrets/AMARILLO_REGISTRY_CREDENTIALS)@${PACKAGE_REGISTRY_URL} ${PLUGINS}
+
+RUN useradd amarillo
+USER amarillo