Skip to content
Snippets Groups Projects
Commit 25724958 authored by Naufal-Nalendra's avatar Naufal-Nalendra
Browse files

Update .gitlab-ci.yml

parent 3dc3b2c9
No related merge requests found
Pipeline #66478 failed with stages
in 4 minutes and 22 seconds
...@@ -55,31 +55,23 @@ deploy_dag: ...@@ -55,31 +55,23 @@ deploy_dag:
# Test the DAG execution # Test the DAG execution
test_dag: test_dag:
stage: test stage: test
image: image: python:3.11
name: apache/airflow:2.7.1-python3.11
dependencies: dependencies:
- deploy_dag - deploy_dag
services:
- name: postgres:13
alias: postgres
variables:
POSTGRES_USER: airflow
POSTGRES_PASSWORD: airflow
POSTGRES_DB: airflow
AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow
AIRFLOW__CORE__LOAD_EXAMPLES: "False"
AIRFLOW__CORE__EXECUTOR: "LocalExecutor"
before_script:
- mkdir -p $AIRFLOW_HOME/dags
- mkdir -p $AIRFLOW_HOME/logs
- mkdir -p $AIRFLOW_HOME/plugins
- cp $AIRFLOW_DAGS_DIR/model_dag.py $AIRFLOW_HOME/dags/
- pip install apache-airflow-providers-apache-spark==2.1.1 pyspark==3.5.0 apache-airflow-providers-openlineage>=1.8.0 pandas mlflow scikit-learn
script: script:
- airflow db init # Activate virtual environment
- nohup airflow scheduler > /dev/null 2>&1 & - python -m venv venv
- nohup airflow webserver -p 8080 > /dev/null 2>&1 & - source venv/bin/activate
- sleep 20 - pip install --upgrade pip
- airflow dags trigger --wait model_training_and_tracking - pip install apache-airflow[celery,postgres,s3]==2.7.1
- airflow tasks state model_training_and_tracking train_and_log_model
- airflow tasks logs model_training_and_tracking train_and_log_model # Validate Airflow DAGs
\ No newline at end of file - export AIRFLOW_HOME=$AIRFLOW_HOME
- airflow dags list
- airflow dags test model_dag 2025-01-01
artifacts:
when: always
paths:
- $AIRFLOW_DAGS_DIR
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment