Update .gitlab-ci.yml

.gitlab-ci.yml

@@ -69,31 +69,17 @@ test_dag:
     AIRFLOW__CORE__LOAD_EXAMPLES: "False"
     AIRFLOW__CORE__EXECUTOR: "LocalExecutor"
   before_script:
-    # Set up Airflow home and ensure required directories exist
     - mkdir -p $AIRFLOW_HOME/dags
     - mkdir -p $AIRFLOW_HOME/logs
     - mkdir -p $AIRFLOW_HOME/plugins
-
-    # Copy the deployed DAG to Airflow's DAGs folder
     - cp $AIRFLOW_DAGS_DIR/model_dag.py $AIRFLOW_HOME/dags/
+    # Install required dependencies
+    - pip install apache-airflow-providers-apache-spark==2.1.1 pyspark==3.5.0 apache-airflow-providers-openlineage>=1.8.0 pandas mlflow scikit-learn
   script:
-    # Initialize Airflow database
     - airflow db init
-
-    # Start the Airflow scheduler in the background
     - airflow scheduler &
-
-    # Start the Airflow webserver in the background
     - airflow webserver -p 8080 &
-
-    # Wait for the scheduler and webserver to start
     - sleep 20
-
-    # Trigger the DAG and wait for it to complete
     - airflow dags trigger --wait model_training_and_tracking
-
-    # Verify task status
     - airflow tasks state model_training_and_tracking train_and_log_model
-
-    # Check logs for successful execution
     - airflow tasks logs model_training_and_tracking train_and_log_model
\ No newline at end of file