Merge branch 'develop' into 'main'

fix: raw data

See merge request !11

jobs/python/datacleanup.py

@@ -31,16 +31,18 @@ schema = StructType([
     StructField("Churn", StringType(), True)
 ])
 
-raw_data_path = 'dataset/raw_data'  
-if os.path.isdir(raw_data_path):  
-    csv_files = [f for f in os.listdir(raw_data_path) if f.endswith('.csv')]  
-    if csv_files:  
-        first_csv_file = os.path.join(raw_data_path, csv_files[0])  
-        df = spark.read.csv(first_csv_file, header=True, schema=schema)  
-    else:  
+raw_data_path = 'dataset/raw_data'    
+fallback_csv_path = 'dataset/telco_customer_churn.csv'    
+
+if os.path.isdir(raw_data_path):    
+    csv_files = [f for f in os.listdir(raw_data_path) if f.endswith('.csv')]    
+    if csv_files:    
+        first_csv_file = os.path.join(raw_data_path, csv_files[0])    
+        df = spark.read.csv(first_csv_file, header=True, schema=schema)    
+    else:    
         raise FileNotFoundError("No CSV files found in the specified directory.")  
-else:  
-    df = spark.read.csv(raw_data_path, header=True, schema=schema)  
+else:
+    df = spark.read.csv(fallback_csv_path, header=True, schema=schema)  
 
 print("Data dimensions from raw data: ", df.count(), len(df.columns))