savestate before github

.gitignore

@@ -41,4 +41,8 @@ server.crt
 AssetWarranty.csv
 FSL_Asset_Maintenance_Information__c.csv
 MaintenancePlan.csv
-MaintenancePlan_beforetransform.csv
+MaintenancePlan_beforetransform.csv
+*_beforetransform.csv
+MaintenanceAsset.csv
+ContractLineItem.csv
+data_for_external_systems/

prepared_steps/19_create_contractlineitems/command.txt

@@ -0,0 +1 @@
+sf sfdmu run --sourceusername csvfile --targetusername rene.kasseboehmer@vaillant.de.devrene

prepared_steps/19_create_contractlineitems/export.json

@@ -0,0 +1,23 @@
+{
+    "allOrNone": true,
+    "excludeIdsFromCSVFiles": true,
+    "objects": [
+        {
+            "query": "SELECT Id, PKey__c FROM ServiceContract WHERE TemplateId__c != null AND Pricebook2.Country__c  = 'NL'",
+            "operation": "Readonly",
+            "externalId": "PKey__c",
+            "master": false
+        },{
+            "query": "SELECT Id, PKey__c FROM Asset WHERE Location.ParentLocation.VisitorAddress.CountryCode = 'NL'",
+            "operation": "Readonly",
+            "externalId": "PKey__c",
+            "master": false
+        },{
+            "query": "SELECT Id, ServiceContractId, AssetId FROM ContractLineItem",
+            "operation": "Insert",
+            "externalId": "Id",
+            "master": true,
+            "useSourceCSVFile": true
+        }
+    ]
+}

prepared_steps/19_create_contractlineitems/fill_fields_and_insert.py

@@ -48,8 +48,6 @@ if __name__ == '__main__':
     df_lm = pd.read_csv('../1_extract_data/results/LastMaintenanceInformation.csv', usecols=['Id', 'Order__r.Closed__c', 'Order__r.Contract__c'])
     df_lo = pd.read_csv('../1_extract_data/results/LastOrderInformation.csv', usecols=['Id', 'Closed__c', 'Contract__c'])
 
-    print(df_sc)
-    print(df_mp)
     #
     # Merge df_mp with df_sc including Id based on PKey__c
     merged_df_mp = pd.merge(df_mp, 
@@ -58,8 +56,6 @@ if __name__ == '__main__':
                             right_on='PKey__c',
                             how='left')
 
-    print(merged_df_mp)
-
     # Rename only 'old_name' to 'new_name'
     merged_df_mp = merged_df_mp.rename(columns={'sf__Id': 'ServiceContractId'})
 
@@ -92,7 +88,22 @@ if __name__ == '__main__':
 
     merged_df_mp['NextSuggestedMaintenanceDate'] = merged_df_mp['ServiceContractId'].apply(get_next_suggested_maintenance_date)
 
-    merged_df_mp['DoesAutoGenerateWorkOrders'] = True
+    # Set DoesAutoGenerateWorkOrders to True only if a date was found (not defaulted to today)
+    def does_auto_generate_work_orders(service_contract_id):
+        # Try to find in LastMaintenanceInformation
+        lm_rows = df_lm[df_lm['Order__r.Contract__c'] == service_contract_id]
+        if not lm_rows.empty:
+            last_date = lm_rows['Order__r.Closed__c'].max()
+        else:
+            # If not found, try LastOrderInformation
+            lo_rows = df_lo[df_lo['Contract__c'] == service_contract_id]
+            if not lo_rows.empty:
+                last_date = lo_rows['Closed__c'].max()
+            else:
+                last_date = None
+        return pd.notnull(last_date)
+
+    merged_df_mp['DoesAutoGenerateWorkOrders'] = merged_df_mp['ServiceContractId'].apply(does_auto_generate_work_orders)
 
     #transform values into int
     merged_df_mp['MaintenanceWindowEndDays'] = pd.to_numeric(merged_df_mp['MaintenanceWindowEndDays'], errors='coerce').fillna(0).astype(int)

prepared_steps/1_extract_data/extract_via_simple_salesforce.py

@@ -4,6 +4,8 @@
 #   --output_path extracted_data
 
 import os
+import re
+import ast
 import itertools
 import threading
 import time
@@ -13,6 +15,8 @@ import pandas as pd
 from sys import path
 path.append('../..')
 from sf_auth import get_sf_connection
+from collections import OrderedDict
+
 
 done = False
 
@@ -123,7 +127,7 @@ if __name__ == '__main__':
         sobject = query_def['sobject']
         query_str = query_def['query'].replace('{country}', args.country)
         use_rest = query_def.get('useREST', False)  # Default to False if not specified
-        
+
         if not selected_sobjects or sobject in selected_sobjects:
             try:
                 output_file = extract_data(
@@ -134,6 +138,7 @@ if __name__ == '__main__':
                     use_rest=use_rest
                 )
                 print(f'Successfully extracted {sobject} to {output_file}')
+
             except Exception as e:
                 print(f'Error extracting {sobject}: {str(e)}')
 

prepared_steps/1_extract_data/queries.json

@@ -34,14 +34,14 @@
         },{
             "sobject": "SCResourceAssignment__c",
             "useREST": true,
-            "query": "SELECT Id, ValidTo__c, ValidFrom__c, Country__c, City__c, PostalCode__c, District__c, Street__c, HouseNo__c, Extension__c, FlatNo__c, Floor__c, GeoY__c, GeoX__c, Resource__c, Resource__r.Employee__r.Name, Stock__c, Stock__r.ID2__c, QualificationProfile__c FROM SCResourceAssignment__c WHERE Country__c = '{country}' AND ValidTo__c >= TODAY"
+            "query": "SELECT Id, ValidTo__c, ValidFrom__c, Country__c, City__c, PostalCode__c, District__c, Street__c, HouseNo__c, Extension__c, FlatNo__c, Floor__c, GeoY__c, GeoX__c, Resource__c, Resource__r.Employee__r.Name, ResourceNumberCalc__c, Stock__c, Stock__r.ID2__c, QualificationProfile__c FROM SCResourceAssignment__c WHERE Country__c = '{country}' AND ValidTo__c >= TODAY"
         },{
             "sobject": "ContractTemplates",
             "useREST": true,
             "query": "SELECT id, name, TemplateName__c, status__c, Brand__r.Name, Country__c, Runtime__c FROM SCContract__c WHERE Template__c = null AND Country__c = '{country}'"
         },{
             "sobject": "SCContract__c",
-            "query": "SELECT id, name, Template__c, status__c, Brand__r.Name, Country__c, Runtime__c, EndDate__c, StartDate__c, Account__c, AccountOwner__c, IoT_Registration_Status__c, Maintenance_price_inclusion_of_VAT__c, MaintenanceAfterDue__c, MaintenanceBeforeDue__c, MaintenanceCreationLeadtime__c, MaintenanceCreationShiftFactor__c, MaintenanceDuration__c, MaintenanceFirstDate__c, MaintenanceLastDate__c, MaintenanceInterval__c, MaintenancePriceRequired__c, util_MaintenancePrice__c, util_MaintenanceDuration__c, util_MaintenanceNextDate__c FROM SCContract__c WHERE Template__c != null AND EndDate__c >= TODAY AND Country__c = '{country}' limit 3"
+            "query": "SELECT id, name, Template__c, status__c, Brand__r.Name, Country__c, Runtime__c, EndDate__c, StartDate__c, Account__c, AccountOwner__c, IoT_Registration_Status__c, Template__r.SAPContractCategory__c, Maintenance_price_inclusion_of_VAT__c, MaintenanceAfterDue__c, MaintenanceBeforeDue__c, MaintenanceCreationLeadtime__c, MaintenanceCreationShiftFactor__c, MaintenanceDuration__c, MaintenanceFirstDate__c, MaintenanceLastDate__c, MaintenanceInterval__c, MaintenancePriceRequired__c, util_MaintenancePrice__c, util_MaintenanceDuration__c, util_MaintenanceNextDate__c FROM SCContract__c WHERE Template__c != null AND EndDate__c >= TODAY AND Country__c = '{country}' limit 3"
         },{
             "sobject": "WarrantyTerm",
             "useREST": true,
@@ -68,6 +68,18 @@
             "sobject": "WorkType",
             "useREST": true,
             "query": "SELECT Id, Name from WorkType"
+        },{
+            "sobject": "SCContractItem__c",
+            "query": "Select Id, Name, InstalledBase__c, Contract__c, MaintenancePrice__c FROM SCContractItem__c WHERE Contract__r.Country__c = '{country}' AND Contract__r.Template__c != null AND Contract__r.EndDate__c >= TODAY "
+        },{
+            "sobject": "PricebookEntry",
+            "query": "SELECT Id, Product2.Product_Code__c FROM PricebookEntry WHERE Pricebook2.Country__c = '{country}'"
+        },{
+            "sobject": "SCStockItem__c",
+            "query": "SELECT Id, PlantERP__c, StockERP__c, Stock__r.Name__c, ArticleNo__c, ArticleNameCalc__c, Qty__c, ERPQty__c, DifferenceQty__c, MinQty__c, MaxQty__c, LastModifiedDate, Stock__r.RunReplenishment__c, Stock__c, Stock__r.StockERP__c, Article__r.EANCode__c, Article__r.Name, Article__c FROM SCStockItem__c WHERE Stock__r.Country__c = '{country}'"
+        },{
+            "sobject": "SCStockOptimizationListItem__c",
+            "query": "SELECT Id, Article_Name__c, Article__r.Name, StockOptimizationList__r.Name, MaxQty__c, LastModifiedDate  FROM SCStockOptimizationListItem__c WHERE StockOptimizationList__r.Plant__r.Country__c = '{country}'"
         }
     ]
 }

prepared_steps/20_create_maintenance_plan/fill_fields_and_insert.py

@@ -0,0 +1,119 @@
+import pandas as pd
+import argparse
+from sys import path
+path.append('../..')
+from utils import bulk_insert_records
+from datetime import datetime, timedelta
+
+# Find last maintenance/order date for each ServiceContractId
+def get_last_maintenance_date(service_contract_id):
+    # Try to find in LastMaintenanceInformation
+    lm_rows = df_lm[df_lm['Order__r.Contract__c'] == service_contract_id]
+    if not lm_rows.empty:
+        # Use the most recent Order__r.Closed__c
+        return lm_rows['Order__r.Closed__c'].max()
+    # If not found, try LastOrderInformation
+    lo_rows = df_lo[df_lo['Contract__c'] == service_contract_id]
+    if not lo_rows.empty:
+        return lo_rows['Closed__c'].max()
+    return None
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Insert MaintenancePlan records via Bulk API')
+    parser.add_argument('--context', type=str, required=True,
+                       help='Salesforce org context (e.g., "qa2", "prod")')
+    parser.add_argument('--csv', type=str, default='MaintenancePlan.csv',
+                       help='CSV file to process (default: MaintenancePlan.csv)')
+    args = parser.parse_args()
+
+    #read_df_sc = pd.read_csv('../16_insert_servicecontract/successful_records.csv', header=0, keep_default_na=False, dtype=str)
+    read_df_mp = pd.read_csv('./MaintenancePlan_beforetransform.csv', header=0, keep_default_na=False, dtype=str)
+
+    #"sf__Id","sf__Created",PKey__c,BillingCountryCode,Term,EndDate,StartDate,AccountId,Service_Recipient__c,IoT_Registration_Status__c,Name,Pricebook2Id,TemplateId__c
+    #reindex_columns_sc = ['sf__Id','sf__Created','PKey__c', 'BillingCountryCode', 'Term', 'EndDate', 'StartDate', 'AccountId', 'Service_Recipient__c', 'IoT_Registration_Status__c', 'Name', 'Pricebook2Id', 'TemplateId__c']
+    #ServiceContract.PKey__c,StartDate,EndDate,AccountId,MaintenanceWindowEndDays,MaintenanceWindowStartDays,GenerationTimeframe,Frequency,GenerationTimeframeType,WorkTypeId
+    reindex_columns_mp = ['ServiceContract.PKey__c', 'StartDate', 'EndDate', 'AccountId', 'MaintenanceWindowEndDays', 'MaintenanceWindowStartDays', 'GenerationTimeframe', 'Frequency', 'GenerationTimeframeType', 'WorkTypeId']
+    reindex_columns_lastmaintenance = []
+
+
+    # Reindex the columns to match the desired format
+    #df_sc = read_df_sc.reindex(reindex_columns_sc, axis=1)
+    df_mp = read_df_mp.reindex(reindex_columns_mp, axis=1)
+
+
+    # Only load 'sf__Id' and 'PKey__c' from the ServiceContract CSV
+    df_sc = pd.read_csv('../16_insert_servicecontract/successful_records.csv', usecols=['sf__Id', 'PKey__c'])
+    # Load LastMaintenanceInformation and LastOrderInformation CSVs for NextSuggestedMaintenanceDate calculation
+    df_lm = pd.read_csv('../1_extract_data/results/LastMaintenanceInformation.csv', usecols=['Id', 'Order__r.Closed__c', 'Order__r.Contract__c'])
+    df_lo = pd.read_csv('../1_extract_data/results/LastOrderInformation.csv', usecols=['Id', 'Closed__c', 'Contract__c'])
+
+    #
+    # Merge df_mp with df_sc including Id based on PKey__c
+    merged_df_mp = pd.merge(df_mp, 
+                            df_sc[['sf__Id', 'PKey__c']], 
+                            left_on='ServiceContract.PKey__c',
+                            right_on='PKey__c',
+                            how='left')
+
+    # Rename only 'old_name' to 'new_name'
+    merged_df_mp = merged_df_mp.rename(columns={'sf__Id': 'ServiceContractId'})
+
+    merged_df_mp = merged_df_mp.drop('ServiceContract.PKey__c', axis=1)
+    merged_df_mp = merged_df_mp.drop('PKey__c', axis=1)
+
+    def get_next_suggested_maintenance_date(service_contract_id):
+        # Try to find in LastMaintenanceInformation
+        lm_rows = df_lm[df_lm['Order__r.Contract__c'] == service_contract_id]
+        if not lm_rows.empty:
+            last_date = lm_rows['Order__r.Closed__c'].max()
+        else:
+            # If not found, try LastOrderInformation
+            lo_rows = df_lo[df_lo['Contract__c'] == service_contract_id]
+            if not lo_rows.empty:
+                last_date = lo_rows['Closed__c'].max()
+            else:
+                last_date = None
+
+        if pd.notnull(last_date):
+            try:
+                dt = pd.to_datetime(last_date)
+                next_date = dt + pd.DateOffset(years=1)
+                return next_date.strftime('%Y-%m-%d')
+            except Exception:
+                pass
+        # If no date found or parsing fails, use today
+        next_date = datetime.today()
+        return next_date.strftime('%Y-%m-%d')
+
+    merged_df_mp['NextSuggestedMaintenanceDate'] = merged_df_mp['ServiceContractId'].apply(get_next_suggested_maintenance_date)
+
+    # Set DoesAutoGenerateWorkOrders to True only if a date was found (not defaulted to today)
+    def does_auto_generate_work_orders(service_contract_id):
+        # Try to find in LastMaintenanceInformation
+        lm_rows = df_lm[df_lm['Order__r.Contract__c'] == service_contract_id]
+        if not lm_rows.empty:
+            last_date = lm_rows['Order__r.Closed__c'].max()
+        else:
+            # If not found, try LastOrderInformation
+            lo_rows = df_lo[df_lo['Contract__c'] == service_contract_id]
+            if not lo_rows.empty:
+                last_date = lo_rows['Closed__c'].max()
+            else:
+                last_date = None
+        return pd.notnull(last_date)
+
+    merged_df_mp['DoesAutoGenerateWorkOrders'] = merged_df_mp['ServiceContractId'].apply(does_auto_generate_work_orders)
+
+    #transform values into int
+    merged_df_mp['MaintenanceWindowEndDays'] = pd.to_numeric(merged_df_mp['MaintenanceWindowEndDays'], errors='coerce').fillna(0).astype(int)
+    merged_df_mp['MaintenanceWindowStartDays'] = pd.to_numeric(merged_df_mp['MaintenanceWindowStartDays'], errors='coerce').fillna(0).astype(int)
+    merged_df_mp['GenerationTimeframe'] = pd.to_numeric(merged_df_mp['GenerationTimeframe'], errors='coerce').fillna(0).astype(int)
+    merged_df_mp['Frequency'] = pd.to_numeric(merged_df_mp['Frequency'], errors='coerce').fillna(0).astype(int)
+
+    #safe csv
+    merged_df_mp.to_csv('./MaintenancePlan.csv', index=False)
+
+
+    bulk_insert_records(args.context, 'MaintenancePlan', args.csv)
+

prepared_steps/21_insert_maintenance_asset/command.txt

@@ -0,0 +1 @@
+python link_ids_and_create_maintenanceassets.py --context qa2 --csv MaintenanceAsset.csv

prepared_steps/21_insert_maintenance_asset/link_ids_and_create_maintenanceassets.py

@@ -0,0 +1,81 @@
+import pandas as pd
+import os
+from sys import path
+import argparse
+path.append('../..')
+from utils import bulk_insert_records
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Insert MaintenancePlan records via Bulk API')
+    parser.add_argument('--context', type=str, required=True,
+                       help='Salesforce org context (e.g., "qa2", "prod")')
+    parser.add_argument('--csv', type=str, default='MaintenancePlan.csv',
+                       help='CSV file to process (default: MaintenanceAsset.csv)')
+    args = parser.parse_args()
+
+    df_s_servicecontracts = pd.read_csv("../16_insert_servicecontract/successful_records.csv")
+    df_s_maintenanceplans = pd.read_csv("../20_create_maintenance_plan/successful_records.csv")
+    df_maintenanceasset_before = pd.read_csv("MaintenanceAsset_beforetransform.csv")
+
+
+    asset_update_path = os.path.join("..", "9_upsert_assets", "target", "Asset_update_target.csv")
+    if os.path.exists(asset_update_path):
+        df_asset_update = pd.read_csv(asset_update_path)
+    else:
+        df_asset_update = None
+
+    asset_insert_path = os.path.join("..", "9_upsert_assets", "target", "Asset_insert_target.csv")
+    if os.path.exists(asset_insert_path):
+        df_asset_insert = pd.read_csv(asset_insert_path)
+    else:
+        df_asset_insert = None
+
+    # Merge the two DataFrames on PKey__c from df_s_servicecontracts and Contract__c from df_s_maintenanceplans
+    merged_df = pd.merge(
+        df_s_servicecontracts[['sf__Id', 'PKey__c']],
+        df_s_maintenanceplans[['sf__Id', 'ServiceContractId']],
+        left_on='sf__Id',
+        right_on='ServiceContractId',
+        how='inner',
+        suffixes=('_servicecontract', '_maintenanceplan')
+    )
+
+    # Create a new DataFrame with only the desired columns
+    result_df = merged_df[['sf__Id_maintenanceplan', 'sf__Id_servicecontract', 'PKey__c']]
+
+    # Merge the sf__Id_maintenanceplan and sf__Id_servicecontract into df_maintenanceasset_before based on PKey__c
+    df_maintenanceasset_before = pd.merge(
+        df_maintenanceasset_before,
+        result_df,
+        left_on='ServiceContract.PKey__c',
+        right_on='PKey__c',
+        how='left'
+    )
+
+    # Find Asset Ids from df_asset_update and df_asset_insert using PKey__c and InstalledBase__c
+    def find_asset_id(row):
+        asset_id = None
+        if df_asset_update is not None:
+            match = df_asset_update[df_asset_update['PKey__c'] == row['InstalledBase__c']]
+            if not match.empty:
+                asset_id = match.iloc[0]['Id']
+        if asset_id is None and df_asset_insert is not None:
+            match = df_asset_insert[df_asset_insert['PKey__c'] == row['InstalledBase__c']]
+            if not match.empty:
+                asset_id = match.iloc[0]['Id']
+        return asset_id
+
+    df_maintenanceasset_before['AssetId'] = df_maintenanceasset_before.apply(find_asset_id, axis=1)
+
+    df_maintenanceasset_before.drop(columns=['PKey__c'], inplace=True)
+    df_maintenanceasset_before.drop(columns=['InstalledBase__c'], inplace=True)
+    df_maintenanceasset_before.drop(columns=['ServiceContract.PKey__c'], inplace=True)
+
+    df_maintenanceasset_before.rename(columns={'sf__Id_maintenanceplan': 'MaintenancePlanId', 'sf__Id_servicecontract': 'ServiceContractId'}, inplace=True)
+
+    #safe csv
+    df_maintenanceasset_before.to_csv('./MaintenanceAsset.csv', index=False)
+
+    #insert records
+    bulk_insert_records(args.context, 'MaintenanceAsset', args.csv)

prepared_steps/2_transform_via_script/TransformScript.py

@@ -1,5 +1,7 @@
 import pandas as pd
 from tqdm import tqdm
+import os
+import shutil
 
 country_mapping = {
     'NL': 'Netherlands',
@@ -45,6 +47,9 @@ read_df_qualificationprofile = pd.read_csv('../1_extract_data/results/SCQualific
 read_df_qualificationprofileitem = pd.read_csv('../1_extract_data/results/SCQualificationProfileItem__c.csv', header=0, keep_default_na=False, dtype=str)
 read_df_lastmaintenance = pd.read_csv('../1_extract_data/results/LastMaintenanceInformation.csv', header=0, keep_default_na=False, dtype=str)
 read_df_worktype = pd.read_csv('../1_extract_data/results/WorkType.csv', header=0, keep_default_na=False, dtype=str)
+read_df_contractitem = pd.read_csv('../1_extract_data/results/SCContractItem__c.csv', header=0, keep_default_na=False, dtype=str)
+df_pricebookentry = pd.read_csv('../1_extract_data/results/PricebookEntry.csv', header=0, keep_default_na=False, dtype=str, usecols=['Id', 'Product2.Product_Code__c'])
+read_df_stockitem = pd.read_csv('../1_extract_data/results/SCStockItem__c.csv', header=0, keep_default_na=False, dtype=str, usecols=['Id', 'PlantERP__c', 'StockERP__c', 'Stock__r.Name__c', 'Stock__r.RunReplenishment__c', 'Stock__r.StockERP__c', 'ArticleNo__c', 'ArticleNameCalc__c', 'Qty__c', 'ERPQty__c', 'DifferenceQty__c', 'MinQty__c', 'MaxQty__c', 'LastModifiedDate', 'Stock__c', 'Article__r.EANCode__c', 'Article__r.Name', 'Article__c'])
 
 # Columns for reindexing
 reindex_columns = ['Id','City__c','Country__c','GeoY__c','GeoX__c','PostalCode__c','Street__c','Extension__c','HouseNo__c','FlatNo__c','Floor__c']
@@ -53,16 +58,17 @@ reindex_columns_product2 = ['Id','Main_Product_Group__c','Family','MaterialType_
 reindex_columns_ibr = ['Id', 'InstalledBaseLocation__c', 'Role__c', 'ValidFrom__c', 'ValidTo__c', 'Account__c']
 reindex_columns_pricelist = ['Id', 'Name', 'Brand__r.Name', 'Country__c']
 reindex_columns_pricelistitem = ['Id', 'Article__r.Name', 'Article__r.EANCode__c', 'Price__c', 'PriceUnit__c', 'Pricelist__c', 'ValidFrom__c', 'ValidTo__c', 'Pricelist__r.Brand__r.Name', 'Pricelist__r.Country__c']
-reindex_columns_resourceassignment = ['Id', 'ValidTo__c', 'ValidFrom__c', 'Country__c', 'City__c', 'PostalCode__c', 'District__c', 'Street__c', 'HouseNo__c', 'Extension__c', 'FlatNo__c', 'Floor__c', 'GeoY__c', 'GeoX__c', 'Resource__c', 'Resource__r.Employee__r.Name', 'Stock__c', 'Stock__r.ID2__c']
+reindex_columns_resourceassignment = ['Id', 'ValidTo__c', 'ValidFrom__c', 'Country__c', 'City__c', 'PostalCode__c', 'District__c', 'Street__c', 'HouseNo__c', 'Extension__c', 'FlatNo__c', 'Floor__c', 'GeoY__c', 'GeoX__c', 'Resource__c', 'Resource__r.Employee__r.Name', 'ResourceNumberCalc__c', 'Stock__c', 'Stock__r.ID2__c']
 reindex_columns_address_iot = ['Id', 'Country', 'CountryCode', 'Street', 'City', 'ParentId', 'PostalCode']
 reindex_columns_location_iot = ['Id', 'Name']
 reindex_columns_servicecontracttemplates = ['Id', 'Name', 'TemplateName__c', 'Status__c', 'Brand__r.Name', 'Country__c', 'Runtime__c']  
-reindex_columns_servicecontracts = ['Id', 'Name', 'Template__c', 'Status__c', 'Brand__r.Name', 'Country__c', 'Runtime__c', 'EndDate__c', 'StartDate__c', 'Account__c', 'AccountOwner__c', 'IoT_Registration_Status__c', 'Maintenance_price_inclusion_of_VAT__c', 'MaintenanceAfterDue__c', 'MaintenanceBeforeDue__c', 'MaintenanceCreationLeadtime__c', 'MaintenanceCreationShiftFactor__c', 'MaintenanceDuration__c', 'MaintenanceFirstDate__c', 'MaintenanceLastDate__c', 'MaintenanceInterval__c', 'MaintenancePriceRequired__c', 'util_MaintenancePrice__c', 'util_MaintenanceDuration__c', 'util_MaintenanceNextDate__c']
+reindex_columns_servicecontracts = ['Id', 'Name', 'Template__c', 'Status__c', 'Brand__r.Name', 'Country__c', 'Runtime__c', 'EndDate__c', 'StartDate__c', 'Account__c', 'AccountOwner__c', 'IoT_Registration_Status__c', 'Template__r.SAPContractCategory__c', 'Maintenance_price_inclusion_of_VAT__c', 'MaintenanceAfterDue__c', 'MaintenanceBeforeDue__c', 'MaintenanceCreationLeadtime__c', 'MaintenanceCreationShiftFactor__c', 'MaintenanceDuration__c', 'MaintenanceFirstDate__c', 'MaintenanceLastDate__c', 'MaintenanceInterval__c', 'MaintenancePriceRequired__c', 'util_MaintenancePrice__c', 'util_MaintenanceDuration__c', 'util_MaintenanceNextDate__c']
 reindex_columns_warrantyterm = ['Id', 'WarrantyTermName', 'WarrantyDuration', 'WarrantyType', 'Pricebook2']
 reindex_columns_qualificationprofile = ['Id', 'Name', 'Description__c', 'ID2__c', 'Standard__c']
 reindex_columns_qualificationprofileitem = ['CertificationId__c', 'Id', 'Info__c', 'Level__c', 'ValidFrom__c', 'ValidTo__c', 'Name', 'Qualification__c']
 reindex_columns_lastmaintenance = ['Id', 'InstalledBase__r.SerialNo__c', 'Order__r.Closed__c']
 reindex_columns_worktype = ['Id', 'Name']
+reindex_columns_contractitem = ['Id', 'Name', 'InstalledBase__c', 'Contract__c']
 
 # Reindex the columns to match the desired format
 df = read_df.reindex(reindex_columns, axis=1)
@@ -81,6 +87,7 @@ df_qualificationprofile = read_df_qualificationprofile.reindex(reindex_columns_q
 df_qualificationprofileitem = read_df_qualificationprofileitem.reindex(reindex_columns_qualificationprofileitem, axis=1)
 df_lastmaintenance = read_df_lastmaintenance.reindex(reindex_columns_lastmaintenance, axis=1)
 df_worktype = read_df_worktype.reindex(reindex_columns_worktype, axis=1)
+df_contractitem = read_df_contractitem.reindex(reindex_columns_contractitem, axis=1)
 
 ##--------------------------------------------------------------------------##
 ## Update for IoT Addresses and Locations
@@ -563,14 +570,16 @@ df_servicecontracttemplates['Term'] = df_servicecontracttemplates['Term'].fillna
 ## Service Contract
 ##--------------------------------------------------------------------------##
 
-# Create new dataframe for maintenance plan fields
+# Create new dataframe for maintenance plan fields only if Template__r.SAPContractCategory__c is 'BM10', 'FC10', 'MP20', 'WX40'
-df_maintenanceplan = df_servicecontract[['Id', 'StartDate__c', 'EndDate__c', 'Account__c', 'Maintenance_price_inclusion_of_VAT__c', 'MaintenanceAfterDue__c', 
+df_maintenanceplan = df_servicecontract[
-                                    'MaintenanceBeforeDue__c', 'MaintenanceCreationLeadtime__c', 
+    df_servicecontract['Template__r.SAPContractCategory__c'].isin(['BM10', 'FC10', 'MP20', 'WX40'])
-                                    'MaintenanceCreationShiftFactor__c', 'MaintenanceDuration__c', 
+][['Id', 'StartDate__c', 'EndDate__c', 'Account__c', 'Maintenance_price_inclusion_of_VAT__c', 'MaintenanceAfterDue__c', 
-                                    'MaintenanceFirstDate__c', 'MaintenanceLastDate__c', 
+   'MaintenanceBeforeDue__c', 'MaintenanceCreationLeadtime__c', 
-                                    'MaintenanceInterval__c', 'MaintenancePriceRequired__c', 
+   'MaintenanceCreationShiftFactor__c', 'MaintenanceDuration__c', 
-                                    'util_MaintenancePrice__c', 'util_MaintenanceDuration__c', 
+   'MaintenanceFirstDate__c', 'MaintenanceLastDate__c', 
-                                    'util_MaintenanceNextDate__c']].copy()
+   'MaintenanceInterval__c', 'MaintenancePriceRequired__c', 
+   'util_MaintenancePrice__c', 'util_MaintenanceDuration__c', 
+   'util_MaintenanceNextDate__c']].copy()
 
 # Drop maintenance fields from service contract dataframe, except Id
 maintenance_fields = ['Maintenance_price_inclusion_of_VAT__c', 'MaintenanceAfterDue__c', 
@@ -591,7 +600,17 @@ df_servicecontract['Pricebook2.Name'] = (
 
 df_servicecontract = df_servicecontract.drop('Brand__r.Name', axis=1)
 
-df_servicecontract.columns = ['PKey__c', 'Name', 'TemplateId__r.PKey__c', 'Status', 'BillingCountryCode', 'Term', 'EndDate', 'StartDate', 'AccountId', 'Service_Recipient__c', 'IoT_Registration_Status__c', 'Pricebook2.Name']
+# Map Template__r.SAPContractCategory__c values to new names
+contract_category_mapping = {
+    "BM10": "Basic Maintenance Contract",
+    "FC10": "Basic Maintenance Contract",
+    "MP20": "Basic Plus Maintenance Contract",
+    "WX40": "Warranty Extension Contract",
+    "RD30": "IoT"
+}
+df_servicecontract["Template__r.SAPContractCategory__c"] = df_servicecontract["Template__r.SAPContractCategory__c"].replace(contract_category_mapping)
+
+df_servicecontract.columns = ['PKey__c', 'Name', 'TemplateId__r.PKey__c', 'Status', 'BillingCountryCode', 'Term', 'EndDate', 'StartDate', 'AccountId', 'Service_Recipient__c', 'IoT_Registration_Status__c', 'FSL_Type_of_Contract__c', 'Pricebook2.Name']
 
 df_servicecontract['IoT_Registration_Status__c'] = df_servicecontract['IoT_Registration_Status__c'].replace('', 'Open')
 #df_servicecontract['Name'] = df_servicecontract['PKey__c']
@@ -603,10 +622,42 @@ df_servicecontract['TemplateCountry__c'] = df_servicecontract['BillingCountryCod
 df_servicecontract['Term'] = pd.to_numeric(df_servicecontract['Term'].str.extract('(\d+)')[0], errors='coerce')
 df_servicecontract['Term'] = df_servicecontract['Term'].fillna(0).astype(int)
 
+##--------------------------------------------------------------------------##
+## ContractLineItem
+##--------------------------------------------------------------------------##
+
+df_contractlineitem = df_contractitem.copy()
+
+df_contractlineitem.columns = ['Id', 'Name', 'Asset.PKey__c', 'ServiceContract.PKey__c']
+
+df_contractlineitem.drop(['Id'], axis=1, inplace=True)
+df_contractlineitem.drop(['Name'], axis=1, inplace=True)
+
+# Map Product2.Product_Code__c to df_contractlineitem based on Asset.PKey__c
+df_contractlineitem = pd.merge(
+    df_contractlineitem,
+    merged_df_ib[['PKey__c', 'Product2.Product_Code__c']],
+    left_on='Asset.PKey__c',
+    right_on='PKey__c',
+    how='left'
+)
+df_contractlineitem = df_contractlineitem.drop('PKey__c', axis=1)
+
+# Merge df_contractlineitem with df_pricebookentry to get the PricebookEntry Id based on Product2.Product_Code__c
+df_contractlineitem = pd.merge(
+    df_contractlineitem,
+    df_pricebookentry[['Product2.Product_Code__c', 'Id']],
+    left_on='Product2.Product_Code__c',
+    right_on='Product2.Product_Code__c',
+    how='left'
+)
+df_contractlineitem = df_contractlineitem.rename(columns={'Id': 'PricebookEntryId'})
+
+print(df_contractlineitem)
+
 ##--------------------------------------------------------------------------##
 ## MaintenancePlan, MaintenanceAsset
 ##--------------------------------------------------------------------------##
-print(df_maintenanceplan.columns)
 
 df_maintenanceplan.columns = ['ServiceContract.PKey__c', 'StartDate', 'EndDate', 'AccountId', 'MaintenancePriceInclusionOfVAT__c',
                               'MaintenanceWindowEndDays', 'MaintenanceWindowStartDays', 'GenerationTimeframe',
@@ -631,21 +682,77 @@ maintenance_worktype_id = df_worktype[df_worktype['Name'] == 'Planned Maintenanc
 # Add WorkTypeId to maintenance plan
 df_maintenanceplan['WorkTypeId'] = maintenance_worktype_id
 
-print(df_maintenanceplan)
+# Map InstalledBase__c from df_contractitem to df_maintenanceasset based on ServiceContract.PKey__c (Contract__c)
+# If there are multiple InstalledBase__c for a Contract__c, join them as a semicolon-separated string
+
+# Group by Contract__c and aggregate InstalledBase__c as a semicolon-separated string
+contractitem_grouped = df_contractitem.groupby('Contract__c')['InstalledBase__c'].apply(lambda x: ';'.join(x.dropna().unique())).reset_index()
+
+# Merge with df_maintenanceplan
+df_maintenanceasset = df_maintenanceplan[['ServiceContract.PKey__c']].copy()
+df_maintenanceasset = df_maintenanceasset.merge(
+    contractitem_grouped,
+    left_on='ServiceContract.PKey__c',
+    right_on='Contract__c',
+    how='left'
+)
+
+df_maintenanceasset = df_maintenanceasset.rename(columns={'InstalledBase__c': 'InstalledBase__c'})
+df_maintenanceasset = df_maintenanceasset[['ServiceContract.PKey__c', 'InstalledBase__c']]
+
+
+
 
 
 ##--------------------------------------------------------------------------##
 ## Skills
 ##--------------------------------------------------------------------------##
 
+##--------------------------------------------------------------------------##
+## Stocks
+##--------------------------------------------------------------------------##
 
+# Merge Resource__r.Employee__r.Name from df_resourceassignment to read_df_stockitem via Stock__c
+read_df_stockitem = pd.merge(
+    read_df_stockitem,
+    df_resourceassignment[['Stock__c', 'ResourceNumberCalc__c']],
+    left_on='Stock__c',
+    right_on='Stock__c',
+    how='left'
+)
 
+##--------------------------------------------------------------------------##
+## StocksOptimizationListItem
+##--------------------------------------------------------------------------##
+
+src_file = '../1_extract_data/results/SCStockOptimizationListItem__c.csv'
+dst_file = './data_for_external_systems/SCStockOptimizationListItem__c.csv'
+shutil.copyfile(src_file, dst_file)
 
 ##--------------------------------------------------------------------------##
 ## Saving to CSV
 ##--------------------------------------------------------------------------##
 
-# Write each DataFrame to a separate CSV file
+# Create directories if they do not exist
+output_dirs = [
+    '../4_upsert_address_and_parent_location',
+    '../6_upsert_child_location',
+    '../9_upsert_assets',
+    '../12_upsert_associated_location',
+    '../13_insert_pricebook2_and_pricebookentries',
+    '../3_update_address_and_location_data_for_migration',
+    '../14_insert_servicecontracttemplates_dummies',
+    '../16_insert_servicecontract',
+    '../20_create_maintenance_plan',
+    '../21_insert_maintenance_asset',
+    '../19_create_contractlineitems',
+    './data_for_external_systems'
+]
+
+for dir_path in output_dirs:
+    if dir_path and not os.path.exists(dir_path):
+        os.makedirs(dir_path, exist_ok=True)
+
 address_df.to_csv('../4_upsert_address_and_parent_location/Address.csv', index=False)
 parent_df.to_csv('../4_upsert_address_and_parent_location/Location.csv', index=False)
 child_df.to_csv('../6_upsert_child_location/Location.csv', index=False)
@@ -658,7 +765,10 @@ df_servicecontracttemplates.to_csv('../14_insert_servicecontracttemplates_dummie
 df_servicecontract.to_csv('../16_insert_servicecontract/ServiceContract_beforetransform.csv', index=False)
 df_assetwarranty_save.to_csv('../9_upsert_assets/AssetWarranty.csv', index=False)
 df_maintenance.to_csv('../9_upsert_assets/FSL_Asset_Maintenance_Information__c.csv', index=False)
-df_maintenanceplan.to_csv('../19_create_maintenance_plan/MaintenancePlan_beforetransform.csv', index=False)
+df_maintenanceplan.to_csv('../20_create_maintenance_plan/MaintenancePlan_beforetransform.csv', index=False)
+df_maintenanceasset.to_csv('../21_insert_maintenance_asset/MaintenanceAsset_beforetransform.csv', index=False)
+df_contractlineitem.to_csv('../19_create_contractlineitems/ContractLineItem.csv', index=False)
+read_df_stockitem.to_csv('./data_for_external_systems/SAP_Stocks.csv', index=False)
 ## end mapping
 
 print('Data has been successfully transformed and saved to CSV files.')

sf_auth.py

@@ -23,7 +23,7 @@ def get_credentials(context):
 
     # Load the .env file
     env_file = find_dotenv(".env")
-    load_dotenv(env_file)
+    load_dotenv(env_file, override=True)
     
     # Load all environment variables
     for key, value in os.environ.items():
@@ -70,7 +70,6 @@ def get_sf_connection(context):
                 sf_params['security_token'] = credentials['SECURITY_TOKEN']
             else:
                 raise ValueError("Neither security token nor organization ID provided")
-                
             return Salesforce(**sf_params)
             
         else: