diff --git a/.gitignore b/.gitignore index 84735d1..e85b4da 100644 --- a/.gitignore +++ b/.gitignore @@ -41,4 +41,8 @@ server.crt AssetWarranty.csv FSL_Asset_Maintenance_Information__c.csv MaintenancePlan.csv -MaintenancePlan_beforetransform.csv \ No newline at end of file +MaintenancePlan_beforetransform.csv +*_beforetransform.csv +MaintenanceAsset.csv +ContractLineItem.csv +data_for_external_systems/ \ No newline at end of file diff --git a/__pycache__/sf_auth.cpython-313.pyc b/__pycache__/sf_auth.cpython-313.pyc index 4a0041a..dc99594 100644 Binary files a/__pycache__/sf_auth.cpython-313.pyc and b/__pycache__/sf_auth.cpython-313.pyc differ diff --git a/prepared_steps/19_create_contractlineitems/command.txt b/prepared_steps/19_create_contractlineitems/command.txt new file mode 100644 index 0000000..4c22459 --- /dev/null +++ b/prepared_steps/19_create_contractlineitems/command.txt @@ -0,0 +1 @@ +sf sfdmu run --sourceusername csvfile --targetusername rene.kasseboehmer@vaillant.de.devrene \ No newline at end of file diff --git a/prepared_steps/19_create_contractlineitems/export.json b/prepared_steps/19_create_contractlineitems/export.json new file mode 100644 index 0000000..ea7c189 --- /dev/null +++ b/prepared_steps/19_create_contractlineitems/export.json @@ -0,0 +1,23 @@ +{ + "allOrNone": true, + "excludeIdsFromCSVFiles": true, + "objects": [ + { + "query": "SELECT Id, PKey__c FROM ServiceContract WHERE TemplateId__c != null AND Pricebook2.Country__c = 'NL'", + "operation": "Readonly", + "externalId": "PKey__c", + "master": false + },{ + "query": "SELECT Id, PKey__c FROM Asset WHERE Location.ParentLocation.VisitorAddress.CountryCode = 'NL'", + "operation": "Readonly", + "externalId": "PKey__c", + "master": false + },{ + "query": "SELECT Id, ServiceContractId, AssetId FROM ContractLineItem", + "operation": "Insert", + "externalId": "Id", + "master": true, + "useSourceCSVFile": true + } + ] +} \ No newline at end of file diff --git a/prepared_steps/19_create_maintenance_plan/fill_fields_and_insert.py b/prepared_steps/19_create_contractlineitems/fill_fields_and_insert.py similarity index 86% rename from prepared_steps/19_create_maintenance_plan/fill_fields_and_insert.py rename to prepared_steps/19_create_contractlineitems/fill_fields_and_insert.py index aef8d85..306b4f7 100644 --- a/prepared_steps/19_create_maintenance_plan/fill_fields_and_insert.py +++ b/prepared_steps/19_create_contractlineitems/fill_fields_and_insert.py @@ -48,8 +48,6 @@ if __name__ == '__main__': df_lm = pd.read_csv('../1_extract_data/results/LastMaintenanceInformation.csv', usecols=['Id', 'Order__r.Closed__c', 'Order__r.Contract__c']) df_lo = pd.read_csv('../1_extract_data/results/LastOrderInformation.csv', usecols=['Id', 'Closed__c', 'Contract__c']) - print(df_sc) - print(df_mp) # # Merge df_mp with df_sc including Id based on PKey__c merged_df_mp = pd.merge(df_mp, @@ -58,8 +56,6 @@ if __name__ == '__main__': right_on='PKey__c', how='left') - print(merged_df_mp) - # Rename only 'old_name' to 'new_name' merged_df_mp = merged_df_mp.rename(columns={'sf__Id': 'ServiceContractId'}) @@ -92,7 +88,22 @@ if __name__ == '__main__': merged_df_mp['NextSuggestedMaintenanceDate'] = merged_df_mp['ServiceContractId'].apply(get_next_suggested_maintenance_date) - merged_df_mp['DoesAutoGenerateWorkOrders'] = True + # Set DoesAutoGenerateWorkOrders to True only if a date was found (not defaulted to today) + def does_auto_generate_work_orders(service_contract_id): + # Try to find in LastMaintenanceInformation + lm_rows = df_lm[df_lm['Order__r.Contract__c'] == service_contract_id] + if not lm_rows.empty: + last_date = lm_rows['Order__r.Closed__c'].max() + else: + # If not found, try LastOrderInformation + lo_rows = df_lo[df_lo['Contract__c'] == service_contract_id] + if not lo_rows.empty: + last_date = lo_rows['Closed__c'].max() + else: + last_date = None + return pd.notnull(last_date) + + merged_df_mp['DoesAutoGenerateWorkOrders'] = merged_df_mp['ServiceContractId'].apply(does_auto_generate_work_orders) #transform values into int merged_df_mp['MaintenanceWindowEndDays'] = pd.to_numeric(merged_df_mp['MaintenanceWindowEndDays'], errors='coerce').fillna(0).astype(int) diff --git a/prepared_steps/1_extract_data/extract_via_simple_salesforce.py b/prepared_steps/1_extract_data/extract_via_simple_salesforce.py index c0a4a13..932d3ec 100644 --- a/prepared_steps/1_extract_data/extract_via_simple_salesforce.py +++ b/prepared_steps/1_extract_data/extract_via_simple_salesforce.py @@ -4,6 +4,8 @@ # --output_path extracted_data import os +import re +import ast import itertools import threading import time @@ -13,6 +15,8 @@ import pandas as pd from sys import path path.append('../..') from sf_auth import get_sf_connection +from collections import OrderedDict + done = False @@ -123,7 +127,7 @@ if __name__ == '__main__': sobject = query_def['sobject'] query_str = query_def['query'].replace('{country}', args.country) use_rest = query_def.get('useREST', False) # Default to False if not specified - + if not selected_sobjects or sobject in selected_sobjects: try: output_file = extract_data( @@ -134,6 +138,7 @@ if __name__ == '__main__': use_rest=use_rest ) print(f'Successfully extracted {sobject} to {output_file}') + except Exception as e: print(f'Error extracting {sobject}: {str(e)}') diff --git a/prepared_steps/1_extract_data/queries.json b/prepared_steps/1_extract_data/queries.json index 4b1eb90..090e219 100644 --- a/prepared_steps/1_extract_data/queries.json +++ b/prepared_steps/1_extract_data/queries.json @@ -34,14 +34,14 @@ },{ "sobject": "SCResourceAssignment__c", "useREST": true, - "query": "SELECT Id, ValidTo__c, ValidFrom__c, Country__c, City__c, PostalCode__c, District__c, Street__c, HouseNo__c, Extension__c, FlatNo__c, Floor__c, GeoY__c, GeoX__c, Resource__c, Resource__r.Employee__r.Name, Stock__c, Stock__r.ID2__c, QualificationProfile__c FROM SCResourceAssignment__c WHERE Country__c = '{country}' AND ValidTo__c >= TODAY" + "query": "SELECT Id, ValidTo__c, ValidFrom__c, Country__c, City__c, PostalCode__c, District__c, Street__c, HouseNo__c, Extension__c, FlatNo__c, Floor__c, GeoY__c, GeoX__c, Resource__c, Resource__r.Employee__r.Name, ResourceNumberCalc__c, Stock__c, Stock__r.ID2__c, QualificationProfile__c FROM SCResourceAssignment__c WHERE Country__c = '{country}' AND ValidTo__c >= TODAY" },{ "sobject": "ContractTemplates", "useREST": true, "query": "SELECT id, name, TemplateName__c, status__c, Brand__r.Name, Country__c, Runtime__c FROM SCContract__c WHERE Template__c = null AND Country__c = '{country}'" },{ "sobject": "SCContract__c", - "query": "SELECT id, name, Template__c, status__c, Brand__r.Name, Country__c, Runtime__c, EndDate__c, StartDate__c, Account__c, AccountOwner__c, IoT_Registration_Status__c, Maintenance_price_inclusion_of_VAT__c, MaintenanceAfterDue__c, MaintenanceBeforeDue__c, MaintenanceCreationLeadtime__c, MaintenanceCreationShiftFactor__c, MaintenanceDuration__c, MaintenanceFirstDate__c, MaintenanceLastDate__c, MaintenanceInterval__c, MaintenancePriceRequired__c, util_MaintenancePrice__c, util_MaintenanceDuration__c, util_MaintenanceNextDate__c FROM SCContract__c WHERE Template__c != null AND EndDate__c >= TODAY AND Country__c = '{country}' limit 3" + "query": "SELECT id, name, Template__c, status__c, Brand__r.Name, Country__c, Runtime__c, EndDate__c, StartDate__c, Account__c, AccountOwner__c, IoT_Registration_Status__c, Template__r.SAPContractCategory__c, Maintenance_price_inclusion_of_VAT__c, MaintenanceAfterDue__c, MaintenanceBeforeDue__c, MaintenanceCreationLeadtime__c, MaintenanceCreationShiftFactor__c, MaintenanceDuration__c, MaintenanceFirstDate__c, MaintenanceLastDate__c, MaintenanceInterval__c, MaintenancePriceRequired__c, util_MaintenancePrice__c, util_MaintenanceDuration__c, util_MaintenanceNextDate__c FROM SCContract__c WHERE Template__c != null AND EndDate__c >= TODAY AND Country__c = '{country}' limit 3" },{ "sobject": "WarrantyTerm", "useREST": true, @@ -68,6 +68,18 @@ "sobject": "WorkType", "useREST": true, "query": "SELECT Id, Name from WorkType" + },{ + "sobject": "SCContractItem__c", + "query": "Select Id, Name, InstalledBase__c, Contract__c, MaintenancePrice__c FROM SCContractItem__c WHERE Contract__r.Country__c = '{country}' AND Contract__r.Template__c != null AND Contract__r.EndDate__c >= TODAY " + },{ + "sobject": "PricebookEntry", + "query": "SELECT Id, Product2.Product_Code__c FROM PricebookEntry WHERE Pricebook2.Country__c = '{country}'" + },{ + "sobject": "SCStockItem__c", + "query": "SELECT Id, PlantERP__c, StockERP__c, Stock__r.Name__c, ArticleNo__c, ArticleNameCalc__c, Qty__c, ERPQty__c, DifferenceQty__c, MinQty__c, MaxQty__c, LastModifiedDate, Stock__r.RunReplenishment__c, Stock__c, Stock__r.StockERP__c, Article__r.EANCode__c, Article__r.Name, Article__c FROM SCStockItem__c WHERE Stock__r.Country__c = '{country}'" + },{ + "sobject": "SCStockOptimizationListItem__c", + "query": "SELECT Id, Article_Name__c, Article__r.Name, StockOptimizationList__r.Name, MaxQty__c, LastModifiedDate FROM SCStockOptimizationListItem__c WHERE StockOptimizationList__r.Plant__r.Country__c = '{country}'" } ] } \ No newline at end of file diff --git a/prepared_steps/19_create_maintenance_plan/command.txt b/prepared_steps/20_create_maintenance_plan/command.txt similarity index 100% rename from prepared_steps/19_create_maintenance_plan/command.txt rename to prepared_steps/20_create_maintenance_plan/command.txt diff --git a/prepared_steps/19_create_maintenance_plan/export.json b/prepared_steps/20_create_maintenance_plan/export.json similarity index 100% rename from prepared_steps/19_create_maintenance_plan/export.json rename to prepared_steps/20_create_maintenance_plan/export.json diff --git a/prepared_steps/20_create_maintenance_plan/fill_fields_and_insert.py b/prepared_steps/20_create_maintenance_plan/fill_fields_and_insert.py new file mode 100644 index 0000000..306b4f7 --- /dev/null +++ b/prepared_steps/20_create_maintenance_plan/fill_fields_and_insert.py @@ -0,0 +1,119 @@ +import pandas as pd +import argparse +from sys import path +path.append('../..') +from utils import bulk_insert_records +from datetime import datetime, timedelta + +# Find last maintenance/order date for each ServiceContractId +def get_last_maintenance_date(service_contract_id): + # Try to find in LastMaintenanceInformation + lm_rows = df_lm[df_lm['Order__r.Contract__c'] == service_contract_id] + if not lm_rows.empty: + # Use the most recent Order__r.Closed__c + return lm_rows['Order__r.Closed__c'].max() + # If not found, try LastOrderInformation + lo_rows = df_lo[df_lo['Contract__c'] == service_contract_id] + if not lo_rows.empty: + return lo_rows['Closed__c'].max() + return None + +if __name__ == '__main__': + + parser = argparse.ArgumentParser(description='Insert MaintenancePlan records via Bulk API') + parser.add_argument('--context', type=str, required=True, + help='Salesforce org context (e.g., "qa2", "prod")') + parser.add_argument('--csv', type=str, default='MaintenancePlan.csv', + help='CSV file to process (default: MaintenancePlan.csv)') + args = parser.parse_args() + + #read_df_sc = pd.read_csv('../16_insert_servicecontract/successful_records.csv', header=0, keep_default_na=False, dtype=str) + read_df_mp = pd.read_csv('./MaintenancePlan_beforetransform.csv', header=0, keep_default_na=False, dtype=str) + + #"sf__Id","sf__Created",PKey__c,BillingCountryCode,Term,EndDate,StartDate,AccountId,Service_Recipient__c,IoT_Registration_Status__c,Name,Pricebook2Id,TemplateId__c + #reindex_columns_sc = ['sf__Id','sf__Created','PKey__c', 'BillingCountryCode', 'Term', 'EndDate', 'StartDate', 'AccountId', 'Service_Recipient__c', 'IoT_Registration_Status__c', 'Name', 'Pricebook2Id', 'TemplateId__c'] + #ServiceContract.PKey__c,StartDate,EndDate,AccountId,MaintenanceWindowEndDays,MaintenanceWindowStartDays,GenerationTimeframe,Frequency,GenerationTimeframeType,WorkTypeId + reindex_columns_mp = ['ServiceContract.PKey__c', 'StartDate', 'EndDate', 'AccountId', 'MaintenanceWindowEndDays', 'MaintenanceWindowStartDays', 'GenerationTimeframe', 'Frequency', 'GenerationTimeframeType', 'WorkTypeId'] + reindex_columns_lastmaintenance = [] + + + # Reindex the columns to match the desired format + #df_sc = read_df_sc.reindex(reindex_columns_sc, axis=1) + df_mp = read_df_mp.reindex(reindex_columns_mp, axis=1) + + + # Only load 'sf__Id' and 'PKey__c' from the ServiceContract CSV + df_sc = pd.read_csv('../16_insert_servicecontract/successful_records.csv', usecols=['sf__Id', 'PKey__c']) + # Load LastMaintenanceInformation and LastOrderInformation CSVs for NextSuggestedMaintenanceDate calculation + df_lm = pd.read_csv('../1_extract_data/results/LastMaintenanceInformation.csv', usecols=['Id', 'Order__r.Closed__c', 'Order__r.Contract__c']) + df_lo = pd.read_csv('../1_extract_data/results/LastOrderInformation.csv', usecols=['Id', 'Closed__c', 'Contract__c']) + + # + # Merge df_mp with df_sc including Id based on PKey__c + merged_df_mp = pd.merge(df_mp, + df_sc[['sf__Id', 'PKey__c']], + left_on='ServiceContract.PKey__c', + right_on='PKey__c', + how='left') + + # Rename only 'old_name' to 'new_name' + merged_df_mp = merged_df_mp.rename(columns={'sf__Id': 'ServiceContractId'}) + + merged_df_mp = merged_df_mp.drop('ServiceContract.PKey__c', axis=1) + merged_df_mp = merged_df_mp.drop('PKey__c', axis=1) + + def get_next_suggested_maintenance_date(service_contract_id): + # Try to find in LastMaintenanceInformation + lm_rows = df_lm[df_lm['Order__r.Contract__c'] == service_contract_id] + if not lm_rows.empty: + last_date = lm_rows['Order__r.Closed__c'].max() + else: + # If not found, try LastOrderInformation + lo_rows = df_lo[df_lo['Contract__c'] == service_contract_id] + if not lo_rows.empty: + last_date = lo_rows['Closed__c'].max() + else: + last_date = None + + if pd.notnull(last_date): + try: + dt = pd.to_datetime(last_date) + next_date = dt + pd.DateOffset(years=1) + return next_date.strftime('%Y-%m-%d') + except Exception: + pass + # If no date found or parsing fails, use today + next_date = datetime.today() + return next_date.strftime('%Y-%m-%d') + + merged_df_mp['NextSuggestedMaintenanceDate'] = merged_df_mp['ServiceContractId'].apply(get_next_suggested_maintenance_date) + + # Set DoesAutoGenerateWorkOrders to True only if a date was found (not defaulted to today) + def does_auto_generate_work_orders(service_contract_id): + # Try to find in LastMaintenanceInformation + lm_rows = df_lm[df_lm['Order__r.Contract__c'] == service_contract_id] + if not lm_rows.empty: + last_date = lm_rows['Order__r.Closed__c'].max() + else: + # If not found, try LastOrderInformation + lo_rows = df_lo[df_lo['Contract__c'] == service_contract_id] + if not lo_rows.empty: + last_date = lo_rows['Closed__c'].max() + else: + last_date = None + return pd.notnull(last_date) + + merged_df_mp['DoesAutoGenerateWorkOrders'] = merged_df_mp['ServiceContractId'].apply(does_auto_generate_work_orders) + + #transform values into int + merged_df_mp['MaintenanceWindowEndDays'] = pd.to_numeric(merged_df_mp['MaintenanceWindowEndDays'], errors='coerce').fillna(0).astype(int) + merged_df_mp['MaintenanceWindowStartDays'] = pd.to_numeric(merged_df_mp['MaintenanceWindowStartDays'], errors='coerce').fillna(0).astype(int) + merged_df_mp['GenerationTimeframe'] = pd.to_numeric(merged_df_mp['GenerationTimeframe'], errors='coerce').fillna(0).astype(int) + merged_df_mp['Frequency'] = pd.to_numeric(merged_df_mp['Frequency'], errors='coerce').fillna(0).astype(int) + + #safe csv + merged_df_mp.to_csv('./MaintenancePlan.csv', index=False) + + + bulk_insert_records(args.context, 'MaintenancePlan', args.csv) + diff --git a/prepared_steps/21_insert_maintenance_asset/command.txt b/prepared_steps/21_insert_maintenance_asset/command.txt new file mode 100644 index 0000000..8b318e4 --- /dev/null +++ b/prepared_steps/21_insert_maintenance_asset/command.txt @@ -0,0 +1 @@ +python link_ids_and_create_maintenanceassets.py --context qa2 --csv MaintenanceAsset.csv \ No newline at end of file diff --git a/prepared_steps/21_insert_maintenance_asset/link_ids_and_create_maintenanceassets.py b/prepared_steps/21_insert_maintenance_asset/link_ids_and_create_maintenanceassets.py new file mode 100644 index 0000000..0b755e8 --- /dev/null +++ b/prepared_steps/21_insert_maintenance_asset/link_ids_and_create_maintenanceassets.py @@ -0,0 +1,81 @@ +import pandas as pd +import os +from sys import path +import argparse +path.append('../..') +from utils import bulk_insert_records + +if __name__ == '__main__': + + parser = argparse.ArgumentParser(description='Insert MaintenancePlan records via Bulk API') + parser.add_argument('--context', type=str, required=True, + help='Salesforce org context (e.g., "qa2", "prod")') + parser.add_argument('--csv', type=str, default='MaintenancePlan.csv', + help='CSV file to process (default: MaintenanceAsset.csv)') + args = parser.parse_args() + + df_s_servicecontracts = pd.read_csv("../16_insert_servicecontract/successful_records.csv") + df_s_maintenanceplans = pd.read_csv("../20_create_maintenance_plan/successful_records.csv") + df_maintenanceasset_before = pd.read_csv("MaintenanceAsset_beforetransform.csv") + + + asset_update_path = os.path.join("..", "9_upsert_assets", "target", "Asset_update_target.csv") + if os.path.exists(asset_update_path): + df_asset_update = pd.read_csv(asset_update_path) + else: + df_asset_update = None + + asset_insert_path = os.path.join("..", "9_upsert_assets", "target", "Asset_insert_target.csv") + if os.path.exists(asset_insert_path): + df_asset_insert = pd.read_csv(asset_insert_path) + else: + df_asset_insert = None + + # Merge the two DataFrames on PKey__c from df_s_servicecontracts and Contract__c from df_s_maintenanceplans + merged_df = pd.merge( + df_s_servicecontracts[['sf__Id', 'PKey__c']], + df_s_maintenanceplans[['sf__Id', 'ServiceContractId']], + left_on='sf__Id', + right_on='ServiceContractId', + how='inner', + suffixes=('_servicecontract', '_maintenanceplan') + ) + + # Create a new DataFrame with only the desired columns + result_df = merged_df[['sf__Id_maintenanceplan', 'sf__Id_servicecontract', 'PKey__c']] + + # Merge the sf__Id_maintenanceplan and sf__Id_servicecontract into df_maintenanceasset_before based on PKey__c + df_maintenanceasset_before = pd.merge( + df_maintenanceasset_before, + result_df, + left_on='ServiceContract.PKey__c', + right_on='PKey__c', + how='left' + ) + + # Find Asset Ids from df_asset_update and df_asset_insert using PKey__c and InstalledBase__c + def find_asset_id(row): + asset_id = None + if df_asset_update is not None: + match = df_asset_update[df_asset_update['PKey__c'] == row['InstalledBase__c']] + if not match.empty: + asset_id = match.iloc[0]['Id'] + if asset_id is None and df_asset_insert is not None: + match = df_asset_insert[df_asset_insert['PKey__c'] == row['InstalledBase__c']] + if not match.empty: + asset_id = match.iloc[0]['Id'] + return asset_id + + df_maintenanceasset_before['AssetId'] = df_maintenanceasset_before.apply(find_asset_id, axis=1) + + df_maintenanceasset_before.drop(columns=['PKey__c'], inplace=True) + df_maintenanceasset_before.drop(columns=['InstalledBase__c'], inplace=True) + df_maintenanceasset_before.drop(columns=['ServiceContract.PKey__c'], inplace=True) + + df_maintenanceasset_before.rename(columns={'sf__Id_maintenanceplan': 'MaintenancePlanId', 'sf__Id_servicecontract': 'ServiceContractId'}, inplace=True) + + #safe csv + df_maintenanceasset_before.to_csv('./MaintenanceAsset.csv', index=False) + + #insert records + bulk_insert_records(args.context, 'MaintenanceAsset', args.csv) \ No newline at end of file diff --git a/prepared_steps/2_transform_via_script/TransformScript.py b/prepared_steps/2_transform_via_script/TransformScript.py index b13c4dd..09f5427 100644 --- a/prepared_steps/2_transform_via_script/TransformScript.py +++ b/prepared_steps/2_transform_via_script/TransformScript.py @@ -1,5 +1,7 @@ import pandas as pd from tqdm import tqdm +import os +import shutil country_mapping = { 'NL': 'Netherlands', @@ -45,6 +47,9 @@ read_df_qualificationprofile = pd.read_csv('../1_extract_data/results/SCQualific read_df_qualificationprofileitem = pd.read_csv('../1_extract_data/results/SCQualificationProfileItem__c.csv', header=0, keep_default_na=False, dtype=str) read_df_lastmaintenance = pd.read_csv('../1_extract_data/results/LastMaintenanceInformation.csv', header=0, keep_default_na=False, dtype=str) read_df_worktype = pd.read_csv('../1_extract_data/results/WorkType.csv', header=0, keep_default_na=False, dtype=str) +read_df_contractitem = pd.read_csv('../1_extract_data/results/SCContractItem__c.csv', header=0, keep_default_na=False, dtype=str) +df_pricebookentry = pd.read_csv('../1_extract_data/results/PricebookEntry.csv', header=0, keep_default_na=False, dtype=str, usecols=['Id', 'Product2.Product_Code__c']) +read_df_stockitem = pd.read_csv('../1_extract_data/results/SCStockItem__c.csv', header=0, keep_default_na=False, dtype=str, usecols=['Id', 'PlantERP__c', 'StockERP__c', 'Stock__r.Name__c', 'Stock__r.RunReplenishment__c', 'Stock__r.StockERP__c', 'ArticleNo__c', 'ArticleNameCalc__c', 'Qty__c', 'ERPQty__c', 'DifferenceQty__c', 'MinQty__c', 'MaxQty__c', 'LastModifiedDate', 'Stock__c', 'Article__r.EANCode__c', 'Article__r.Name', 'Article__c']) # Columns for reindexing reindex_columns = ['Id','City__c','Country__c','GeoY__c','GeoX__c','PostalCode__c','Street__c','Extension__c','HouseNo__c','FlatNo__c','Floor__c'] @@ -53,16 +58,17 @@ reindex_columns_product2 = ['Id','Main_Product_Group__c','Family','MaterialType_ reindex_columns_ibr = ['Id', 'InstalledBaseLocation__c', 'Role__c', 'ValidFrom__c', 'ValidTo__c', 'Account__c'] reindex_columns_pricelist = ['Id', 'Name', 'Brand__r.Name', 'Country__c'] reindex_columns_pricelistitem = ['Id', 'Article__r.Name', 'Article__r.EANCode__c', 'Price__c', 'PriceUnit__c', 'Pricelist__c', 'ValidFrom__c', 'ValidTo__c', 'Pricelist__r.Brand__r.Name', 'Pricelist__r.Country__c'] -reindex_columns_resourceassignment = ['Id', 'ValidTo__c', 'ValidFrom__c', 'Country__c', 'City__c', 'PostalCode__c', 'District__c', 'Street__c', 'HouseNo__c', 'Extension__c', 'FlatNo__c', 'Floor__c', 'GeoY__c', 'GeoX__c', 'Resource__c', 'Resource__r.Employee__r.Name', 'Stock__c', 'Stock__r.ID2__c'] +reindex_columns_resourceassignment = ['Id', 'ValidTo__c', 'ValidFrom__c', 'Country__c', 'City__c', 'PostalCode__c', 'District__c', 'Street__c', 'HouseNo__c', 'Extension__c', 'FlatNo__c', 'Floor__c', 'GeoY__c', 'GeoX__c', 'Resource__c', 'Resource__r.Employee__r.Name', 'ResourceNumberCalc__c', 'Stock__c', 'Stock__r.ID2__c'] reindex_columns_address_iot = ['Id', 'Country', 'CountryCode', 'Street', 'City', 'ParentId', 'PostalCode'] reindex_columns_location_iot = ['Id', 'Name'] reindex_columns_servicecontracttemplates = ['Id', 'Name', 'TemplateName__c', 'Status__c', 'Brand__r.Name', 'Country__c', 'Runtime__c'] -reindex_columns_servicecontracts = ['Id', 'Name', 'Template__c', 'Status__c', 'Brand__r.Name', 'Country__c', 'Runtime__c', 'EndDate__c', 'StartDate__c', 'Account__c', 'AccountOwner__c', 'IoT_Registration_Status__c', 'Maintenance_price_inclusion_of_VAT__c', 'MaintenanceAfterDue__c', 'MaintenanceBeforeDue__c', 'MaintenanceCreationLeadtime__c', 'MaintenanceCreationShiftFactor__c', 'MaintenanceDuration__c', 'MaintenanceFirstDate__c', 'MaintenanceLastDate__c', 'MaintenanceInterval__c', 'MaintenancePriceRequired__c', 'util_MaintenancePrice__c', 'util_MaintenanceDuration__c', 'util_MaintenanceNextDate__c'] +reindex_columns_servicecontracts = ['Id', 'Name', 'Template__c', 'Status__c', 'Brand__r.Name', 'Country__c', 'Runtime__c', 'EndDate__c', 'StartDate__c', 'Account__c', 'AccountOwner__c', 'IoT_Registration_Status__c', 'Template__r.SAPContractCategory__c', 'Maintenance_price_inclusion_of_VAT__c', 'MaintenanceAfterDue__c', 'MaintenanceBeforeDue__c', 'MaintenanceCreationLeadtime__c', 'MaintenanceCreationShiftFactor__c', 'MaintenanceDuration__c', 'MaintenanceFirstDate__c', 'MaintenanceLastDate__c', 'MaintenanceInterval__c', 'MaintenancePriceRequired__c', 'util_MaintenancePrice__c', 'util_MaintenanceDuration__c', 'util_MaintenanceNextDate__c'] reindex_columns_warrantyterm = ['Id', 'WarrantyTermName', 'WarrantyDuration', 'WarrantyType', 'Pricebook2'] reindex_columns_qualificationprofile = ['Id', 'Name', 'Description__c', 'ID2__c', 'Standard__c'] reindex_columns_qualificationprofileitem = ['CertificationId__c', 'Id', 'Info__c', 'Level__c', 'ValidFrom__c', 'ValidTo__c', 'Name', 'Qualification__c'] reindex_columns_lastmaintenance = ['Id', 'InstalledBase__r.SerialNo__c', 'Order__r.Closed__c'] reindex_columns_worktype = ['Id', 'Name'] +reindex_columns_contractitem = ['Id', 'Name', 'InstalledBase__c', 'Contract__c'] # Reindex the columns to match the desired format df = read_df.reindex(reindex_columns, axis=1) @@ -81,6 +87,7 @@ df_qualificationprofile = read_df_qualificationprofile.reindex(reindex_columns_q df_qualificationprofileitem = read_df_qualificationprofileitem.reindex(reindex_columns_qualificationprofileitem, axis=1) df_lastmaintenance = read_df_lastmaintenance.reindex(reindex_columns_lastmaintenance, axis=1) df_worktype = read_df_worktype.reindex(reindex_columns_worktype, axis=1) +df_contractitem = read_df_contractitem.reindex(reindex_columns_contractitem, axis=1) ##--------------------------------------------------------------------------## ## Update for IoT Addresses and Locations @@ -563,14 +570,16 @@ df_servicecontracttemplates['Term'] = df_servicecontracttemplates['Term'].fillna ## Service Contract ##--------------------------------------------------------------------------## -# Create new dataframe for maintenance plan fields -df_maintenanceplan = df_servicecontract[['Id', 'StartDate__c', 'EndDate__c', 'Account__c', 'Maintenance_price_inclusion_of_VAT__c', 'MaintenanceAfterDue__c', - 'MaintenanceBeforeDue__c', 'MaintenanceCreationLeadtime__c', - 'MaintenanceCreationShiftFactor__c', 'MaintenanceDuration__c', - 'MaintenanceFirstDate__c', 'MaintenanceLastDate__c', - 'MaintenanceInterval__c', 'MaintenancePriceRequired__c', - 'util_MaintenancePrice__c', 'util_MaintenanceDuration__c', - 'util_MaintenanceNextDate__c']].copy() +# Create new dataframe for maintenance plan fields only if Template__r.SAPContractCategory__c is 'BM10', 'FC10', 'MP20', 'WX40' +df_maintenanceplan = df_servicecontract[ + df_servicecontract['Template__r.SAPContractCategory__c'].isin(['BM10', 'FC10', 'MP20', 'WX40']) +][['Id', 'StartDate__c', 'EndDate__c', 'Account__c', 'Maintenance_price_inclusion_of_VAT__c', 'MaintenanceAfterDue__c', + 'MaintenanceBeforeDue__c', 'MaintenanceCreationLeadtime__c', + 'MaintenanceCreationShiftFactor__c', 'MaintenanceDuration__c', + 'MaintenanceFirstDate__c', 'MaintenanceLastDate__c', + 'MaintenanceInterval__c', 'MaintenancePriceRequired__c', + 'util_MaintenancePrice__c', 'util_MaintenanceDuration__c', + 'util_MaintenanceNextDate__c']].copy() # Drop maintenance fields from service contract dataframe, except Id maintenance_fields = ['Maintenance_price_inclusion_of_VAT__c', 'MaintenanceAfterDue__c', @@ -591,7 +600,17 @@ df_servicecontract['Pricebook2.Name'] = ( df_servicecontract = df_servicecontract.drop('Brand__r.Name', axis=1) -df_servicecontract.columns = ['PKey__c', 'Name', 'TemplateId__r.PKey__c', 'Status', 'BillingCountryCode', 'Term', 'EndDate', 'StartDate', 'AccountId', 'Service_Recipient__c', 'IoT_Registration_Status__c', 'Pricebook2.Name'] +# Map Template__r.SAPContractCategory__c values to new names +contract_category_mapping = { + "BM10": "Basic Maintenance Contract", + "FC10": "Basic Maintenance Contract", + "MP20": "Basic Plus Maintenance Contract", + "WX40": "Warranty Extension Contract", + "RD30": "IoT" +} +df_servicecontract["Template__r.SAPContractCategory__c"] = df_servicecontract["Template__r.SAPContractCategory__c"].replace(contract_category_mapping) + +df_servicecontract.columns = ['PKey__c', 'Name', 'TemplateId__r.PKey__c', 'Status', 'BillingCountryCode', 'Term', 'EndDate', 'StartDate', 'AccountId', 'Service_Recipient__c', 'IoT_Registration_Status__c', 'FSL_Type_of_Contract__c', 'Pricebook2.Name'] df_servicecontract['IoT_Registration_Status__c'] = df_servicecontract['IoT_Registration_Status__c'].replace('', 'Open') #df_servicecontract['Name'] = df_servicecontract['PKey__c'] @@ -603,10 +622,42 @@ df_servicecontract['TemplateCountry__c'] = df_servicecontract['BillingCountryCod df_servicecontract['Term'] = pd.to_numeric(df_servicecontract['Term'].str.extract('(\d+)')[0], errors='coerce') df_servicecontract['Term'] = df_servicecontract['Term'].fillna(0).astype(int) +##--------------------------------------------------------------------------## +## ContractLineItem +##--------------------------------------------------------------------------## + +df_contractlineitem = df_contractitem.copy() + +df_contractlineitem.columns = ['Id', 'Name', 'Asset.PKey__c', 'ServiceContract.PKey__c'] + +df_contractlineitem.drop(['Id'], axis=1, inplace=True) +df_contractlineitem.drop(['Name'], axis=1, inplace=True) + +# Map Product2.Product_Code__c to df_contractlineitem based on Asset.PKey__c +df_contractlineitem = pd.merge( + df_contractlineitem, + merged_df_ib[['PKey__c', 'Product2.Product_Code__c']], + left_on='Asset.PKey__c', + right_on='PKey__c', + how='left' +) +df_contractlineitem = df_contractlineitem.drop('PKey__c', axis=1) + +# Merge df_contractlineitem with df_pricebookentry to get the PricebookEntry Id based on Product2.Product_Code__c +df_contractlineitem = pd.merge( + df_contractlineitem, + df_pricebookentry[['Product2.Product_Code__c', 'Id']], + left_on='Product2.Product_Code__c', + right_on='Product2.Product_Code__c', + how='left' +) +df_contractlineitem = df_contractlineitem.rename(columns={'Id': 'PricebookEntryId'}) + +print(df_contractlineitem) + ##--------------------------------------------------------------------------## ## MaintenancePlan, MaintenanceAsset ##--------------------------------------------------------------------------## -print(df_maintenanceplan.columns) df_maintenanceplan.columns = ['ServiceContract.PKey__c', 'StartDate', 'EndDate', 'AccountId', 'MaintenancePriceInclusionOfVAT__c', 'MaintenanceWindowEndDays', 'MaintenanceWindowStartDays', 'GenerationTimeframe', @@ -631,21 +682,77 @@ maintenance_worktype_id = df_worktype[df_worktype['Name'] == 'Planned Maintenanc # Add WorkTypeId to maintenance plan df_maintenanceplan['WorkTypeId'] = maintenance_worktype_id -print(df_maintenanceplan) +# Map InstalledBase__c from df_contractitem to df_maintenanceasset based on ServiceContract.PKey__c (Contract__c) +# If there are multiple InstalledBase__c for a Contract__c, join them as a semicolon-separated string + +# Group by Contract__c and aggregate InstalledBase__c as a semicolon-separated string +contractitem_grouped = df_contractitem.groupby('Contract__c')['InstalledBase__c'].apply(lambda x: ';'.join(x.dropna().unique())).reset_index() + +# Merge with df_maintenanceplan +df_maintenanceasset = df_maintenanceplan[['ServiceContract.PKey__c']].copy() +df_maintenanceasset = df_maintenanceasset.merge( + contractitem_grouped, + left_on='ServiceContract.PKey__c', + right_on='Contract__c', + how='left' +) + +df_maintenanceasset = df_maintenanceasset.rename(columns={'InstalledBase__c': 'InstalledBase__c'}) +df_maintenanceasset = df_maintenanceasset[['ServiceContract.PKey__c', 'InstalledBase__c']] + + + ##--------------------------------------------------------------------------## ## Skills ##--------------------------------------------------------------------------## +##--------------------------------------------------------------------------## +## Stocks +##--------------------------------------------------------------------------## +# Merge Resource__r.Employee__r.Name from df_resourceassignment to read_df_stockitem via Stock__c +read_df_stockitem = pd.merge( + read_df_stockitem, + df_resourceassignment[['Stock__c', 'ResourceNumberCalc__c']], + left_on='Stock__c', + right_on='Stock__c', + how='left' +) +##--------------------------------------------------------------------------## +## StocksOptimizationListItem +##--------------------------------------------------------------------------## + +src_file = '../1_extract_data/results/SCStockOptimizationListItem__c.csv' +dst_file = './data_for_external_systems/SCStockOptimizationListItem__c.csv' +shutil.copyfile(src_file, dst_file) ##--------------------------------------------------------------------------## ## Saving to CSV ##--------------------------------------------------------------------------## -# Write each DataFrame to a separate CSV file +# Create directories if they do not exist +output_dirs = [ + '../4_upsert_address_and_parent_location', + '../6_upsert_child_location', + '../9_upsert_assets', + '../12_upsert_associated_location', + '../13_insert_pricebook2_and_pricebookentries', + '../3_update_address_and_location_data_for_migration', + '../14_insert_servicecontracttemplates_dummies', + '../16_insert_servicecontract', + '../20_create_maintenance_plan', + '../21_insert_maintenance_asset', + '../19_create_contractlineitems', + './data_for_external_systems' +] + +for dir_path in output_dirs: + if dir_path and not os.path.exists(dir_path): + os.makedirs(dir_path, exist_ok=True) + address_df.to_csv('../4_upsert_address_and_parent_location/Address.csv', index=False) parent_df.to_csv('../4_upsert_address_and_parent_location/Location.csv', index=False) child_df.to_csv('../6_upsert_child_location/Location.csv', index=False) @@ -658,7 +765,10 @@ df_servicecontracttemplates.to_csv('../14_insert_servicecontracttemplates_dummie df_servicecontract.to_csv('../16_insert_servicecontract/ServiceContract_beforetransform.csv', index=False) df_assetwarranty_save.to_csv('../9_upsert_assets/AssetWarranty.csv', index=False) df_maintenance.to_csv('../9_upsert_assets/FSL_Asset_Maintenance_Information__c.csv', index=False) -df_maintenanceplan.to_csv('../19_create_maintenance_plan/MaintenancePlan_beforetransform.csv', index=False) +df_maintenanceplan.to_csv('../20_create_maintenance_plan/MaintenancePlan_beforetransform.csv', index=False) +df_maintenanceasset.to_csv('../21_insert_maintenance_asset/MaintenanceAsset_beforetransform.csv', index=False) +df_contractlineitem.to_csv('../19_create_contractlineitems/ContractLineItem.csv', index=False) +read_df_stockitem.to_csv('./data_for_external_systems/SAP_Stocks.csv', index=False) ## end mapping print('Data has been successfully transformed and saved to CSV files.') \ No newline at end of file diff --git a/sf_auth.py b/sf_auth.py index b0ee50f..bdca901 100644 --- a/sf_auth.py +++ b/sf_auth.py @@ -23,7 +23,7 @@ def get_credentials(context): # Load the .env file env_file = find_dotenv(".env") - load_dotenv(env_file) + load_dotenv(env_file, override=True) # Load all environment variables for key, value in os.environ.items(): @@ -70,7 +70,6 @@ def get_sf_connection(context): sf_params['security_token'] = credentials['SECURITY_TOKEN'] else: raise ValueError("Neither security token nor organization ID provided") - return Salesforce(**sf_params) else: