From b09b136ad794fc11757b9124678bd6cef58a44de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rene=20Ka=C3=9Feb=C3=B6hmer?= Date: Wed, 9 Jul 2025 11:15:51 +0200 Subject: [PATCH] savestate before github --- .gitignore | 6 +- __pycache__/sf_auth.cpython-313.pyc | Bin 2743 -> 2756 bytes .../19_create_contractlineitems/command.txt | 1 + .../19_create_contractlineitems/export.json | 23 +++ .../fill_fields_and_insert.py | 21 ++- .../extract_via_simple_salesforce.py | 7 +- prepared_steps/1_extract_data/queries.json | 16 +- .../command.txt | 0 .../export.json | 0 .../fill_fields_and_insert.py | 119 +++++++++++++++ .../21_insert_maintenance_asset/command.txt | 1 + .../link_ids_and_create_maintenanceassets.py | 81 ++++++++++ .../2_transform_via_script/TransformScript.py | 140 ++++++++++++++++-- sf_auth.py | 3 +- 14 files changed, 392 insertions(+), 26 deletions(-) create mode 100644 prepared_steps/19_create_contractlineitems/command.txt create mode 100644 prepared_steps/19_create_contractlineitems/export.json rename prepared_steps/{19_create_maintenance_plan => 19_create_contractlineitems}/fill_fields_and_insert.py (86%) rename prepared_steps/{19_create_maintenance_plan => 20_create_maintenance_plan}/command.txt (100%) rename prepared_steps/{19_create_maintenance_plan => 20_create_maintenance_plan}/export.json (100%) create mode 100644 prepared_steps/20_create_maintenance_plan/fill_fields_and_insert.py create mode 100644 prepared_steps/21_insert_maintenance_asset/command.txt create mode 100644 prepared_steps/21_insert_maintenance_asset/link_ids_and_create_maintenanceassets.py diff --git a/.gitignore b/.gitignore index 84735d1..e85b4da 100644 --- a/.gitignore +++ b/.gitignore @@ -41,4 +41,8 @@ server.crt AssetWarranty.csv FSL_Asset_Maintenance_Information__c.csv MaintenancePlan.csv -MaintenancePlan_beforetransform.csv \ No newline at end of file +MaintenancePlan_beforetransform.csv +*_beforetransform.csv +MaintenanceAsset.csv +ContractLineItem.csv +data_for_external_systems/ \ No newline at end of file diff --git a/__pycache__/sf_auth.cpython-313.pyc b/__pycache__/sf_auth.cpython-313.pyc index 4a0041a289e58e3bb11c31fe7f7b582126cd2926..dc995943d15e6eef8520a912581472e6b3c50336 100644 GIT binary patch delta 237 zcmdlkdPJ1>GcPX}0}z~gnv-#OBJWH_y@|{7*@D@FIV_nb%Q5;ga!$@+)E3}0W-?_2 zsbgRW=3-@Fn7CGM@=-=HM()XX7+3LzXfod7$S+GRD#}bro!rLspGzL75U7lyc;n=L z<{;JuCYM+=Ha}tZWMPz_tjFP}BmtBwk^~aBIE&Nb3lfVGbBk{=rWUgRg%p6GNN(~x z4sk})$=f)*7^NpOavos3Gx-o_1*6DhYc4$%knu&PAVM2ND1Zn#AaRSsCO1E&G$+-r R$YOFYmm;S&V;Cb?8UT{>I`jYl delta 219 zcmX>ix?Pm_GcPX}0}#}2RLwXxk#{De!NleH9HxvQF$RXoGK{{A?31$?wFTIXnP8&9 z9IOls6W7X3KFlb_$T|5Id_Vp2Zx*e1%1F^G9Y+7Dl= TODAY" + "query": "SELECT Id, ValidTo__c, ValidFrom__c, Country__c, City__c, PostalCode__c, District__c, Street__c, HouseNo__c, Extension__c, FlatNo__c, Floor__c, GeoY__c, GeoX__c, Resource__c, Resource__r.Employee__r.Name, ResourceNumberCalc__c, Stock__c, Stock__r.ID2__c, QualificationProfile__c FROM SCResourceAssignment__c WHERE Country__c = '{country}' AND ValidTo__c >= TODAY" },{ "sobject": "ContractTemplates", "useREST": true, "query": "SELECT id, name, TemplateName__c, status__c, Brand__r.Name, Country__c, Runtime__c FROM SCContract__c WHERE Template__c = null AND Country__c = '{country}'" },{ "sobject": "SCContract__c", - "query": "SELECT id, name, Template__c, status__c, Brand__r.Name, Country__c, Runtime__c, EndDate__c, StartDate__c, Account__c, AccountOwner__c, IoT_Registration_Status__c, Maintenance_price_inclusion_of_VAT__c, MaintenanceAfterDue__c, MaintenanceBeforeDue__c, MaintenanceCreationLeadtime__c, MaintenanceCreationShiftFactor__c, MaintenanceDuration__c, MaintenanceFirstDate__c, MaintenanceLastDate__c, MaintenanceInterval__c, MaintenancePriceRequired__c, util_MaintenancePrice__c, util_MaintenanceDuration__c, util_MaintenanceNextDate__c FROM SCContract__c WHERE Template__c != null AND EndDate__c >= TODAY AND Country__c = '{country}' limit 3" + "query": "SELECT id, name, Template__c, status__c, Brand__r.Name, Country__c, Runtime__c, EndDate__c, StartDate__c, Account__c, AccountOwner__c, IoT_Registration_Status__c, Template__r.SAPContractCategory__c, Maintenance_price_inclusion_of_VAT__c, MaintenanceAfterDue__c, MaintenanceBeforeDue__c, MaintenanceCreationLeadtime__c, MaintenanceCreationShiftFactor__c, MaintenanceDuration__c, MaintenanceFirstDate__c, MaintenanceLastDate__c, MaintenanceInterval__c, MaintenancePriceRequired__c, util_MaintenancePrice__c, util_MaintenanceDuration__c, util_MaintenanceNextDate__c FROM SCContract__c WHERE Template__c != null AND EndDate__c >= TODAY AND Country__c = '{country}' limit 3" },{ "sobject": "WarrantyTerm", "useREST": true, @@ -68,6 +68,18 @@ "sobject": "WorkType", "useREST": true, "query": "SELECT Id, Name from WorkType" + },{ + "sobject": "SCContractItem__c", + "query": "Select Id, Name, InstalledBase__c, Contract__c, MaintenancePrice__c FROM SCContractItem__c WHERE Contract__r.Country__c = '{country}' AND Contract__r.Template__c != null AND Contract__r.EndDate__c >= TODAY " + },{ + "sobject": "PricebookEntry", + "query": "SELECT Id, Product2.Product_Code__c FROM PricebookEntry WHERE Pricebook2.Country__c = '{country}'" + },{ + "sobject": "SCStockItem__c", + "query": "SELECT Id, PlantERP__c, StockERP__c, Stock__r.Name__c, ArticleNo__c, ArticleNameCalc__c, Qty__c, ERPQty__c, DifferenceQty__c, MinQty__c, MaxQty__c, LastModifiedDate, Stock__r.RunReplenishment__c, Stock__c, Stock__r.StockERP__c, Article__r.EANCode__c, Article__r.Name, Article__c FROM SCStockItem__c WHERE Stock__r.Country__c = '{country}'" + },{ + "sobject": "SCStockOptimizationListItem__c", + "query": "SELECT Id, Article_Name__c, Article__r.Name, StockOptimizationList__r.Name, MaxQty__c, LastModifiedDate FROM SCStockOptimizationListItem__c WHERE StockOptimizationList__r.Plant__r.Country__c = '{country}'" } ] } \ No newline at end of file diff --git a/prepared_steps/19_create_maintenance_plan/command.txt b/prepared_steps/20_create_maintenance_plan/command.txt similarity index 100% rename from prepared_steps/19_create_maintenance_plan/command.txt rename to prepared_steps/20_create_maintenance_plan/command.txt diff --git a/prepared_steps/19_create_maintenance_plan/export.json b/prepared_steps/20_create_maintenance_plan/export.json similarity index 100% rename from prepared_steps/19_create_maintenance_plan/export.json rename to prepared_steps/20_create_maintenance_plan/export.json diff --git a/prepared_steps/20_create_maintenance_plan/fill_fields_and_insert.py b/prepared_steps/20_create_maintenance_plan/fill_fields_and_insert.py new file mode 100644 index 0000000..306b4f7 --- /dev/null +++ b/prepared_steps/20_create_maintenance_plan/fill_fields_and_insert.py @@ -0,0 +1,119 @@ +import pandas as pd +import argparse +from sys import path +path.append('../..') +from utils import bulk_insert_records +from datetime import datetime, timedelta + +# Find last maintenance/order date for each ServiceContractId +def get_last_maintenance_date(service_contract_id): + # Try to find in LastMaintenanceInformation + lm_rows = df_lm[df_lm['Order__r.Contract__c'] == service_contract_id] + if not lm_rows.empty: + # Use the most recent Order__r.Closed__c + return lm_rows['Order__r.Closed__c'].max() + # If not found, try LastOrderInformation + lo_rows = df_lo[df_lo['Contract__c'] == service_contract_id] + if not lo_rows.empty: + return lo_rows['Closed__c'].max() + return None + +if __name__ == '__main__': + + parser = argparse.ArgumentParser(description='Insert MaintenancePlan records via Bulk API') + parser.add_argument('--context', type=str, required=True, + help='Salesforce org context (e.g., "qa2", "prod")') + parser.add_argument('--csv', type=str, default='MaintenancePlan.csv', + help='CSV file to process (default: MaintenancePlan.csv)') + args = parser.parse_args() + + #read_df_sc = pd.read_csv('../16_insert_servicecontract/successful_records.csv', header=0, keep_default_na=False, dtype=str) + read_df_mp = pd.read_csv('./MaintenancePlan_beforetransform.csv', header=0, keep_default_na=False, dtype=str) + + #"sf__Id","sf__Created",PKey__c,BillingCountryCode,Term,EndDate,StartDate,AccountId,Service_Recipient__c,IoT_Registration_Status__c,Name,Pricebook2Id,TemplateId__c + #reindex_columns_sc = ['sf__Id','sf__Created','PKey__c', 'BillingCountryCode', 'Term', 'EndDate', 'StartDate', 'AccountId', 'Service_Recipient__c', 'IoT_Registration_Status__c', 'Name', 'Pricebook2Id', 'TemplateId__c'] + #ServiceContract.PKey__c,StartDate,EndDate,AccountId,MaintenanceWindowEndDays,MaintenanceWindowStartDays,GenerationTimeframe,Frequency,GenerationTimeframeType,WorkTypeId + reindex_columns_mp = ['ServiceContract.PKey__c', 'StartDate', 'EndDate', 'AccountId', 'MaintenanceWindowEndDays', 'MaintenanceWindowStartDays', 'GenerationTimeframe', 'Frequency', 'GenerationTimeframeType', 'WorkTypeId'] + reindex_columns_lastmaintenance = [] + + + # Reindex the columns to match the desired format + #df_sc = read_df_sc.reindex(reindex_columns_sc, axis=1) + df_mp = read_df_mp.reindex(reindex_columns_mp, axis=1) + + + # Only load 'sf__Id' and 'PKey__c' from the ServiceContract CSV + df_sc = pd.read_csv('../16_insert_servicecontract/successful_records.csv', usecols=['sf__Id', 'PKey__c']) + # Load LastMaintenanceInformation and LastOrderInformation CSVs for NextSuggestedMaintenanceDate calculation + df_lm = pd.read_csv('../1_extract_data/results/LastMaintenanceInformation.csv', usecols=['Id', 'Order__r.Closed__c', 'Order__r.Contract__c']) + df_lo = pd.read_csv('../1_extract_data/results/LastOrderInformation.csv', usecols=['Id', 'Closed__c', 'Contract__c']) + + # + # Merge df_mp with df_sc including Id based on PKey__c + merged_df_mp = pd.merge(df_mp, + df_sc[['sf__Id', 'PKey__c']], + left_on='ServiceContract.PKey__c', + right_on='PKey__c', + how='left') + + # Rename only 'old_name' to 'new_name' + merged_df_mp = merged_df_mp.rename(columns={'sf__Id': 'ServiceContractId'}) + + merged_df_mp = merged_df_mp.drop('ServiceContract.PKey__c', axis=1) + merged_df_mp = merged_df_mp.drop('PKey__c', axis=1) + + def get_next_suggested_maintenance_date(service_contract_id): + # Try to find in LastMaintenanceInformation + lm_rows = df_lm[df_lm['Order__r.Contract__c'] == service_contract_id] + if not lm_rows.empty: + last_date = lm_rows['Order__r.Closed__c'].max() + else: + # If not found, try LastOrderInformation + lo_rows = df_lo[df_lo['Contract__c'] == service_contract_id] + if not lo_rows.empty: + last_date = lo_rows['Closed__c'].max() + else: + last_date = None + + if pd.notnull(last_date): + try: + dt = pd.to_datetime(last_date) + next_date = dt + pd.DateOffset(years=1) + return next_date.strftime('%Y-%m-%d') + except Exception: + pass + # If no date found or parsing fails, use today + next_date = datetime.today() + return next_date.strftime('%Y-%m-%d') + + merged_df_mp['NextSuggestedMaintenanceDate'] = merged_df_mp['ServiceContractId'].apply(get_next_suggested_maintenance_date) + + # Set DoesAutoGenerateWorkOrders to True only if a date was found (not defaulted to today) + def does_auto_generate_work_orders(service_contract_id): + # Try to find in LastMaintenanceInformation + lm_rows = df_lm[df_lm['Order__r.Contract__c'] == service_contract_id] + if not lm_rows.empty: + last_date = lm_rows['Order__r.Closed__c'].max() + else: + # If not found, try LastOrderInformation + lo_rows = df_lo[df_lo['Contract__c'] == service_contract_id] + if not lo_rows.empty: + last_date = lo_rows['Closed__c'].max() + else: + last_date = None + return pd.notnull(last_date) + + merged_df_mp['DoesAutoGenerateWorkOrders'] = merged_df_mp['ServiceContractId'].apply(does_auto_generate_work_orders) + + #transform values into int + merged_df_mp['MaintenanceWindowEndDays'] = pd.to_numeric(merged_df_mp['MaintenanceWindowEndDays'], errors='coerce').fillna(0).astype(int) + merged_df_mp['MaintenanceWindowStartDays'] = pd.to_numeric(merged_df_mp['MaintenanceWindowStartDays'], errors='coerce').fillna(0).astype(int) + merged_df_mp['GenerationTimeframe'] = pd.to_numeric(merged_df_mp['GenerationTimeframe'], errors='coerce').fillna(0).astype(int) + merged_df_mp['Frequency'] = pd.to_numeric(merged_df_mp['Frequency'], errors='coerce').fillna(0).astype(int) + + #safe csv + merged_df_mp.to_csv('./MaintenancePlan.csv', index=False) + + + bulk_insert_records(args.context, 'MaintenancePlan', args.csv) + diff --git a/prepared_steps/21_insert_maintenance_asset/command.txt b/prepared_steps/21_insert_maintenance_asset/command.txt new file mode 100644 index 0000000..8b318e4 --- /dev/null +++ b/prepared_steps/21_insert_maintenance_asset/command.txt @@ -0,0 +1 @@ +python link_ids_and_create_maintenanceassets.py --context qa2 --csv MaintenanceAsset.csv \ No newline at end of file diff --git a/prepared_steps/21_insert_maintenance_asset/link_ids_and_create_maintenanceassets.py b/prepared_steps/21_insert_maintenance_asset/link_ids_and_create_maintenanceassets.py new file mode 100644 index 0000000..0b755e8 --- /dev/null +++ b/prepared_steps/21_insert_maintenance_asset/link_ids_and_create_maintenanceassets.py @@ -0,0 +1,81 @@ +import pandas as pd +import os +from sys import path +import argparse +path.append('../..') +from utils import bulk_insert_records + +if __name__ == '__main__': + + parser = argparse.ArgumentParser(description='Insert MaintenancePlan records via Bulk API') + parser.add_argument('--context', type=str, required=True, + help='Salesforce org context (e.g., "qa2", "prod")') + parser.add_argument('--csv', type=str, default='MaintenancePlan.csv', + help='CSV file to process (default: MaintenanceAsset.csv)') + args = parser.parse_args() + + df_s_servicecontracts = pd.read_csv("../16_insert_servicecontract/successful_records.csv") + df_s_maintenanceplans = pd.read_csv("../20_create_maintenance_plan/successful_records.csv") + df_maintenanceasset_before = pd.read_csv("MaintenanceAsset_beforetransform.csv") + + + asset_update_path = os.path.join("..", "9_upsert_assets", "target", "Asset_update_target.csv") + if os.path.exists(asset_update_path): + df_asset_update = pd.read_csv(asset_update_path) + else: + df_asset_update = None + + asset_insert_path = os.path.join("..", "9_upsert_assets", "target", "Asset_insert_target.csv") + if os.path.exists(asset_insert_path): + df_asset_insert = pd.read_csv(asset_insert_path) + else: + df_asset_insert = None + + # Merge the two DataFrames on PKey__c from df_s_servicecontracts and Contract__c from df_s_maintenanceplans + merged_df = pd.merge( + df_s_servicecontracts[['sf__Id', 'PKey__c']], + df_s_maintenanceplans[['sf__Id', 'ServiceContractId']], + left_on='sf__Id', + right_on='ServiceContractId', + how='inner', + suffixes=('_servicecontract', '_maintenanceplan') + ) + + # Create a new DataFrame with only the desired columns + result_df = merged_df[['sf__Id_maintenanceplan', 'sf__Id_servicecontract', 'PKey__c']] + + # Merge the sf__Id_maintenanceplan and sf__Id_servicecontract into df_maintenanceasset_before based on PKey__c + df_maintenanceasset_before = pd.merge( + df_maintenanceasset_before, + result_df, + left_on='ServiceContract.PKey__c', + right_on='PKey__c', + how='left' + ) + + # Find Asset Ids from df_asset_update and df_asset_insert using PKey__c and InstalledBase__c + def find_asset_id(row): + asset_id = None + if df_asset_update is not None: + match = df_asset_update[df_asset_update['PKey__c'] == row['InstalledBase__c']] + if not match.empty: + asset_id = match.iloc[0]['Id'] + if asset_id is None and df_asset_insert is not None: + match = df_asset_insert[df_asset_insert['PKey__c'] == row['InstalledBase__c']] + if not match.empty: + asset_id = match.iloc[0]['Id'] + return asset_id + + df_maintenanceasset_before['AssetId'] = df_maintenanceasset_before.apply(find_asset_id, axis=1) + + df_maintenanceasset_before.drop(columns=['PKey__c'], inplace=True) + df_maintenanceasset_before.drop(columns=['InstalledBase__c'], inplace=True) + df_maintenanceasset_before.drop(columns=['ServiceContract.PKey__c'], inplace=True) + + df_maintenanceasset_before.rename(columns={'sf__Id_maintenanceplan': 'MaintenancePlanId', 'sf__Id_servicecontract': 'ServiceContractId'}, inplace=True) + + #safe csv + df_maintenanceasset_before.to_csv('./MaintenanceAsset.csv', index=False) + + #insert records + bulk_insert_records(args.context, 'MaintenanceAsset', args.csv) \ No newline at end of file diff --git a/prepared_steps/2_transform_via_script/TransformScript.py b/prepared_steps/2_transform_via_script/TransformScript.py index b13c4dd..09f5427 100644 --- a/prepared_steps/2_transform_via_script/TransformScript.py +++ b/prepared_steps/2_transform_via_script/TransformScript.py @@ -1,5 +1,7 @@ import pandas as pd from tqdm import tqdm +import os +import shutil country_mapping = { 'NL': 'Netherlands', @@ -45,6 +47,9 @@ read_df_qualificationprofile = pd.read_csv('../1_extract_data/results/SCQualific read_df_qualificationprofileitem = pd.read_csv('../1_extract_data/results/SCQualificationProfileItem__c.csv', header=0, keep_default_na=False, dtype=str) read_df_lastmaintenance = pd.read_csv('../1_extract_data/results/LastMaintenanceInformation.csv', header=0, keep_default_na=False, dtype=str) read_df_worktype = pd.read_csv('../1_extract_data/results/WorkType.csv', header=0, keep_default_na=False, dtype=str) +read_df_contractitem = pd.read_csv('../1_extract_data/results/SCContractItem__c.csv', header=0, keep_default_na=False, dtype=str) +df_pricebookentry = pd.read_csv('../1_extract_data/results/PricebookEntry.csv', header=0, keep_default_na=False, dtype=str, usecols=['Id', 'Product2.Product_Code__c']) +read_df_stockitem = pd.read_csv('../1_extract_data/results/SCStockItem__c.csv', header=0, keep_default_na=False, dtype=str, usecols=['Id', 'PlantERP__c', 'StockERP__c', 'Stock__r.Name__c', 'Stock__r.RunReplenishment__c', 'Stock__r.StockERP__c', 'ArticleNo__c', 'ArticleNameCalc__c', 'Qty__c', 'ERPQty__c', 'DifferenceQty__c', 'MinQty__c', 'MaxQty__c', 'LastModifiedDate', 'Stock__c', 'Article__r.EANCode__c', 'Article__r.Name', 'Article__c']) # Columns for reindexing reindex_columns = ['Id','City__c','Country__c','GeoY__c','GeoX__c','PostalCode__c','Street__c','Extension__c','HouseNo__c','FlatNo__c','Floor__c'] @@ -53,16 +58,17 @@ reindex_columns_product2 = ['Id','Main_Product_Group__c','Family','MaterialType_ reindex_columns_ibr = ['Id', 'InstalledBaseLocation__c', 'Role__c', 'ValidFrom__c', 'ValidTo__c', 'Account__c'] reindex_columns_pricelist = ['Id', 'Name', 'Brand__r.Name', 'Country__c'] reindex_columns_pricelistitem = ['Id', 'Article__r.Name', 'Article__r.EANCode__c', 'Price__c', 'PriceUnit__c', 'Pricelist__c', 'ValidFrom__c', 'ValidTo__c', 'Pricelist__r.Brand__r.Name', 'Pricelist__r.Country__c'] -reindex_columns_resourceassignment = ['Id', 'ValidTo__c', 'ValidFrom__c', 'Country__c', 'City__c', 'PostalCode__c', 'District__c', 'Street__c', 'HouseNo__c', 'Extension__c', 'FlatNo__c', 'Floor__c', 'GeoY__c', 'GeoX__c', 'Resource__c', 'Resource__r.Employee__r.Name', 'Stock__c', 'Stock__r.ID2__c'] +reindex_columns_resourceassignment = ['Id', 'ValidTo__c', 'ValidFrom__c', 'Country__c', 'City__c', 'PostalCode__c', 'District__c', 'Street__c', 'HouseNo__c', 'Extension__c', 'FlatNo__c', 'Floor__c', 'GeoY__c', 'GeoX__c', 'Resource__c', 'Resource__r.Employee__r.Name', 'ResourceNumberCalc__c', 'Stock__c', 'Stock__r.ID2__c'] reindex_columns_address_iot = ['Id', 'Country', 'CountryCode', 'Street', 'City', 'ParentId', 'PostalCode'] reindex_columns_location_iot = ['Id', 'Name'] reindex_columns_servicecontracttemplates = ['Id', 'Name', 'TemplateName__c', 'Status__c', 'Brand__r.Name', 'Country__c', 'Runtime__c'] -reindex_columns_servicecontracts = ['Id', 'Name', 'Template__c', 'Status__c', 'Brand__r.Name', 'Country__c', 'Runtime__c', 'EndDate__c', 'StartDate__c', 'Account__c', 'AccountOwner__c', 'IoT_Registration_Status__c', 'Maintenance_price_inclusion_of_VAT__c', 'MaintenanceAfterDue__c', 'MaintenanceBeforeDue__c', 'MaintenanceCreationLeadtime__c', 'MaintenanceCreationShiftFactor__c', 'MaintenanceDuration__c', 'MaintenanceFirstDate__c', 'MaintenanceLastDate__c', 'MaintenanceInterval__c', 'MaintenancePriceRequired__c', 'util_MaintenancePrice__c', 'util_MaintenanceDuration__c', 'util_MaintenanceNextDate__c'] +reindex_columns_servicecontracts = ['Id', 'Name', 'Template__c', 'Status__c', 'Brand__r.Name', 'Country__c', 'Runtime__c', 'EndDate__c', 'StartDate__c', 'Account__c', 'AccountOwner__c', 'IoT_Registration_Status__c', 'Template__r.SAPContractCategory__c', 'Maintenance_price_inclusion_of_VAT__c', 'MaintenanceAfterDue__c', 'MaintenanceBeforeDue__c', 'MaintenanceCreationLeadtime__c', 'MaintenanceCreationShiftFactor__c', 'MaintenanceDuration__c', 'MaintenanceFirstDate__c', 'MaintenanceLastDate__c', 'MaintenanceInterval__c', 'MaintenancePriceRequired__c', 'util_MaintenancePrice__c', 'util_MaintenanceDuration__c', 'util_MaintenanceNextDate__c'] reindex_columns_warrantyterm = ['Id', 'WarrantyTermName', 'WarrantyDuration', 'WarrantyType', 'Pricebook2'] reindex_columns_qualificationprofile = ['Id', 'Name', 'Description__c', 'ID2__c', 'Standard__c'] reindex_columns_qualificationprofileitem = ['CertificationId__c', 'Id', 'Info__c', 'Level__c', 'ValidFrom__c', 'ValidTo__c', 'Name', 'Qualification__c'] reindex_columns_lastmaintenance = ['Id', 'InstalledBase__r.SerialNo__c', 'Order__r.Closed__c'] reindex_columns_worktype = ['Id', 'Name'] +reindex_columns_contractitem = ['Id', 'Name', 'InstalledBase__c', 'Contract__c'] # Reindex the columns to match the desired format df = read_df.reindex(reindex_columns, axis=1) @@ -81,6 +87,7 @@ df_qualificationprofile = read_df_qualificationprofile.reindex(reindex_columns_q df_qualificationprofileitem = read_df_qualificationprofileitem.reindex(reindex_columns_qualificationprofileitem, axis=1) df_lastmaintenance = read_df_lastmaintenance.reindex(reindex_columns_lastmaintenance, axis=1) df_worktype = read_df_worktype.reindex(reindex_columns_worktype, axis=1) +df_contractitem = read_df_contractitem.reindex(reindex_columns_contractitem, axis=1) ##--------------------------------------------------------------------------## ## Update for IoT Addresses and Locations @@ -563,14 +570,16 @@ df_servicecontracttemplates['Term'] = df_servicecontracttemplates['Term'].fillna ## Service Contract ##--------------------------------------------------------------------------## -# Create new dataframe for maintenance plan fields -df_maintenanceplan = df_servicecontract[['Id', 'StartDate__c', 'EndDate__c', 'Account__c', 'Maintenance_price_inclusion_of_VAT__c', 'MaintenanceAfterDue__c', - 'MaintenanceBeforeDue__c', 'MaintenanceCreationLeadtime__c', - 'MaintenanceCreationShiftFactor__c', 'MaintenanceDuration__c', - 'MaintenanceFirstDate__c', 'MaintenanceLastDate__c', - 'MaintenanceInterval__c', 'MaintenancePriceRequired__c', - 'util_MaintenancePrice__c', 'util_MaintenanceDuration__c', - 'util_MaintenanceNextDate__c']].copy() +# Create new dataframe for maintenance plan fields only if Template__r.SAPContractCategory__c is 'BM10', 'FC10', 'MP20', 'WX40' +df_maintenanceplan = df_servicecontract[ + df_servicecontract['Template__r.SAPContractCategory__c'].isin(['BM10', 'FC10', 'MP20', 'WX40']) +][['Id', 'StartDate__c', 'EndDate__c', 'Account__c', 'Maintenance_price_inclusion_of_VAT__c', 'MaintenanceAfterDue__c', + 'MaintenanceBeforeDue__c', 'MaintenanceCreationLeadtime__c', + 'MaintenanceCreationShiftFactor__c', 'MaintenanceDuration__c', + 'MaintenanceFirstDate__c', 'MaintenanceLastDate__c', + 'MaintenanceInterval__c', 'MaintenancePriceRequired__c', + 'util_MaintenancePrice__c', 'util_MaintenanceDuration__c', + 'util_MaintenanceNextDate__c']].copy() # Drop maintenance fields from service contract dataframe, except Id maintenance_fields = ['Maintenance_price_inclusion_of_VAT__c', 'MaintenanceAfterDue__c', @@ -591,7 +600,17 @@ df_servicecontract['Pricebook2.Name'] = ( df_servicecontract = df_servicecontract.drop('Brand__r.Name', axis=1) -df_servicecontract.columns = ['PKey__c', 'Name', 'TemplateId__r.PKey__c', 'Status', 'BillingCountryCode', 'Term', 'EndDate', 'StartDate', 'AccountId', 'Service_Recipient__c', 'IoT_Registration_Status__c', 'Pricebook2.Name'] +# Map Template__r.SAPContractCategory__c values to new names +contract_category_mapping = { + "BM10": "Basic Maintenance Contract", + "FC10": "Basic Maintenance Contract", + "MP20": "Basic Plus Maintenance Contract", + "WX40": "Warranty Extension Contract", + "RD30": "IoT" +} +df_servicecontract["Template__r.SAPContractCategory__c"] = df_servicecontract["Template__r.SAPContractCategory__c"].replace(contract_category_mapping) + +df_servicecontract.columns = ['PKey__c', 'Name', 'TemplateId__r.PKey__c', 'Status', 'BillingCountryCode', 'Term', 'EndDate', 'StartDate', 'AccountId', 'Service_Recipient__c', 'IoT_Registration_Status__c', 'FSL_Type_of_Contract__c', 'Pricebook2.Name'] df_servicecontract['IoT_Registration_Status__c'] = df_servicecontract['IoT_Registration_Status__c'].replace('', 'Open') #df_servicecontract['Name'] = df_servicecontract['PKey__c'] @@ -603,10 +622,42 @@ df_servicecontract['TemplateCountry__c'] = df_servicecontract['BillingCountryCod df_servicecontract['Term'] = pd.to_numeric(df_servicecontract['Term'].str.extract('(\d+)')[0], errors='coerce') df_servicecontract['Term'] = df_servicecontract['Term'].fillna(0).astype(int) +##--------------------------------------------------------------------------## +## ContractLineItem +##--------------------------------------------------------------------------## + +df_contractlineitem = df_contractitem.copy() + +df_contractlineitem.columns = ['Id', 'Name', 'Asset.PKey__c', 'ServiceContract.PKey__c'] + +df_contractlineitem.drop(['Id'], axis=1, inplace=True) +df_contractlineitem.drop(['Name'], axis=1, inplace=True) + +# Map Product2.Product_Code__c to df_contractlineitem based on Asset.PKey__c +df_contractlineitem = pd.merge( + df_contractlineitem, + merged_df_ib[['PKey__c', 'Product2.Product_Code__c']], + left_on='Asset.PKey__c', + right_on='PKey__c', + how='left' +) +df_contractlineitem = df_contractlineitem.drop('PKey__c', axis=1) + +# Merge df_contractlineitem with df_pricebookentry to get the PricebookEntry Id based on Product2.Product_Code__c +df_contractlineitem = pd.merge( + df_contractlineitem, + df_pricebookentry[['Product2.Product_Code__c', 'Id']], + left_on='Product2.Product_Code__c', + right_on='Product2.Product_Code__c', + how='left' +) +df_contractlineitem = df_contractlineitem.rename(columns={'Id': 'PricebookEntryId'}) + +print(df_contractlineitem) + ##--------------------------------------------------------------------------## ## MaintenancePlan, MaintenanceAsset ##--------------------------------------------------------------------------## -print(df_maintenanceplan.columns) df_maintenanceplan.columns = ['ServiceContract.PKey__c', 'StartDate', 'EndDate', 'AccountId', 'MaintenancePriceInclusionOfVAT__c', 'MaintenanceWindowEndDays', 'MaintenanceWindowStartDays', 'GenerationTimeframe', @@ -631,21 +682,77 @@ maintenance_worktype_id = df_worktype[df_worktype['Name'] == 'Planned Maintenanc # Add WorkTypeId to maintenance plan df_maintenanceplan['WorkTypeId'] = maintenance_worktype_id -print(df_maintenanceplan) +# Map InstalledBase__c from df_contractitem to df_maintenanceasset based on ServiceContract.PKey__c (Contract__c) +# If there are multiple InstalledBase__c for a Contract__c, join them as a semicolon-separated string + +# Group by Contract__c and aggregate InstalledBase__c as a semicolon-separated string +contractitem_grouped = df_contractitem.groupby('Contract__c')['InstalledBase__c'].apply(lambda x: ';'.join(x.dropna().unique())).reset_index() + +# Merge with df_maintenanceplan +df_maintenanceasset = df_maintenanceplan[['ServiceContract.PKey__c']].copy() +df_maintenanceasset = df_maintenanceasset.merge( + contractitem_grouped, + left_on='ServiceContract.PKey__c', + right_on='Contract__c', + how='left' +) + +df_maintenanceasset = df_maintenanceasset.rename(columns={'InstalledBase__c': 'InstalledBase__c'}) +df_maintenanceasset = df_maintenanceasset[['ServiceContract.PKey__c', 'InstalledBase__c']] + + + ##--------------------------------------------------------------------------## ## Skills ##--------------------------------------------------------------------------## +##--------------------------------------------------------------------------## +## Stocks +##--------------------------------------------------------------------------## +# Merge Resource__r.Employee__r.Name from df_resourceassignment to read_df_stockitem via Stock__c +read_df_stockitem = pd.merge( + read_df_stockitem, + df_resourceassignment[['Stock__c', 'ResourceNumberCalc__c']], + left_on='Stock__c', + right_on='Stock__c', + how='left' +) +##--------------------------------------------------------------------------## +## StocksOptimizationListItem +##--------------------------------------------------------------------------## + +src_file = '../1_extract_data/results/SCStockOptimizationListItem__c.csv' +dst_file = './data_for_external_systems/SCStockOptimizationListItem__c.csv' +shutil.copyfile(src_file, dst_file) ##--------------------------------------------------------------------------## ## Saving to CSV ##--------------------------------------------------------------------------## -# Write each DataFrame to a separate CSV file +# Create directories if they do not exist +output_dirs = [ + '../4_upsert_address_and_parent_location', + '../6_upsert_child_location', + '../9_upsert_assets', + '../12_upsert_associated_location', + '../13_insert_pricebook2_and_pricebookentries', + '../3_update_address_and_location_data_for_migration', + '../14_insert_servicecontracttemplates_dummies', + '../16_insert_servicecontract', + '../20_create_maintenance_plan', + '../21_insert_maintenance_asset', + '../19_create_contractlineitems', + './data_for_external_systems' +] + +for dir_path in output_dirs: + if dir_path and not os.path.exists(dir_path): + os.makedirs(dir_path, exist_ok=True) + address_df.to_csv('../4_upsert_address_and_parent_location/Address.csv', index=False) parent_df.to_csv('../4_upsert_address_and_parent_location/Location.csv', index=False) child_df.to_csv('../6_upsert_child_location/Location.csv', index=False) @@ -658,7 +765,10 @@ df_servicecontracttemplates.to_csv('../14_insert_servicecontracttemplates_dummie df_servicecontract.to_csv('../16_insert_servicecontract/ServiceContract_beforetransform.csv', index=False) df_assetwarranty_save.to_csv('../9_upsert_assets/AssetWarranty.csv', index=False) df_maintenance.to_csv('../9_upsert_assets/FSL_Asset_Maintenance_Information__c.csv', index=False) -df_maintenanceplan.to_csv('../19_create_maintenance_plan/MaintenancePlan_beforetransform.csv', index=False) +df_maintenanceplan.to_csv('../20_create_maintenance_plan/MaintenancePlan_beforetransform.csv', index=False) +df_maintenanceasset.to_csv('../21_insert_maintenance_asset/MaintenanceAsset_beforetransform.csv', index=False) +df_contractlineitem.to_csv('../19_create_contractlineitems/ContractLineItem.csv', index=False) +read_df_stockitem.to_csv('./data_for_external_systems/SAP_Stocks.csv', index=False) ## end mapping print('Data has been successfully transformed and saved to CSV files.') \ No newline at end of file diff --git a/sf_auth.py b/sf_auth.py index b0ee50f..bdca901 100644 --- a/sf_auth.py +++ b/sf_auth.py @@ -23,7 +23,7 @@ def get_credentials(context): # Load the .env file env_file = find_dotenv(".env") - load_dotenv(env_file) + load_dotenv(env_file, override=True) # Load all environment variables for key, value in os.environ.items(): @@ -70,7 +70,6 @@ def get_sf_connection(context): sf_params['security_token'] = credentials['SECURITY_TOKEN'] else: raise ValueError("Neither security token nor organization ID provided") - return Salesforce(**sf_params) else: