From 11a8b09dac0c74a17091f60ac72ee68dc75b3195 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rene=20Ka=C3=9Feb=C3=B6hmer?= Date: Wed, 23 Apr 2025 13:51:56 +0200 Subject: [PATCH] update address & location IoT --- .../FillLocationId.py | 0 .../command.txt | 0 .../AssociatedLocation_test.csv | 0 .../ValueMapping.csv | 0 .../command.txt | 0 .../export.json | 0 .../command.txt | 0 .../export.json | 0 .../{12_insert_ => 13_insert_}/command.txt | 0 .../{12_insert_ => 13_insert_}/export.json | 0 prepared_steps/1_extract_data/queries.json | 4 +- .../2_transform_via_script/TransformScript.py | 49 +++++++++++++------ .../command.txt | 0 .../export.json | 16 ++++++ .../command.txt | 0 .../export.json | 5 +- .../command.txt | 0 .../export.json | 0 .../command.txt | 0 .../export.json | 0 .../command.txt | 0 .../export.json | 0 .../ValueMapping.csv | 0 prepared_steps/8_upsert_assets/command.txt | 1 + .../export.json | 0 .../command.txt | 0 .../export.json | 0 27 files changed, 57 insertions(+), 18 deletions(-) rename prepared_steps/{9_fill_location_id_for_associated_location => 10_fill_location_id_for_associated_location}/FillLocationId.py (100%) rename prepared_steps/{9_fill_location_id_for_associated_location => 10_fill_location_id_for_associated_location}/command.txt (100%) rename prepared_steps/{10_upsert_associated_location => 11_upsert_associated_location}/AssociatedLocation_test.csv (100%) rename prepared_steps/{10_upsert_associated_location => 11_upsert_associated_location}/ValueMapping.csv (100%) rename prepared_steps/{10_upsert_associated_location => 11_upsert_associated_location}/command.txt (100%) rename prepared_steps/{10_upsert_associated_location => 11_upsert_associated_location}/export.json (100%) rename prepared_steps/{11_insert_pricebook2_and_pricebookentries => 12_insert_pricebook2_and_pricebookentries}/command.txt (100%) rename prepared_steps/{11_insert_pricebook2_and_pricebookentries => 12_insert_pricebook2_and_pricebookentries}/export.json (100%) rename prepared_steps/{12_insert_ => 13_insert_}/command.txt (100%) rename prepared_steps/{12_insert_ => 13_insert_}/export.json (100%) rename prepared_steps/{3_upsert_address_and_parent_location => 3_update_address_and_location_data_for_migration}/command.txt (100%) create mode 100644 prepared_steps/3_update_address_and_location_data_for_migration/export.json rename prepared_steps/{5_upsert_child_location => 4_upsert_address_and_parent_location}/command.txt (100%) rename prepared_steps/{3_upsert_address_and_parent_location => 4_upsert_address_and_parent_location}/export.json (80%) rename prepared_steps/{4_fill_visitoraddress_on_parentlocation => 5_fill_visitoraddress_on_parentlocation}/command.txt (100%) rename prepared_steps/{4_fill_visitoraddress_on_parentlocation => 5_fill_visitoraddress_on_parentlocation}/export.json (100%) rename prepared_steps/{7_upsert_assets => 6_upsert_child_location}/command.txt (100%) rename prepared_steps/{5_upsert_child_location => 6_upsert_child_location}/export.json (100%) rename prepared_steps/{6_build_lookup_child_to_parent_location => 7_build_lookup_child_to_parent_location}/command.txt (100%) rename prepared_steps/{6_build_lookup_child_to_parent_location => 7_build_lookup_child_to_parent_location}/export.json (100%) rename prepared_steps/{7_upsert_assets => 8_upsert_assets}/ValueMapping.csv (100%) create mode 100644 prepared_steps/8_upsert_assets/command.txt rename prepared_steps/{7_upsert_assets => 8_upsert_assets}/export.json (100%) rename prepared_steps/{8_update_personaccounts => 9_update_personaccounts}/command.txt (100%) rename prepared_steps/{8_update_personaccounts => 9_update_personaccounts}/export.json (100%) diff --git a/prepared_steps/9_fill_location_id_for_associated_location/FillLocationId.py b/prepared_steps/10_fill_location_id_for_associated_location/FillLocationId.py similarity index 100% rename from prepared_steps/9_fill_location_id_for_associated_location/FillLocationId.py rename to prepared_steps/10_fill_location_id_for_associated_location/FillLocationId.py diff --git a/prepared_steps/9_fill_location_id_for_associated_location/command.txt b/prepared_steps/10_fill_location_id_for_associated_location/command.txt similarity index 100% rename from prepared_steps/9_fill_location_id_for_associated_location/command.txt rename to prepared_steps/10_fill_location_id_for_associated_location/command.txt diff --git a/prepared_steps/10_upsert_associated_location/AssociatedLocation_test.csv b/prepared_steps/11_upsert_associated_location/AssociatedLocation_test.csv similarity index 100% rename from prepared_steps/10_upsert_associated_location/AssociatedLocation_test.csv rename to prepared_steps/11_upsert_associated_location/AssociatedLocation_test.csv diff --git a/prepared_steps/10_upsert_associated_location/ValueMapping.csv b/prepared_steps/11_upsert_associated_location/ValueMapping.csv similarity index 100% rename from prepared_steps/10_upsert_associated_location/ValueMapping.csv rename to prepared_steps/11_upsert_associated_location/ValueMapping.csv diff --git a/prepared_steps/10_upsert_associated_location/command.txt b/prepared_steps/11_upsert_associated_location/command.txt similarity index 100% rename from prepared_steps/10_upsert_associated_location/command.txt rename to prepared_steps/11_upsert_associated_location/command.txt diff --git a/prepared_steps/10_upsert_associated_location/export.json b/prepared_steps/11_upsert_associated_location/export.json similarity index 100% rename from prepared_steps/10_upsert_associated_location/export.json rename to prepared_steps/11_upsert_associated_location/export.json diff --git a/prepared_steps/11_insert_pricebook2_and_pricebookentries/command.txt b/prepared_steps/12_insert_pricebook2_and_pricebookentries/command.txt similarity index 100% rename from prepared_steps/11_insert_pricebook2_and_pricebookentries/command.txt rename to prepared_steps/12_insert_pricebook2_and_pricebookentries/command.txt diff --git a/prepared_steps/11_insert_pricebook2_and_pricebookentries/export.json b/prepared_steps/12_insert_pricebook2_and_pricebookentries/export.json similarity index 100% rename from prepared_steps/11_insert_pricebook2_and_pricebookentries/export.json rename to prepared_steps/12_insert_pricebook2_and_pricebookentries/export.json diff --git a/prepared_steps/12_insert_/command.txt b/prepared_steps/13_insert_/command.txt similarity index 100% rename from prepared_steps/12_insert_/command.txt rename to prepared_steps/13_insert_/command.txt diff --git a/prepared_steps/12_insert_/export.json b/prepared_steps/13_insert_/export.json similarity index 100% rename from prepared_steps/12_insert_/export.json rename to prepared_steps/13_insert_/export.json diff --git a/prepared_steps/1_extract_data/queries.json b/prepared_steps/1_extract_data/queries.json index d5411f6..b7a3852 100644 --- a/prepared_steps/1_extract_data/queries.json +++ b/prepared_steps/1_extract_data/queries.json @@ -2,10 +2,10 @@ [ { "sobject": "SCInstalledBaseLocation__c", - "query": "SELECT Id, City__c, Country__c, GeoY__c, GeoX__c, PostalCode__c, Street__c, Extension__c, HouseNo__c, FlatNo__c, Floor__c FROM SCInstalledBaseLocation__c WHERE Country__c = '{country}' AND Id = 'a1B1r0000099EsfEAE' limit 1" + "query": "SELECT Id, City__c, Country__c, GeoY__c, GeoX__c, PostalCode__c, Street__c, Extension__c, HouseNo__c, FlatNo__c, Floor__c FROM SCInstalledBaseLocation__c WHERE Country__c = '{country}' limit 3" },{ "sobject": "SCInstalledBase__c", - "query": "SELECT Id, Name, CommissioningDate__c,InstallationDate__c,ProductEnergy__c, ProductUnitClass__c,ArticleNo__c,SerialNo__c, SerialNoException__c, ProductUnitType__c, InstalledBaseLocation__c FROM SCInstalledBase__c WHERE Country__c = '{country}' AND InstalledBaseLocation__c = 'a1B1r0000099EsfEAE'" + "query": "SELECT Id, Name, CommissioningDate__c,InstallationDate__c,ProductEnergy__c, ProductUnitClass__c,ArticleNo__c,SerialNo__c, SerialNoException__c, ProductUnitType__c, InstalledBaseLocation__c FROM SCInstalledBase__c WHERE Country__c = '{country}' limit 3" },{ "sobject": "Asset", "query": "SELECT Id, Serialnumber FROM Asset WHERE Location.ParentLocation.Name LIKE '%{country}'" diff --git a/prepared_steps/2_transform_via_script/TransformScript.py b/prepared_steps/2_transform_via_script/TransformScript.py index 389747c..2ece1fe 100644 --- a/prepared_steps/2_transform_via_script/TransformScript.py +++ b/prepared_steps/2_transform_via_script/TransformScript.py @@ -13,13 +13,8 @@ read_df_ibr = pd.read_csv('../1_extract_data/results/SCInstalledBaseRole__c.csv' read_df_pricelist = pd.read_csv('../1_extract_data/results/SCPriceList__c.csv', header=0, keep_default_na=False, dtype=str) read_df_pricelistitem = pd.read_csv('../1_extract_data/results/SCPriceListItem__c.csv', header=0, keep_default_na=False, dtype=str) read_df_resourceassignment = pd.read_csv('../1_extract_data/results/SCResourceAssignment__c.csv', header=0, keep_default_na=False, dtype=str) - -for row in read_df.to_dict('records'): - try: - # Your processing logic here - pass - except KeyError as e: - print(f'KeyError: {e}') +read_df_address_iot = pd.read_csv('../1_extract_data/results/Address.csv', header=0, keep_default_na=False, dtype=str) +read_df_location_iot = pd.read_csv('../1_extract_data/results/ParentLocation.csv', header=0, keep_default_na=False, dtype=str) # Columns for reindexing reindex_columns = ['Id','City__c','Country__c','GeoY__c','GeoX__c','PostalCode__c','Street__c','Extension__c','HouseNo__c','FlatNo__c','Floor__c'] @@ -29,6 +24,8 @@ reindex_columns_ibr = ['Id', 'InstalledBaseLocation__c', 'Role__c', 'ValidFrom__ reindex_columns_pricelist = ['Id', 'Name', 'Brand__r.Name', 'Country__c'] reindex_columns_pricelistitem = ['Id', 'Article__r.Name', 'Article__r.EANCode__c', 'Price__c', 'PriceUnit__c', 'Pricelist__c', 'ValidFrom__c', 'ValidTo__c', 'Pricelist__r.Brand__r.Name', 'Pricelist__r.Country__c'] reindex_columns_resourceassignment = ['Id', 'ValidTo__c', 'ValidFrom__c', 'Country__c', 'City__c', 'District__c', 'Street__c', 'HouseNo__c', 'Extension__c', 'FlatNo__c', 'Floor__c', 'GeoY__c', 'GeoX__c', 'Resource__c', 'Stock__c'] +reindex_columns_address_iot = ['Id', 'Country', 'CountryCode', 'Street', 'City', 'ParentId', 'PostalCode'] +reindex_columns_location_iot = ['Id', 'Name'] # Reindex the columns to match the desired format df = read_df.reindex(reindex_columns, axis=1) @@ -38,6 +35,29 @@ df_ibr = read_df_ibr.reindex(reindex_columns_ibr, axis=1) df_pricelist = read_df_pricelist.reindex(reindex_columns_pricelist, axis=1) df_pricelistitem = read_df_pricelistitem.reindex(reindex_columns_pricelistitem, axis=1) df_resourceassignment = read_df_resourceassignment.reindex(reindex_columns_resourceassignment, axis=1) +df_address_iot = read_df_address_iot.reindex(reindex_columns_address_iot, axis=1) +df_location_iot = read_df_location_iot.reindex(reindex_columns_location_iot, axis=1) + +##--------------------------------------------------------------------------## +## Update for IoT Addresses and Locations +##--------------------------------------------------------------------------## + +df_address_iot['PKey__c'] = ( + df_address_iot['Street'].astype(str) + ', ' + + df_address_iot['PostalCode'].astype(str) + ' ' + + df_address_iot['City'].astype(str) + ', ' + + df_address_iot['CountryCode'].astype(str) +) + +merged_df_location_iot = pd.merge(df_location_iot, + df_address_iot[['Id', 'ParentId', 'PKey__c']], + left_on='Id', + right_on='ParentId', + how='left') + +merged_df_location_iot = merged_df_location_iot.drop('ParentId', axis=1) + +merged_df_location_iot.columns = ['Id', 'Name', 'VisitorAddressId', 'PKey__c'] ##--------------------------------------------------------------------------## ## Address and Location (Parent and Child) @@ -305,13 +325,14 @@ df_pricelistitem.columns = columns_pricebookentry # Write each DataFrame to a separate CSV file -address_df.to_csv('../3_upsert_address_and_parent_location/Address.csv', index=False) -parent_df.to_csv('../3_upsert_address_and_parent_location/Location.csv', index=False) -child_df.to_csv('../5_upsert_child_location/Location.csv', index=False) -merged_df_ib.to_csv('../7_upsert_assets/Asset.csv', index=False) -df_ibr.to_csv('../10_upsert_associated_location/AssociatedLocation.csv', index=False) -df_pricelist.to_csv('../11_insert_pricebook2_and_pricebookentries/Pricebook2.csv', index=False) -df_pricelistitem.to_csv('../11_insert_pricebook2_and_pricebookentries/PricebookEntry.csv', index=False) +address_df.to_csv('../4_upsert_address_and_parent_location/Address.csv', index=False) +parent_df.to_csv('../4_upsert_address_and_parent_location/Location.csv', index=False) +child_df.to_csv('../6_upsert_child_location/Location.csv', index=False) +merged_df_ib.to_csv('../8_upsert_assets/Asset.csv', index=False) +df_ibr.to_csv('../11_upsert_associated_location/AssociatedLocation.csv', index=False) +df_pricelist.to_csv('../12_insert_pricebook2_and_pricebookentries/Pricebook2.csv', index=False) +df_pricelistitem.to_csv('../12_insert_pricebook2_and_pricebookentries/PricebookEntry.csv', index=False) +merged_df_location_iot.to_csv('../3_update_address_and_location_data_for_migration/Location.csv', index=False) ## end mapping diff --git a/prepared_steps/3_upsert_address_and_parent_location/command.txt b/prepared_steps/3_update_address_and_location_data_for_migration/command.txt similarity index 100% rename from prepared_steps/3_upsert_address_and_parent_location/command.txt rename to prepared_steps/3_update_address_and_location_data_for_migration/command.txt diff --git a/prepared_steps/3_update_address_and_location_data_for_migration/export.json b/prepared_steps/3_update_address_and_location_data_for_migration/export.json new file mode 100644 index 0000000..e1c7878 --- /dev/null +++ b/prepared_steps/3_update_address_and_location_data_for_migration/export.json @@ -0,0 +1,16 @@ +{ + "allOrNone": true, + "objects": [ + { + "query": "SELECT Id FROM Address WHERE CountryCode = 'NL'", + "operation": "Readonly", + "master": false + },{ + "query": "SELECT Id,Name,VisitorAddressId,PKey__c FROM Location", + "operation": "Update", + "externalId": "Id", + "master": true, + "useSourceCSVFile": true + } + ] +} \ No newline at end of file diff --git a/prepared_steps/5_upsert_child_location/command.txt b/prepared_steps/4_upsert_address_and_parent_location/command.txt similarity index 100% rename from prepared_steps/5_upsert_child_location/command.txt rename to prepared_steps/4_upsert_address_and_parent_location/command.txt diff --git a/prepared_steps/3_upsert_address_and_parent_location/export.json b/prepared_steps/4_upsert_address_and_parent_location/export.json similarity index 80% rename from prepared_steps/3_upsert_address_and_parent_location/export.json rename to prepared_steps/4_upsert_address_and_parent_location/export.json index 6867d4d..c41eb64 100644 --- a/prepared_steps/3_upsert_address_and_parent_location/export.json +++ b/prepared_steps/4_upsert_address_and_parent_location/export.json @@ -5,11 +5,12 @@ "objects": [ { "query": "SELECT Name,DuplicateCheck__c,IsInventoryLocation,IsMobile,LocationType FROM Location", - "operation": "Insert", + "operation": "Upsert", "externalId": "Name" },{ "query": "SELECT City, CountryCode, PostalCode, Street, Latitude, Longitude, PKey__c, Country, Pkey__c, ParentId$Location FROM Address", - "operation": "Insert" + "operation": "Upsert", + "externalId": "PKey__c" } ] } \ No newline at end of file diff --git a/prepared_steps/4_fill_visitoraddress_on_parentlocation/command.txt b/prepared_steps/5_fill_visitoraddress_on_parentlocation/command.txt similarity index 100% rename from prepared_steps/4_fill_visitoraddress_on_parentlocation/command.txt rename to prepared_steps/5_fill_visitoraddress_on_parentlocation/command.txt diff --git a/prepared_steps/4_fill_visitoraddress_on_parentlocation/export.json b/prepared_steps/5_fill_visitoraddress_on_parentlocation/export.json similarity index 100% rename from prepared_steps/4_fill_visitoraddress_on_parentlocation/export.json rename to prepared_steps/5_fill_visitoraddress_on_parentlocation/export.json diff --git a/prepared_steps/7_upsert_assets/command.txt b/prepared_steps/6_upsert_child_location/command.txt similarity index 100% rename from prepared_steps/7_upsert_assets/command.txt rename to prepared_steps/6_upsert_child_location/command.txt diff --git a/prepared_steps/5_upsert_child_location/export.json b/prepared_steps/6_upsert_child_location/export.json similarity index 100% rename from prepared_steps/5_upsert_child_location/export.json rename to prepared_steps/6_upsert_child_location/export.json diff --git a/prepared_steps/6_build_lookup_child_to_parent_location/command.txt b/prepared_steps/7_build_lookup_child_to_parent_location/command.txt similarity index 100% rename from prepared_steps/6_build_lookup_child_to_parent_location/command.txt rename to prepared_steps/7_build_lookup_child_to_parent_location/command.txt diff --git a/prepared_steps/6_build_lookup_child_to_parent_location/export.json b/prepared_steps/7_build_lookup_child_to_parent_location/export.json similarity index 100% rename from prepared_steps/6_build_lookup_child_to_parent_location/export.json rename to prepared_steps/7_build_lookup_child_to_parent_location/export.json diff --git a/prepared_steps/7_upsert_assets/ValueMapping.csv b/prepared_steps/8_upsert_assets/ValueMapping.csv similarity index 100% rename from prepared_steps/7_upsert_assets/ValueMapping.csv rename to prepared_steps/8_upsert_assets/ValueMapping.csv diff --git a/prepared_steps/8_upsert_assets/command.txt b/prepared_steps/8_upsert_assets/command.txt new file mode 100644 index 0000000..4c22459 --- /dev/null +++ b/prepared_steps/8_upsert_assets/command.txt @@ -0,0 +1 @@ +sf sfdmu run --sourceusername csvfile --targetusername rene.kasseboehmer@vaillant.de.devrene \ No newline at end of file diff --git a/prepared_steps/7_upsert_assets/export.json b/prepared_steps/8_upsert_assets/export.json similarity index 100% rename from prepared_steps/7_upsert_assets/export.json rename to prepared_steps/8_upsert_assets/export.json diff --git a/prepared_steps/8_update_personaccounts/command.txt b/prepared_steps/9_update_personaccounts/command.txt similarity index 100% rename from prepared_steps/8_update_personaccounts/command.txt rename to prepared_steps/9_update_personaccounts/command.txt diff --git a/prepared_steps/8_update_personaccounts/export.json b/prepared_steps/9_update_personaccounts/export.json similarity index 100% rename from prepared_steps/8_update_personaccounts/export.json rename to prepared_steps/9_update_personaccounts/export.json