diff --git a/prepared_steps/10_upsert_associated_location/ValueMapping.csv b/prepared_steps/10_upsert_associated_location/ValueMapping.csv new file mode 100644 index 0000000..6d2b5a0 --- /dev/null +++ b/prepared_steps/10_upsert_associated_location/ValueMapping.csv @@ -0,0 +1,5 @@ +ObjectName,FieldName,RawValue,Value +AssociatedLocation,Type,Installer (installation),Installer +AssociatedLocation,Type,Installer (first ignition),Installer +AssociatedLocation,Type,Installer (service),Installer +AssociatedLocation,Type,Installer (Timex),Installer \ No newline at end of file diff --git a/prepared_steps/10_upsert_associated_location/command.txt b/prepared_steps/10_upsert_associated_location/command.txt new file mode 100644 index 0000000..0389686 --- /dev/null +++ b/prepared_steps/10_upsert_associated_location/command.txt @@ -0,0 +1 @@ +sf sfdmu run --sourceusername rene.kasseboehmer@vaillant.de.devrene --targetusername rene.kasseboehmer@vaillant.de.devrene \ No newline at end of file diff --git a/prepared_steps/10_upsert_associated_location/export.json b/prepared_steps/10_upsert_associated_location/export.json new file mode 100644 index 0000000..40dfcd4 --- /dev/null +++ b/prepared_steps/10_upsert_associated_location/export.json @@ -0,0 +1,16 @@ +{ + "allOrNone": true, + "excludeIdsFromCSVFiles": true, + + "objects": [ +{ + "query": "SELECT Type,ActiveFrom,ActiveTo,ParentRecordId$Account,PKey__c, LocationId FROM AssociatedLocation", + "operation": "Insert", + "externalId": "PKey__c", + "useSourceCSVFile": true, + "master": true, + "useValuesMapping": true, + "excludedFields": ["PKey__c"] + } + ] +} \ No newline at end of file diff --git a/prepared_steps/1_extract_data/queries.json b/prepared_steps/1_extract_data/queries.json index dfbcb06..d893960 100644 --- a/prepared_steps/1_extract_data/queries.json +++ b/prepared_steps/1_extract_data/queries.json @@ -2,10 +2,10 @@ [ { "sobject": "SCInstalledBaseLocation__c", - "query": "SELECT Id, City__c, Country__c, GeoY__c, GeoX__c, PostalCode__c, Street__c, Extension__c, HouseNo__c, FlatNo__c, Floor__c FROM SCInstalledBaseLocation__c WHERE Country__c = 'NL' limit 1" + "query": "SELECT Id, City__c, Country__c, GeoY__c, GeoX__c, PostalCode__c, Street__c, Extension__c, HouseNo__c, FlatNo__c, Floor__c FROM SCInstalledBaseLocation__c WHERE Country__c = 'NL' AND Id = 'a1B1r0000099EsfEAE' limit 1" },{ "sobject": "SCInstalledBase__c", - "query": "SELECT Id, Name, CommissioningDate__c,InstallationDate__c,ProductEnergy__c, ProductUnitClass__c,ArticleNo__c,SerialNo__c, SerialNoException__c, ProductUnitType__c, InstalledBaseLocation__c FROM SCInstalledBase__c WHERE Country__c = 'NL' limit 1" + "query": "SELECT Id, Name, CommissioningDate__c,InstallationDate__c,ProductEnergy__c, ProductUnitClass__c,ArticleNo__c,SerialNo__c, SerialNoException__c, ProductUnitType__c, InstalledBaseLocation__c FROM SCInstalledBase__c WHERE Country__c = 'NL' AND InstalledBaseLocation__c = 'a1B1r0000099EsfEAE'" },{ "sobject": "Asset", "query": "SELECT Id, Serialnumber FROM Asset WHERE Location.ParentLocation.Name LIKE '%NL'" @@ -21,6 +21,9 @@ },{ "sobject": "Product2", "query": "SELECT Id, Main_Product_Group__c, Family, MaterialType__c, Name, Product_Code__c, ProductCode, EAN_Product_Code__c FROM Product2" + },{ + "sobject": "SCInstalledBaseRole__c", + "query": "SELECT Id, InstalledBaseLocation__c, Role__c, ValidFrom__c, ValidTo__c, Account__c FROM SCInstalledBaseRole__c WHERE InstalledBaseLocation__r.Country__c = 'NL' AND InstalledBaseLocation__c = 'a1B1r0000099EsfEAE'" } ] } \ No newline at end of file diff --git a/prepared_steps/1_extract_data/results/SCInstalledBaseRole__c.csv b/prepared_steps/1_extract_data/results/SCInstalledBaseRole__c.csv new file mode 100644 index 0000000..2b0780f --- /dev/null +++ b/prepared_steps/1_extract_data/results/SCInstalledBaseRole__c.csv @@ -0,0 +1,11 @@ +"Id","InstalledBaseLocation__c","Role__c","ValidFrom__c","ValidTo__c","Account__c" +"a0a1r00000JtNV5AAN","a1B1r0000099EsfEAE","Installer (installation)","2018-03-27","","0012000000eCN6TAAW" +"a0a1r00000JtNV6AAN","a1B1r0000099EsfEAE","Owner","2018-04-06","","0011r00001mmKQ8AAM" +"a0a1r00000JtNVUAA3","a1B1r0000099EsfEAE","Installer (installation)","2018-03-27","","0012000000eCN6TAAW" +"a0a1r00000JtNVVAA3","a1B1r0000099EsfEAE","Owner","2018-04-06","","0011r00001mmKQ8AAM" +"a0a1r00000JtNVtAAN","a1B1r0000099EsfEAE","Installer (installation)","2018-03-27","","0012000000eCN6TAAW" +"a0a1r00000JtNVuAAN","a1B1r0000099EsfEAE","Owner","2018-04-06","","0011r00001mmKQ8AAM" +"a0a1r00000KpzPNAAZ","a1B1r0000099EsfEAE","Installer (installation)","2018-03-27","","0012000000eCN6TAAW" +"a0a1r00000KpzPOAAZ","a1B1r0000099EsfEAE","Owner","2018-03-28","","0011r00001mmKQ8AAM" +"a0a1r00000KpzQQAAZ","a1B1r0000099EsfEAE","Installer (installation)","2018-03-27","","0012000000eCN6TAAW" +"a0a1r00000KpzQRAAZ","a1B1r0000099EsfEAE","Owner","2018-03-28","","0011r00001mmKQ8AAM" diff --git a/prepared_steps/2_transform_via_script/LocationScript.py b/prepared_steps/2_transform_via_script/LocationScript.py index 879a9c8..4bc8373 100644 --- a/prepared_steps/2_transform_via_script/LocationScript.py +++ b/prepared_steps/2_transform_via_script/LocationScript.py @@ -9,6 +9,7 @@ country_mapping = { read_df = pd.read_csv('../1_extract_data/results/SCInstalledBaseLocation__c.csv', header=0, keep_default_na=False, dtype=str) read_df_ib = pd.read_csv('../1_extract_data/results/SCInstalledBase__c.csv', header=0, keep_default_na=False, dtype=str) read_df_product2 = pd.read_csv('../1_extract_data/results/Product2.csv', header=0, keep_default_na=False, dtype=str) +read_df_ibr = pd.read_csv('../1_extract_data/results/SCInstalledBaseRole__c.csv', header=0, keep_default_na=False, dtype=str) for row in read_df.to_dict('records'): try: @@ -28,11 +29,14 @@ reindex_columns_ib = ['Id','Name','CommissioningDate__c','InstallationDate__c',' #"Id","Main_Product_Group__c","Family","MaterialType__c","Name","Product_Code__c","ProductCode","EAN_Product_Code__c" reindex_columns_product2 = ['Id','Main_Product_Group__c','Family','MaterialType__c','Name','Product_Code__c','ProductCode','EAN_Product_Code__c'] #reindex_columns_product2 = ['EAN_Product_Code__c','Family','Id','Main_Product_Group__c','MaterialType__c','Name','Product_Code__c','ProductCode'] +#"Id","InstalledBaseLocation__c","Role__c","ValidFrom__c","ValidTo__c","Account__c" +reindex_columns_ibr = ['Id', 'InstalledBaseLocation__c', 'Role__c', 'ValidFrom__c', 'ValidTo__c', 'Account__c'] # Reindex the columns to match the desired format df = read_df.reindex(reindex_columns, axis=1) df_ib = read_df_ib.reindex(reindex_columns_ib, axis=1) df_product2 = read_df_product2.reindex(reindex_columns_product2, axis=1) +df_ibr = read_df_ibr.reindex(reindex_columns_ibr, axis=1) df['Street'] = ( df['Street__c'].astype(str) + ' ' + @@ -117,7 +121,7 @@ parent_df['IsMobile'] = 'false' parent_df['LocationType'] = 'Site' ## 3. Child_Location.csv -child_columns = ['Extension__c', 'FlatNo__c', 'Floor__c', 'City__c', 'Country__c', +child_columns = ['Id', 'Extension__c', 'FlatNo__c', 'Floor__c', 'City__c', 'Country__c', 'PostalCode__c', 'Street', 'PKey__c'] # Modify child_df by explicitly creating a new DataFrame child_df = df[child_columns].copy() # Add .copy() to create an explicit copy @@ -142,7 +146,7 @@ child_df['ExternalReference'] = ( ) # Rename columns to match the desired format -child_df.columns = ['Extension__c', 'Flat__c', 'Floor__c', 'City', 'Country', +child_df.columns = ['Id', 'Extension__c', 'Flat__c', 'Floor__c', 'City', 'Country', 'PostalCode', 'Street', 'PKey__c', 'Name', 'ExternalReference'] child_df = child_df.drop_duplicates(subset=['Extension__c', 'Flat__c', 'Floor__c','City', 'Country', 'PostalCode', 'Street'], keep='first') @@ -180,11 +184,33 @@ merged_df_ib = merged_df_ib.drop('Product_Code__c', axis=1) merged_df_ib = merged_df_ib.drop_duplicates(subset=['Name','SerialNumber'], keep='first') +# Merging LPG into one Value for Assets +merged_df_ib = merged_df_ib.replace({'Kind_of_Energy__c': {'4': '3', '5': '3'}}) + +## 5. SCInstalledBaseRole__c.csv + +df_ibr = pd.merge(df_ibr, + child_df[['Id', 'ExternalReference']], + left_on='InstalledBaseLocation__c', + right_on='Id', + how='left') + +df_ibr = df_ibr.drop_duplicates(subset=['InstalledBaseLocation__c', 'Role__c', 'Account__c'], keep='first') + +df_ibr = df_ibr.drop('Id_x', axis=1) +df_ibr = df_ibr.drop('Id_y', axis=1) +df_ibr = df_ibr.drop('InstalledBaseLocation__c', axis=1) + +print(df_ibr.columns) + +df_ibr.columns = ['Type', 'ActiveFrom', 'ActiveTo', 'ParentRecordId', 'Location.ExternalReference'] + #remove kind_of_energy__c and kind_of_installation if field dependency to main product group is not correct # Create the mapping dictionary kind_of_energy_mapping = { '1': ['A2', 'A1', 'B2', 'B1', 'E1', '14', 'E3', '17'], '2': ['A2', 'A1', 'B2', 'B1', 'E1', '14', 'E3', '17'], + '3': ['A2', 'A1', 'B2', 'B1', 'E1', '14', 'E3', '17'], 'G': ['A2', 'A1', 'B2', 'B1', 'E1', 'E3', '17'], '6': ['B3', '11', '13', '14', '17'], '8': ['B4'], @@ -227,6 +253,7 @@ address_df.to_csv('../3_upsert_address_and_parent_location/Address.csv', index=F parent_df.to_csv('../3_upsert_address_and_parent_location/Location.csv', index=False) child_df.to_csv('../5_upsert_child_location/Location.csv', index=False) merged_df_ib.to_csv('../7_upsert_assets/Asset.csv', index=False) +df_ibr.to_csv('../9_upsert_associated_location/AssociatedLocation.csv', index=False) ## end mapping diff --git a/prepared_steps/7_upsert_assets/ValueMapping.csv b/prepared_steps/7_upsert_assets/ValueMapping.csv index f013545..8ccecf0 100644 --- a/prepared_steps/7_upsert_assets/ValueMapping.csv +++ b/prepared_steps/7_upsert_assets/ValueMapping.csv @@ -1,3 +1,8 @@ ObjectName,FieldName,RawValue,Value Asset,Kind_of_Energy__c,4,3 -Asset,Kind_of_Energy__c,5,3 \ No newline at end of file +Asset,Kind_of_Energy__c,5,3 +Asset,Serialnumber_Exception__c,350001,Missing +Asset,Serialnumber_Exception__c,350002,Not Readable +Asset,Serialnumber_Exception__c,350003,Not Readable +Asset,Serialnumber_Exception__c,350004,Rejected by the system +Asset,Serialnumber_Exception__c,350005,Missing \ No newline at end of file diff --git a/prepared_steps/7_upsert_assets/export.json b/prepared_steps/7_upsert_assets/export.json index 60d158c..3905330 100644 --- a/prepared_steps/7_upsert_assets/export.json +++ b/prepared_steps/7_upsert_assets/export.json @@ -15,7 +15,8 @@ },{ "query": "SELECT Product2Id,InstallDate,Name,Kind_of_Energy__c,Main_Product_Group__c,SerialNumber,Serialnumber_Exception__c,LocationId FROM Asset", "operation": "Insert", - "master": true + "master": true, + "useValuesMapping": true } ] } \ No newline at end of file diff --git a/prepared_steps/9_fill_location_id_for_associated_location/FillLocationId.py b/prepared_steps/9_fill_location_id_for_associated_location/FillLocationId.py new file mode 100644 index 0000000..877aa99 --- /dev/null +++ b/prepared_steps/9_fill_location_id_for_associated_location/FillLocationId.py @@ -0,0 +1,31 @@ +import pandas as pd + +read_df_al = pd.read_csv('../10_upsert_associated_location/AssociatedLocation.csv', header=0, keep_default_na=False, dtype=str) +read_df_l = pd.read_csv('../5_upsert_child_location/target/Location_insert_target.csv', header=0, keep_default_na=False, dtype=str) + +#Type,ActiveFrom,ActiveTo,ParentRecordId,PKey__c +reindex_columns_al = ['Type','ActiveFrom','ActiveTo','ParentRecordId','PKey__c'] +#Errors,ExternalReference,Id,IsInventoryLocation,IsMobile,LocationType,Name,PKey__c +reindex_columns_l = ['Errors','ExternalReference','Id','IsInventoryLocation','IsMobile','LocationType','Name','PKey__c'] + +# Reindex the columns to match the desired format +df_al = read_df_al.reindex(reindex_columns_al, axis=1) +df_l = read_df_l.reindex(reindex_columns_l, axis=1) + +# Merge df_al with df_l including Id abse on PKey__c +merged_df_al = pd.merge(df_al, + df_l[['Id', 'ExternalReference']], + left_on='PKey__c', + right_on='ExternalReference', + how='left') + +#drop External Reference +merged_df_al = merged_df_al.drop('ExternalReference', axis=1) + +#Rename columns +merged_df_al.columns = ['Type','ActiveFrom','ActiveTo','ParentRecordId','PKey__c','LocationId'] + + +#safe csv +merged_df_al.to_csv('../10_upsert_associated_location/AssociatedLocation.csv', index=False) + diff --git a/prepared_steps/9_fill_location_id_for_associated_location/command.txt b/prepared_steps/9_fill_location_id_for_associated_location/command.txt new file mode 100644 index 0000000..e0ee6e0 --- /dev/null +++ b/prepared_steps/9_fill_location_id_for_associated_location/command.txt @@ -0,0 +1 @@ +python .\FillLocationId.py \ No newline at end of file