Rene/migration_via_sfdmu
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Script Added + Retrieving data start

Browse Source
This commit is contained in:
Rene Kaßeböhmer
2025-04-01 11:28:44 +02:00
parent b1b9d3db8a
commit 9b1103268c
17 changed files with 187 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
prepared_steps/1/Address.csv
View File

@ -1,2 +0,0 @@
Parent.Name,City,Country,PostalCode,Street,Latitude,Longitude,CountryCode,PKey__c
"Plantsoen de Pas 12, 6601 BK WIJCHEN, NL",WIJCHEN,Netherlands,6601 BK,Plantsoen de Pas 12,51.81388993,5.72578937,NL,"Plantsoen de Pas 12, 6601 BK WIJCHEN, NL"
1 Parent.Name City Country PostalCode Street Latitude Longitude CountryCode PKey__c
2 Plantsoen de Pas 12, 6601 BK WIJCHEN, NL WIJCHEN Netherlands 6601 BK Plantsoen de Pas 12 51.81388993 5.72578937 NL Plantsoen de Pas 12, 6601 BK WIJCHEN, NL

2
prepared_steps/1/Location.csv
View File

@ -1,2 +0,0 @@
Name,DuplicateCheck__c,IsInventoryLocation,IsMobile,LocationType
"Plantsoen de Pas 12, 6601 BK WIJCHEN, NL",false,false,false,Site
1 Name DuplicateCheck__c IsInventoryLocation IsMobile LocationType
2 Plantsoen de Pas 12, 6601 BK WIJCHEN, NL false false false Site

2
prepared_steps/1/command.txt
View File

@ -1 +1 @@
sf sfdmu run --sourceusername csvfile --targetusername rene.kasseboehmer@vaillant.de.devrene
sf sfdmu run --sourceusername rene.kasseboehmer@vaillant.de --targetusername csvfile

10
prepared_steps/1/export.json
View File

@ -1,15 +1,9 @@
{
"allOrNone": true,
"useSeparatedCSVFiles": true,
"excludeIdsFromCSVFiles": true,
"objects": [
{
"query": "SELECT Name,DuplicateCheck__c,IsInventoryLocation,IsMobile,LocationType FROM Location",
"operation": "Insert",
"externalId": "Name"
},{
"query": "SELECT City, Country, PostalCode, Street, Latitude, Longitude, CountryCode, Pkey__c, ParentId$Location FROM Address",
"operation": "Insert"
"query": "SELECT Id, City__c, Country__c, GeoY__c, GeoX__c, PostalCode__c, Street__c, Extension__c, HouseNo__c, FlatNo__c, Floor__c FROM SCInstalledBaseLocation__c WHERE Country__c = 'NL' limit 1",
"externalId": "Id"
}
]
}

117
prepared_steps/2/LocationScript.py Normal file
View File

@ -0,0 +1,117 @@
import pandas as pd
country_mapping = {
'NL': 'Netherlands'
}
# Read the input CSV file, assuming the second row is the header
read_df = pd.read_csv('../1/SCInstalledBaseLocation__c.csv', header=0, keep_default_na=False)
for row in read_df.to_dict('records'):
try:
# Your processing logic here
pass
except KeyError as e:
print(f'KeyError: {e}')
# Columns for reindexing
reindex_columns = ['City__c','Country__c','Extension__c','FlatNo__c','Floor__c','GeoX__c','GeoY__c','HouseNo__c','Id','PostalCode__c','Street__c']
# Reindex the columns to match the desired format
df = read_df.reindex(reindex_columns, axis=1)
df['Street'] = (
df['Street__c'].astype(str) + ' ' +
df['HouseNo__c'].astype(str) + ' ' +
df['Extension__c'].astype(str)
)
# Remove any trailing spaces that may result from missing values
df['Street'] = df['Street'].str.rstrip()
df['PKey__c'] = (
df['Street'].astype(str) + ', ' +
df['PostalCode__c'].astype(str) + ' ' +
df['City__c'].astype(str) + ', ' +
df['Country__c'].astype(str)
)
## 1. Address.csv
# Columns needed for Address table based on the input CSV structure
address_columns = ['City__c', 'Country__c',
'PostalCode__c', 'Street',
'GeoY__c', 'GeoX__c', 'PKey__c']
# Extract data for Address
address_df = df[address_columns].copy()
address_df['CountryCode'] = address_df['Country__c'].map(country_mapping)
# Now create the 'Name' column without any warnings
address_df['Parent.Name'] = address_df['PKey__c']
# Rename columns to match the desired format
address_df.columns = ['City', 'CountryCode', 'PostalCode', 'Street', 'Latitude', 'Longitude', 'PKey__c', 'Country', 'Parent.Name']
# Check for duplicates in Address table based on OldId, City, CountryCode, PostalCode, and Street
address_df = address_df.drop_duplicates(subset=['City', 'Country', 'PostalCode', 'Street'], keep='first')
## 2. Parent_Location.csv
parent_columns = ['City__c', 'Country__c',
'PostalCode__c', 'Street', 'PKey__c']
parent_df = df[parent_columns]
# Rename columns to match the desired format
parent_df.columns = ['City', 'Country',
'PostalCode', 'Street', 'Name']
parent_df['CountryCode'] = parent_df['Country'].map(country_mapping)
# Check for duplicates in Parent Location based on OldId and ConstructionEndDate
parent_df = parent_df.drop_duplicates(subset=['Name'], keep='first')
parent_df = parent_df.drop('Street', axis=1)
parent_df = parent_df.drop('PostalCode', axis=1)
parent_df = parent_df.drop('City', axis=1)
parent_df = parent_df.drop('CountryCode', axis=1)
parent_df = parent_df.drop('Country', axis=1)
parent_df['DuplicateCheck__c'] = 'false'
parent_df['IsInventoryLocation'] = 'false'
parent_df['IsMobile'] = 'false'
parent_df['LocationType'] = 'Site'
## 3. Child_Location.csv
child_columns = ['Extension__c', 'FlatNo__c', 'Floor__c', 'City__c', 'Country__c',
'PostalCode__c', 'Street', 'PKey__c']
# Modify child_df by explicitly creating a new DataFrame
child_df = df[child_columns].copy() # Add .copy() to create an explicit copy
# Now create the 'Name' column without any warnings
child_df['Name'] = (
child_df['Floor__c'].astype(str) + '-' +
child_df['FlatNo__c'].astype(str) + '-' +
child_df['Extension__c'].astype(str)
)
# Rename columns to match the desired format
child_df.columns = ['Extension', 'Flat', 'Floor', 'City', 'Country',
'PostalCode', 'Street', 'PKey__c', 'Name']
child_df = child_df.drop_duplicates(subset=['Extension', 'Flat', 'Floor','City', 'Country', 'PostalCode', 'Street'], keep='first')
child_df = child_df.drop('Country', axis=1)
child_df = child_df.drop('PostalCode', axis=1)
child_df = child_df.drop('City', axis=1)
child_df = child_df.drop('Street', axis=1)
child_df['DuplicateCheck__c'] = 'false'
child_df['IsInventoryLocation'] = 'false'
child_df['IsMobile'] = 'false'
child_df['LocationType'] = 'Site'
# Write each DataFrame to a separate CSV file
address_df.to_csv('../3/Address.csv', index=False)
parent_df.to_csv('../3/Location.csv', index=False)
child_df.to_csv('../5/Location.csv', index=False)
print('Data has been successfully split into Address.csv, Parent_Location.csv, and Child_Location.csv files with duplicate checks applied.')

1
prepared_steps/2/command.txt Normal file
View File

@ -0,0 +1 @@
python .\LocationScript.py

1
prepared_steps/2/command_dev.txt
View File

@ -1 +0,0 @@
sf sfdmu run --sourceusername rene.kasseboehmer@vaillant.de.devrene --targetusername rene.kasseboehmer@vaillant.de.devrene

4
prepared_steps/3/Location.csv
View File

@ -1,2 +1,2 @@
ParentLocation.Name,Extension,Flat,Floor,Name,DuplicateCheck__c,IsInventoryLocation,IsMobile,LocationType,PKey__c
"Plantsoen de Pas 12, 6601 BK WIJCHEN, NL",test,,,--,false,false,false,Site,"Plantsoen de Pas 12, 6601 BK WIJCHEN, NL"
Name,DuplicateCheck__c,IsInventoryLocation,IsMobile,LocationType
"Plantsoen de Pas 12, 6601 BK WIJCHEN, NL",false,false,false,Site

1 ParentLocation.Name Name Extension DuplicateCheck__c Flat IsInventoryLocation Floor IsMobile LocationType PKey__c
2 Plantsoen de Pas 12, 6601 BK WIJCHEN, NL -- Plantsoen de Pas 12, 6601 BK WIJCHEN, NL test false false false Site Plantsoen de Pas 12, 6601 BK WIJCHEN, NL

0
prepared_steps/3/command_dev.txt → prepared_steps/3/command.txt
View File

8
prepared_steps/3/export.json
View File

@ -1,11 +1,15 @@
{
"allOrNone": true,
"useSeparatedCSVFiles": true,
"excludeIdsFromCSVFiles": true,
"objects": [
{
"query": "SELECT Extension,Flat,Floor,Name,DuplicateCheck__c,IsInventoryLocation,IsMobile,LocationType,PKey__c FROM Location",
"query": "SELECT Name,DuplicateCheck__c,IsInventoryLocation,IsMobile,LocationType FROM Location",
"operation": "Insert",
"useSourceCSVFile": true
"externalId": "Name"
},{
"query": "SELECT City, CountryCode, PostalCode, Street, Latitude, Longitude, PKey__c, Country, Pkey__c, ParentId$Location FROM Address",
"operation": "Insert"
}
]
}

2
prepared_steps/4/command_dev.txt
View File

@ -1 +1 @@
sf sfdmu run --sourceusername csvfile --targetusername rene.kasseboehmer@vaillant.de.devrene
sf sfdmu run --sourceusername rene.kasseboehmer@vaillant.de.devrene --targetusername rene.kasseboehmer@vaillant.de.devrene

20
prepared_steps/4/export.json
View File

@ -2,32 +2,36 @@
"allOrNone": true,
"objects": [
{
"query": "SELECT Id, Name, Pkey__c, ParentLocationId FROM Location WHERE Pkey__c != null AND PKey__c like '%NL' AND ParentLocationId = null AND VisitorAddressId = null",
"query": "SELECT Id, Pkey__c FROM Address WHERE CountryCode = 'NL' AND PKey__c != null",
"operation": "Readonly"
},{
"query": "SELECT Id, Name, VisitorAdressId FROM Location WHERE VisitorAddressId = null",
"operation": "Update",
"beforeUpdateAddons": [
{
"module": "core:RecordsTransform",
"description": "Updates ParentLocationId with source Address.ParentId based on Pkey__c",
"description": "Updates VisitorAddressId with source Address.Id based on Pkey__c",
"args": {
"fields": [
{
"alias": "sourceAddressParentIdFromPkey",
"alias": "sourceAddressIdFromPkey",
"sourceObject": "Address",
"sourceField": "ParentId",
"lookupExpression": "source.PKey__c == target.Pkey__c",
"sourceField": "Id",
"lookupExpression": "source.PKey__c == target.Name",
"lookupSource": "source"
}
],
"transformations": [
{
"targetObject": "Location",
"targetField": "ParentLocationId",
"formula": "formula.sourceAddressParentIdFromPkey"
"targetField": "VisitorAddressId",
"formula": "formula.sourceAddressIdFromPkey"
}
]
}
}
]
],
"externalId": "Name"
}
]
}

1
prepared_steps/5/command_dev.txt Normal file
View File

@ -0,0 +1 @@
sf sfdmu run --sourceusername csvfile --targetusername rene.kasseboehmer@vaillant.de.devrene

11
prepared_steps/5/export.json Normal file
View File

@ -0,0 +1,11 @@
{
"allOrNone": true,
"excludeIdsFromCSVFiles": true,
"objects": [
{
"query": "SELECT Extension,Flat,Floor,Name,DuplicateCheck__c,IsInventoryLocation,IsMobile,LocationType,PKey__c FROM Location",
"operation": "Insert",
"useSourceCSVFile": true
}
]
}

1
prepared_steps/6/command_dev.txt Normal file
View File

@ -0,0 +1 @@
sf sfdmu run --sourceusername csvfile --targetusername rene.kasseboehmer@vaillant.de.devrene

20
prepared_steps/2/export.json → prepared_steps/6/export.json
View File

@ -2,36 +2,32 @@
"allOrNone": true,
"objects": [
{
"query": "SELECT Id, Pkey__c FROM Address WHERE CountryCode = 'NL' AND PKey__c != null",
"operation": "Readonly"
},{
"query": "SELECT Id, Name, VisitorAdressId FROM Location WHERE VisitorAddressId = null",
"query": "SELECT Id, Name, Pkey__c, ParentLocationId FROM Location WHERE Pkey__c != null AND PKey__c like '%NL' AND ParentLocationId = null AND VisitorAddressId = null",
"operation": "Update",
"beforeUpdateAddons": [
{
"module": "core:RecordsTransform",
"description": "Updates VisitorAddressId with source Address.Id based on Pkey__c",
"description": "Updates ParentLocationId with source Address.ParentId based on Pkey__c",
"args": {
"fields": [
{
"alias": "sourceAddressIdFromPkey",
"alias": "sourceAddressParentIdFromPkey",
"sourceObject": "Address",
"sourceField": "Id",
"lookupExpression": "source.PKey__c == target.Name",
"sourceField": "ParentId",
"lookupExpression": "source.PKey__c == target.Pkey__c",
"lookupSource": "source"
}
],
"transformations": [
{
"targetObject": "Location",
"targetField": "VisitorAddressId",
"formula": "formula.sourceAddressIdFromPkey"
"targetField": "ParentLocationId",
"formula": "formula.sourceAddressParentIdFromPkey"
}
]
}
}
],
"externalId": "Name"
]
}
]
}