From 9b1103268cf3b7da4b12434739451d5cccadf1c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rene=20Ka=C3=9Feb=C3=B6hmer?= Date: Tue, 1 Apr 2025 11:28:44 +0200 Subject: [PATCH] Script Added + Retrieving data start --- README | 24 ++++ prepared_steps/1/Address.csv | 2 - prepared_steps/1/Location.csv | 2 - prepared_steps/1/command.txt | 2 +- prepared_steps/1/export.json | 10 +- prepared_steps/2/LocationScript.py | 117 ++++++++++++++++++ prepared_steps/2/command.txt | 1 + prepared_steps/2/command_dev.txt | 1 - prepared_steps/3/Location.csv | 4 +- .../3/{command_dev.txt => command.txt} | 0 prepared_steps/3/export.json | 8 +- prepared_steps/4/command_dev.txt | 2 +- prepared_steps/4/export.json | 20 +-- prepared_steps/5/command_dev.txt | 1 + prepared_steps/5/export.json | 11 ++ prepared_steps/6/command_dev.txt | 1 + prepared_steps/{2 => 6}/export.json | 20 ++- 17 files changed, 187 insertions(+), 39 deletions(-) create mode 100644 README delete mode 100644 prepared_steps/1/Address.csv delete mode 100644 prepared_steps/1/Location.csv create mode 100644 prepared_steps/2/LocationScript.py create mode 100644 prepared_steps/2/command.txt delete mode 100644 prepared_steps/2/command_dev.txt rename prepared_steps/3/{command_dev.txt => command.txt} (100%) create mode 100644 prepared_steps/5/command_dev.txt create mode 100644 prepared_steps/5/export.json create mode 100644 prepared_steps/6/command_dev.txt rename prepared_steps/{2 => 6}/export.json (57%) diff --git a/README b/README new file mode 100644 index 0000000..3ba2f17 --- /dev/null +++ b/README @@ -0,0 +1,24 @@ +# FSL Migration from Legacy with SFDMU & Python (Pandas) + +This git is representing a migration from Legacy GMS solution to FSL Standard + +## Installation + +Install Python [python](https://www.python.org/) to install python 3.13+. +After installation open a terminal/powershell to install pandas globally + + +```bash +python -m pip install pandas +``` + +## Usage + +open a terminal/powershell +cd into the root folder of the project +Go into each "prepared_step" folder an execute the command in command.txt + + +## License + +2025 Vaillant Group Business Services GmbH \ No newline at end of file diff --git a/prepared_steps/1/Address.csv b/prepared_steps/1/Address.csv deleted file mode 100644 index ecd01cf..0000000 --- a/prepared_steps/1/Address.csv +++ /dev/null @@ -1,2 +0,0 @@ -Parent.Name,City,Country,PostalCode,Street,Latitude,Longitude,CountryCode,PKey__c -"Plantsoen de Pas 12, 6601 BK WIJCHEN, NL",WIJCHEN,Netherlands,6601 BK,Plantsoen de Pas 12,51.81388993,5.72578937,NL,"Plantsoen de Pas 12, 6601 BK WIJCHEN, NL" \ No newline at end of file diff --git a/prepared_steps/1/Location.csv b/prepared_steps/1/Location.csv deleted file mode 100644 index 2b9fbb7..0000000 --- a/prepared_steps/1/Location.csv +++ /dev/null @@ -1,2 +0,0 @@ -Name,DuplicateCheck__c,IsInventoryLocation,IsMobile,LocationType -"Plantsoen de Pas 12, 6601 BK WIJCHEN, NL",false,false,false,Site \ No newline at end of file diff --git a/prepared_steps/1/command.txt b/prepared_steps/1/command.txt index 4c22459..a28faa4 100644 --- a/prepared_steps/1/command.txt +++ b/prepared_steps/1/command.txt @@ -1 +1 @@ -sf sfdmu run --sourceusername csvfile --targetusername rene.kasseboehmer@vaillant.de.devrene \ No newline at end of file +sf sfdmu run --sourceusername rene.kasseboehmer@vaillant.de --targetusername csvfile \ No newline at end of file diff --git a/prepared_steps/1/export.json b/prepared_steps/1/export.json index 306dabd..91e6a02 100644 --- a/prepared_steps/1/export.json +++ b/prepared_steps/1/export.json @@ -1,15 +1,9 @@ { "allOrNone": true, - "useSeparatedCSVFiles": true, - "excludeIdsFromCSVFiles": true, "objects": [ { - "query": "SELECT Name,DuplicateCheck__c,IsInventoryLocation,IsMobile,LocationType FROM Location", - "operation": "Insert", - "externalId": "Name" - },{ - "query": "SELECT City, Country, PostalCode, Street, Latitude, Longitude, CountryCode, Pkey__c, ParentId$Location FROM Address", - "operation": "Insert" + "query": "SELECT Id, City__c, Country__c, GeoY__c, GeoX__c, PostalCode__c, Street__c, Extension__c, HouseNo__c, FlatNo__c, Floor__c FROM SCInstalledBaseLocation__c WHERE Country__c = 'NL' limit 1", + "externalId": "Id" } ] } \ No newline at end of file diff --git a/prepared_steps/2/LocationScript.py b/prepared_steps/2/LocationScript.py new file mode 100644 index 0000000..8c34a8e --- /dev/null +++ b/prepared_steps/2/LocationScript.py @@ -0,0 +1,117 @@ +import pandas as pd + +country_mapping = { + 'NL': 'Netherlands' + } + +# Read the input CSV file, assuming the second row is the header +read_df = pd.read_csv('../1/SCInstalledBaseLocation__c.csv', header=0, keep_default_na=False) +for row in read_df.to_dict('records'): + try: + # Your processing logic here + pass + except KeyError as e: + print(f'KeyError: {e}') + +# Columns for reindexing +reindex_columns = ['City__c','Country__c','Extension__c','FlatNo__c','Floor__c','GeoX__c','GeoY__c','HouseNo__c','Id','PostalCode__c','Street__c'] + +# Reindex the columns to match the desired format +df = read_df.reindex(reindex_columns, axis=1) + +df['Street'] = ( + df['Street__c'].astype(str) + ' ' + + df['HouseNo__c'].astype(str) + ' ' + + df['Extension__c'].astype(str) +) + +# Remove any trailing spaces that may result from missing values +df['Street'] = df['Street'].str.rstrip() + +df['PKey__c'] = ( + df['Street'].astype(str) + ', ' + + df['PostalCode__c'].astype(str) + ' ' + + df['City__c'].astype(str) + ', ' + + df['Country__c'].astype(str) +) + +## 1. Address.csv +# Columns needed for Address table based on the input CSV structure +address_columns = ['City__c', 'Country__c', + 'PostalCode__c', 'Street', + 'GeoY__c', 'GeoX__c', 'PKey__c'] + +# Extract data for Address +address_df = df[address_columns].copy() + +address_df['CountryCode'] = address_df['Country__c'].map(country_mapping) + +# Now create the 'Name' column without any warnings +address_df['Parent.Name'] = address_df['PKey__c'] + +# Rename columns to match the desired format +address_df.columns = ['City', 'CountryCode', 'PostalCode', 'Street', 'Latitude', 'Longitude', 'PKey__c', 'Country', 'Parent.Name'] + +# Check for duplicates in Address table based on OldId, City, CountryCode, PostalCode, and Street +address_df = address_df.drop_duplicates(subset=['City', 'Country', 'PostalCode', 'Street'], keep='first') + +## 2. Parent_Location.csv +parent_columns = ['City__c', 'Country__c', + 'PostalCode__c', 'Street', 'PKey__c'] +parent_df = df[parent_columns] +# Rename columns to match the desired format +parent_df.columns = ['City', 'Country', + 'PostalCode', 'Street', 'Name'] + +parent_df['CountryCode'] = parent_df['Country'].map(country_mapping) + +# Check for duplicates in Parent Location based on OldId and ConstructionEndDate +parent_df = parent_df.drop_duplicates(subset=['Name'], keep='first') + +parent_df = parent_df.drop('Street', axis=1) +parent_df = parent_df.drop('PostalCode', axis=1) +parent_df = parent_df.drop('City', axis=1) +parent_df = parent_df.drop('CountryCode', axis=1) +parent_df = parent_df.drop('Country', axis=1) + +parent_df['DuplicateCheck__c'] = 'false' +parent_df['IsInventoryLocation'] = 'false' +parent_df['IsMobile'] = 'false' +parent_df['LocationType'] = 'Site' + +## 3. Child_Location.csv +child_columns = ['Extension__c', 'FlatNo__c', 'Floor__c', 'City__c', 'Country__c', + 'PostalCode__c', 'Street', 'PKey__c'] +# Modify child_df by explicitly creating a new DataFrame +child_df = df[child_columns].copy() # Add .copy() to create an explicit copy + +# Now create the 'Name' column without any warnings +child_df['Name'] = ( + child_df['Floor__c'].astype(str) + '-' + + child_df['FlatNo__c'].astype(str) + '-' + + child_df['Extension__c'].astype(str) +) + +# Rename columns to match the desired format +child_df.columns = ['Extension', 'Flat', 'Floor', 'City', 'Country', + 'PostalCode', 'Street', 'PKey__c', 'Name'] + +child_df = child_df.drop_duplicates(subset=['Extension', 'Flat', 'Floor','City', 'Country', 'PostalCode', 'Street'], keep='first') + +child_df = child_df.drop('Country', axis=1) +child_df = child_df.drop('PostalCode', axis=1) +child_df = child_df.drop('City', axis=1) +child_df = child_df.drop('Street', axis=1) + +child_df['DuplicateCheck__c'] = 'false' +child_df['IsInventoryLocation'] = 'false' +child_df['IsMobile'] = 'false' +child_df['LocationType'] = 'Site' + + +# Write each DataFrame to a separate CSV file +address_df.to_csv('../3/Address.csv', index=False) +parent_df.to_csv('../3/Location.csv', index=False) +child_df.to_csv('../5/Location.csv', index=False) + +print('Data has been successfully split into Address.csv, Parent_Location.csv, and Child_Location.csv files with duplicate checks applied.') \ No newline at end of file diff --git a/prepared_steps/2/command.txt b/prepared_steps/2/command.txt new file mode 100644 index 0000000..aab9be0 --- /dev/null +++ b/prepared_steps/2/command.txt @@ -0,0 +1 @@ +python .\LocationScript.py \ No newline at end of file diff --git a/prepared_steps/2/command_dev.txt b/prepared_steps/2/command_dev.txt deleted file mode 100644 index 0389686..0000000 --- a/prepared_steps/2/command_dev.txt +++ /dev/null @@ -1 +0,0 @@ -sf sfdmu run --sourceusername rene.kasseboehmer@vaillant.de.devrene --targetusername rene.kasseboehmer@vaillant.de.devrene \ No newline at end of file diff --git a/prepared_steps/3/Location.csv b/prepared_steps/3/Location.csv index 2fd7634..69e3156 100644 --- a/prepared_steps/3/Location.csv +++ b/prepared_steps/3/Location.csv @@ -1,2 +1,2 @@ -ParentLocation.Name,Extension,Flat,Floor,Name,DuplicateCheck__c,IsInventoryLocation,IsMobile,LocationType,PKey__c -"Plantsoen de Pas 12, 6601 BK WIJCHEN, NL",test,,,--,false,false,false,Site,"Plantsoen de Pas 12, 6601 BK WIJCHEN, NL" \ No newline at end of file +Name,DuplicateCheck__c,IsInventoryLocation,IsMobile,LocationType +"Plantsoen de Pas 12, 6601 BK WIJCHEN, NL",false,false,false,Site diff --git a/prepared_steps/3/command_dev.txt b/prepared_steps/3/command.txt similarity index 100% rename from prepared_steps/3/command_dev.txt rename to prepared_steps/3/command.txt diff --git a/prepared_steps/3/export.json b/prepared_steps/3/export.json index d8a4ac6..6867d4d 100644 --- a/prepared_steps/3/export.json +++ b/prepared_steps/3/export.json @@ -1,11 +1,15 @@ { "allOrNone": true, + "useSeparatedCSVFiles": true, "excludeIdsFromCSVFiles": true, "objects": [ { - "query": "SELECT Extension,Flat,Floor,Name,DuplicateCheck__c,IsInventoryLocation,IsMobile,LocationType,PKey__c FROM Location", + "query": "SELECT Name,DuplicateCheck__c,IsInventoryLocation,IsMobile,LocationType FROM Location", "operation": "Insert", - "useSourceCSVFile": true + "externalId": "Name" + },{ + "query": "SELECT City, CountryCode, PostalCode, Street, Latitude, Longitude, PKey__c, Country, Pkey__c, ParentId$Location FROM Address", + "operation": "Insert" } ] } \ No newline at end of file diff --git a/prepared_steps/4/command_dev.txt b/prepared_steps/4/command_dev.txt index 4c22459..0389686 100644 --- a/prepared_steps/4/command_dev.txt +++ b/prepared_steps/4/command_dev.txt @@ -1 +1 @@ -sf sfdmu run --sourceusername csvfile --targetusername rene.kasseboehmer@vaillant.de.devrene \ No newline at end of file +sf sfdmu run --sourceusername rene.kasseboehmer@vaillant.de.devrene --targetusername rene.kasseboehmer@vaillant.de.devrene \ No newline at end of file diff --git a/prepared_steps/4/export.json b/prepared_steps/4/export.json index 75eac37..ba7ea4f 100644 --- a/prepared_steps/4/export.json +++ b/prepared_steps/4/export.json @@ -2,32 +2,36 @@ "allOrNone": true, "objects": [ { - "query": "SELECT Id, Name, Pkey__c, ParentLocationId FROM Location WHERE Pkey__c != null AND PKey__c like '%NL' AND ParentLocationId = null AND VisitorAddressId = null", + "query": "SELECT Id, Pkey__c FROM Address WHERE CountryCode = 'NL' AND PKey__c != null", + "operation": "Readonly" + },{ + "query": "SELECT Id, Name, VisitorAdressId FROM Location WHERE VisitorAddressId = null", "operation": "Update", "beforeUpdateAddons": [ { "module": "core:RecordsTransform", - "description": "Updates ParentLocationId with source Address.ParentId based on Pkey__c", + "description": "Updates VisitorAddressId with source Address.Id based on Pkey__c", "args": { "fields": [ { - "alias": "sourceAddressParentIdFromPkey", + "alias": "sourceAddressIdFromPkey", "sourceObject": "Address", - "sourceField": "ParentId", - "lookupExpression": "source.PKey__c == target.Pkey__c", + "sourceField": "Id", + "lookupExpression": "source.PKey__c == target.Name", "lookupSource": "source" } ], "transformations": [ { "targetObject": "Location", - "targetField": "ParentLocationId", - "formula": "formula.sourceAddressParentIdFromPkey" + "targetField": "VisitorAddressId", + "formula": "formula.sourceAddressIdFromPkey" } ] } } - ] + ], + "externalId": "Name" } ] } \ No newline at end of file diff --git a/prepared_steps/5/command_dev.txt b/prepared_steps/5/command_dev.txt new file mode 100644 index 0000000..4c22459 --- /dev/null +++ b/prepared_steps/5/command_dev.txt @@ -0,0 +1 @@ +sf sfdmu run --sourceusername csvfile --targetusername rene.kasseboehmer@vaillant.de.devrene \ No newline at end of file diff --git a/prepared_steps/5/export.json b/prepared_steps/5/export.json new file mode 100644 index 0000000..d8a4ac6 --- /dev/null +++ b/prepared_steps/5/export.json @@ -0,0 +1,11 @@ +{ + "allOrNone": true, + "excludeIdsFromCSVFiles": true, + "objects": [ + { + "query": "SELECT Extension,Flat,Floor,Name,DuplicateCheck__c,IsInventoryLocation,IsMobile,LocationType,PKey__c FROM Location", + "operation": "Insert", + "useSourceCSVFile": true + } + ] +} \ No newline at end of file diff --git a/prepared_steps/6/command_dev.txt b/prepared_steps/6/command_dev.txt new file mode 100644 index 0000000..4c22459 --- /dev/null +++ b/prepared_steps/6/command_dev.txt @@ -0,0 +1 @@ +sf sfdmu run --sourceusername csvfile --targetusername rene.kasseboehmer@vaillant.de.devrene \ No newline at end of file diff --git a/prepared_steps/2/export.json b/prepared_steps/6/export.json similarity index 57% rename from prepared_steps/2/export.json rename to prepared_steps/6/export.json index ba7ea4f..75eac37 100644 --- a/prepared_steps/2/export.json +++ b/prepared_steps/6/export.json @@ -2,36 +2,32 @@ "allOrNone": true, "objects": [ { - "query": "SELECT Id, Pkey__c FROM Address WHERE CountryCode = 'NL' AND PKey__c != null", - "operation": "Readonly" - },{ - "query": "SELECT Id, Name, VisitorAdressId FROM Location WHERE VisitorAddressId = null", + "query": "SELECT Id, Name, Pkey__c, ParentLocationId FROM Location WHERE Pkey__c != null AND PKey__c like '%NL' AND ParentLocationId = null AND VisitorAddressId = null", "operation": "Update", "beforeUpdateAddons": [ { "module": "core:RecordsTransform", - "description": "Updates VisitorAddressId with source Address.Id based on Pkey__c", + "description": "Updates ParentLocationId with source Address.ParentId based on Pkey__c", "args": { "fields": [ { - "alias": "sourceAddressIdFromPkey", + "alias": "sourceAddressParentIdFromPkey", "sourceObject": "Address", - "sourceField": "Id", - "lookupExpression": "source.PKey__c == target.Name", + "sourceField": "ParentId", + "lookupExpression": "source.PKey__c == target.Pkey__c", "lookupSource": "source" } ], "transformations": [ { "targetObject": "Location", - "targetField": "VisitorAddressId", - "formula": "formula.sourceAddressIdFromPkey" + "targetField": "ParentLocationId", + "formula": "formula.sourceAddressParentIdFromPkey" } ] } } - ], - "externalId": "Name" + ] } ] } \ No newline at end of file