From 8485b38dba0f851dce8a77ab68362bcc810ac985 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rene=20Ka=C3=9Feb=C3=B6hmer?= Date: Thu, 17 Apr 2025 10:32:35 +0200 Subject: [PATCH] finished pricebook2 & pricebookentry --- .gitignore | 4 +- README.md | 7 +-- .../AssociatedLocation_test.csv | 3 ++ .../command.txt | 1 + .../export.json | 23 ++++++++ prepared_steps/12_insert_/command.txt | 1 + prepared_steps/12_insert_/export.json | 10 ++++ .../extract_via_simple_salesforce.py | 6 ++- prepared_steps/1_extract_data/queries.json | 20 ++++--- .../results/SCPriceListItem__c.csv | 2 + .../1_extract_data/results/SCPriceList__c.csv | 3 ++ .../{LocationScript.py => TransformScript.py} | 53 ++++++++++++++++++- .../2_transform_via_script/command.txt | 2 +- 13 files changed, 119 insertions(+), 16 deletions(-) create mode 100644 prepared_steps/10_upsert_associated_location/AssociatedLocation_test.csv create mode 100644 prepared_steps/11_insert_pricebook2_and_pricebookentries/command.txt create mode 100644 prepared_steps/11_insert_pricebook2_and_pricebookentries/export.json create mode 100644 prepared_steps/12_insert_/command.txt create mode 100644 prepared_steps/12_insert_/export.json create mode 100644 prepared_steps/1_extract_data/results/SCPriceListItem__c.csv create mode 100644 prepared_steps/1_extract_data/results/SCPriceList__c.csv rename prepared_steps/2_transform_via_script/{LocationScript.py => TransformScript.py} (82%) diff --git a/.gitignore b/.gitignore index 49cd72a..97c4c10 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,6 @@ Asset.csv .env venv/ *Location.csv -Product2.csv \ No newline at end of file +Product2.csv +Pricebook2.csv +PricebookEntry.csv \ No newline at end of file diff --git a/README.md b/README.md index 3ba2f17..a06c1ee 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,16 @@ -# FSL Migration from Legacy with SFDMU & Python (Pandas) +# FSL Migration from Legacy with SFDMU & Python This git is representing a migration from Legacy GMS solution to FSL Standard ## Installation Install Python [python](https://www.python.org/) to install python 3.13+. -After installation open a terminal/powershell to install pandas globally +After installation open a terminal/powershell to install the packages globally ```bash -python -m pip install pandas +cd ./migration_via_sfdmu-main +python -m pip install -r ./requirements.txt ``` ## Usage diff --git a/prepared_steps/10_upsert_associated_location/AssociatedLocation_test.csv b/prepared_steps/10_upsert_associated_location/AssociatedLocation_test.csv new file mode 100644 index 0000000..3c4c65c --- /dev/null +++ b/prepared_steps/10_upsert_associated_location/AssociatedLocation_test.csv @@ -0,0 +1,3 @@ +Type,ActiveFrom,ActiveTo,ParentRecordId,PKey__c,LocationId +Installer (installation),2018-03-27,,0019Z00000fmCAYQA2,"Mr. P.J.M. Aalberselaan 19, 1181 XJ AMSTELVEEN, NL;;;",1319Z000000ij1RQAQ +Owner,2018-04-06,,0019Z00000fmCAYQA2,"Mr. P.J.M. Aalberselaan 19, 1181 XJ AMSTELVEEN, NL;;;",1319Z000000ij1RQAQ diff --git a/prepared_steps/11_insert_pricebook2_and_pricebookentries/command.txt b/prepared_steps/11_insert_pricebook2_and_pricebookentries/command.txt new file mode 100644 index 0000000..4c22459 --- /dev/null +++ b/prepared_steps/11_insert_pricebook2_and_pricebookentries/command.txt @@ -0,0 +1 @@ +sf sfdmu run --sourceusername csvfile --targetusername rene.kasseboehmer@vaillant.de.devrene \ No newline at end of file diff --git a/prepared_steps/11_insert_pricebook2_and_pricebookentries/export.json b/prepared_steps/11_insert_pricebook2_and_pricebookentries/export.json new file mode 100644 index 0000000..1d1f701 --- /dev/null +++ b/prepared_steps/11_insert_pricebook2_and_pricebookentries/export.json @@ -0,0 +1,23 @@ +{ + "allOrNone": true, + "excludeIdsFromCSVFiles": true, + "objects": [ + { + "query": "SELECT Id, ProductCode FROM Product2", + "operation": "Readonly", + "master": false, + "externalId": "ProductCode" + },{ + "query": "SELECT Business_Type__c, IsStandard, IsActive, Name, Brand__c, Country__c FROM Pricebook2", + "operation": "Insert", + "externalId": "Name", + "master": true, + "useSourceCSVFile": true + },{ + "query": "SELECT IsActive, Product2Id, UnitPrice, Pricebook2Id FROM PricebookEntry", + "operation": "Insert", + "master": true, + "useSourceCSVFile": true + } + ] +} \ No newline at end of file diff --git a/prepared_steps/12_insert_/command.txt b/prepared_steps/12_insert_/command.txt new file mode 100644 index 0000000..4c22459 --- /dev/null +++ b/prepared_steps/12_insert_/command.txt @@ -0,0 +1 @@ +sf sfdmu run --sourceusername csvfile --targetusername rene.kasseboehmer@vaillant.de.devrene \ No newline at end of file diff --git a/prepared_steps/12_insert_/export.json b/prepared_steps/12_insert_/export.json new file mode 100644 index 0000000..3e26984 --- /dev/null +++ b/prepared_steps/12_insert_/export.json @@ -0,0 +1,10 @@ +{ + "allOrNone": true, + "excludeIdsFromCSVFiles": true, + "objects": [ + { + "query": "SELECT Business_Type__c, IsStandard, IsActive, Name, Brand__c, Country__c FROM Pricebook2", + "operation": "Insert" + } + ] +} \ No newline at end of file diff --git a/prepared_steps/1_extract_data/extract_via_simple_salesforce.py b/prepared_steps/1_extract_data/extract_via_simple_salesforce.py index b58fb44..8673cb8 100644 --- a/prepared_steps/1_extract_data/extract_via_simple_salesforce.py +++ b/prepared_steps/1_extract_data/extract_via_simple_salesforce.py @@ -147,11 +147,13 @@ def extract_data(object_id, query, output_path='output', context='qa2'): if __name__ == '__main__': import argparse - + # Parse command-line arguments parser = argparse.ArgumentParser(description='Extract Salesforce data via Bulk API') parser.add_argument('--context', type=str, required=True, help='Context name (e.g., "qa2", "prod")') + parser.add_argument('--country', type=str, required=True, + help='CountryCode (e.g., "NL", "UK")') parser.add_argument('--output_path', type=str, required=False, help='./') parser.add_argument('--sobjects', type=str, required=False, @@ -170,7 +172,7 @@ if __name__ == '__main__': for query_def in queries: sobject = query_def['sobject'] - query_str = query_def['query'] + query_str = query_def['query'].replace('{country}', args.country) # Skip if --sobjects is provided and current sobject isn't in the list if selected_sobjects and sobject not in selected_sobjects: diff --git a/prepared_steps/1_extract_data/queries.json b/prepared_steps/1_extract_data/queries.json index d893960..f399c92 100644 --- a/prepared_steps/1_extract_data/queries.json +++ b/prepared_steps/1_extract_data/queries.json @@ -2,28 +2,34 @@ [ { "sobject": "SCInstalledBaseLocation__c", - "query": "SELECT Id, City__c, Country__c, GeoY__c, GeoX__c, PostalCode__c, Street__c, Extension__c, HouseNo__c, FlatNo__c, Floor__c FROM SCInstalledBaseLocation__c WHERE Country__c = 'NL' AND Id = 'a1B1r0000099EsfEAE' limit 1" + "query": "SELECT Id, City__c, Country__c, GeoY__c, GeoX__c, PostalCode__c, Street__c, Extension__c, HouseNo__c, FlatNo__c, Floor__c FROM SCInstalledBaseLocation__c WHERE Country__c = '{country}' AND Id = 'a1B1r0000099EsfEAE' limit 1" },{ "sobject": "SCInstalledBase__c", - "query": "SELECT Id, Name, CommissioningDate__c,InstallationDate__c,ProductEnergy__c, ProductUnitClass__c,ArticleNo__c,SerialNo__c, SerialNoException__c, ProductUnitType__c, InstalledBaseLocation__c FROM SCInstalledBase__c WHERE Country__c = 'NL' AND InstalledBaseLocation__c = 'a1B1r0000099EsfEAE'" + "query": "SELECT Id, Name, CommissioningDate__c,InstallationDate__c,ProductEnergy__c, ProductUnitClass__c,ArticleNo__c,SerialNo__c, SerialNoException__c, ProductUnitType__c, InstalledBaseLocation__c FROM SCInstalledBase__c WHERE Country__c = '{country}' AND InstalledBaseLocation__c = 'a1B1r0000099EsfEAE'" },{ "sobject": "Asset", - "query": "SELECT Id, Serialnumber FROM Asset WHERE Location.ParentLocation.Name LIKE '%NL'" + "query": "SELECT Id, Serialnumber FROM Asset WHERE Location.ParentLocation.Name LIKE '%{country}'" },{ "sobject": "Address", - "query": "SELECT Id, Country, CountryCode, Street, City, ParentId, PostalCode FROM Address WHERE CountryCode = 'NL'" + "query": "SELECT Id, Country, CountryCode, Street, City, ParentId, PostalCode FROM Address WHERE CountryCode = '{country}'" },{ "sobject": "ParentLocation", - "query": "SELECT Id, Name FROM Location WHERE Name LIKE '%NL'" + "query": "SELECT Id, Name FROM Location WHERE Name LIKE '%{country}'" },{ "sobject": "ChildLocation", - "query": "SELECT Id, ParentLocationId, Name FROM Location WHERE ParentLocation.Name LIKE '%NL'" + "query": "SELECT Id, ParentLocationId, Name FROM Location WHERE ParentLocation.Name LIKE '%{country}'" },{ "sobject": "Product2", "query": "SELECT Id, Main_Product_Group__c, Family, MaterialType__c, Name, Product_Code__c, ProductCode, EAN_Product_Code__c FROM Product2" },{ "sobject": "SCInstalledBaseRole__c", - "query": "SELECT Id, InstalledBaseLocation__c, Role__c, ValidFrom__c, ValidTo__c, Account__c FROM SCInstalledBaseRole__c WHERE InstalledBaseLocation__r.Country__c = 'NL' AND InstalledBaseLocation__c = 'a1B1r0000099EsfEAE'" + "query": "SELECT Id, InstalledBaseLocation__c, Role__c, ValidFrom__c, ValidTo__c, Account__c FROM SCInstalledBaseRole__c WHERE InstalledBaseLocation__r.Country__c = '{country}' AND InstalledBaseLocation__c = 'a1B1r0000099EsfEAE'" + },{ + "sobject": "SCPriceList__c", + "query": "SELECT Id, Name, Brand__r.Name, Country__c from SCPriceList__c WHERE Country__c = '{country}' AND PriceList__c != null" + },{ + "sobject": "SCPriceListItem__c", + "query": "SELECT Id, Article__r.Name, Price__c, PriceUnit__c, Pricelist__c, ValidFrom__c, ValidTo__c, Article__r.EANCode__c, Pricelist__r.Brand__r.Name, Pricelist__r.Country__c FROM SCPriceListItem__c WHERE Country__c = '{country}' AND PriceList__c != null AND (ValidTo__c >= TODAY OR ValidFrom__c >= TODAY) AND Article__c = 'a0Gw000001R9slpEAB'" } ] } \ No newline at end of file diff --git a/prepared_steps/1_extract_data/results/SCPriceListItem__c.csv b/prepared_steps/1_extract_data/results/SCPriceListItem__c.csv new file mode 100644 index 0000000..8bff817 --- /dev/null +++ b/prepared_steps/1_extract_data/results/SCPriceListItem__c.csv @@ -0,0 +1,2 @@ +"Id","Article__r.Name","Article__r.EANCode__c","Price__c","PriceUnit__c","Pricelist__c","ValidFrom__c","ValidTo__c","Pricelist__r.Brand__r.Name","Pricelist__r.Country__c" +"a0p6900000eZ3nbAAC","0010021907","4024074807507","2011.85","1.0","a0q20000000PO8OAAW","2021-01-01","2099-12-31","Vaillant","NL" diff --git a/prepared_steps/1_extract_data/results/SCPriceList__c.csv b/prepared_steps/1_extract_data/results/SCPriceList__c.csv new file mode 100644 index 0000000..a76b1b4 --- /dev/null +++ b/prepared_steps/1_extract_data/results/SCPriceList__c.csv @@ -0,0 +1,3 @@ +"Id","Name","Brand__r.Name","Country__c" +"a0q20000000PO8OAAW","NL Vaillant","Vaillant","NL" +"a0q20000000PO8TAAW","NL AWB","AWB","NL" diff --git a/prepared_steps/2_transform_via_script/LocationScript.py b/prepared_steps/2_transform_via_script/TransformScript.py similarity index 82% rename from prepared_steps/2_transform_via_script/LocationScript.py rename to prepared_steps/2_transform_via_script/TransformScript.py index 4bc8373..ab57643 100644 --- a/prepared_steps/2_transform_via_script/LocationScript.py +++ b/prepared_steps/2_transform_via_script/TransformScript.py @@ -10,6 +10,8 @@ read_df = pd.read_csv('../1_extract_data/results/SCInstalledBaseLocation__c.csv' read_df_ib = pd.read_csv('../1_extract_data/results/SCInstalledBase__c.csv', header=0, keep_default_na=False, dtype=str) read_df_product2 = pd.read_csv('../1_extract_data/results/Product2.csv', header=0, keep_default_na=False, dtype=str) read_df_ibr = pd.read_csv('../1_extract_data/results/SCInstalledBaseRole__c.csv', header=0, keep_default_na=False, dtype=str) +read_df_pricelist = pd.read_csv('../1_extract_data/results/SCPriceList__c.csv', header=0, keep_default_na=False, dtype=str) +read_df_pricelistitem = pd.read_csv('../1_extract_data/results/SCPriceListItem__c.csv', header=0, keep_default_na=False, dtype=str) for row in read_df.to_dict('records'): try: @@ -31,13 +33,19 @@ reindex_columns_product2 = ['Id','Main_Product_Group__c','Family','MaterialType_ #reindex_columns_product2 = ['EAN_Product_Code__c','Family','Id','Main_Product_Group__c','MaterialType__c','Name','Product_Code__c','ProductCode'] #"Id","InstalledBaseLocation__c","Role__c","ValidFrom__c","ValidTo__c","Account__c" reindex_columns_ibr = ['Id', 'InstalledBaseLocation__c', 'Role__c', 'ValidFrom__c', 'ValidTo__c', 'Account__c'] +reindex_columns_pricelist = ['Id', 'Name', 'Brand__r.Name', 'Country__c'] +reindex_columns_pricelistitem = ['Id', 'Article__r.Name', 'Article__r.EANCode__c', 'Price__c', 'PriceUnit__c', 'Pricelist__c', 'ValidFrom__c', 'ValidTo__c', 'Pricelist__r.Brand__r.Name', 'Pricelist__r.Country__c'] # Reindex the columns to match the desired format df = read_df.reindex(reindex_columns, axis=1) df_ib = read_df_ib.reindex(reindex_columns_ib, axis=1) df_product2 = read_df_product2.reindex(reindex_columns_product2, axis=1) df_ibr = read_df_ibr.reindex(reindex_columns_ibr, axis=1) +df_pricelist = read_df_pricelist.reindex(reindex_columns_pricelist, axis=1) +df_pricelistitem = read_df_pricelistitem.reindex(reindex_columns_pricelistitem, axis=1) +#creating street column +# Concatenate 'Street__c' and 'HouseNo__c' to create the 'Street' column df['Street'] = ( df['Street__c'].astype(str) + ' ' + df['HouseNo__c'].astype(str) @@ -246,14 +254,55 @@ for index, row in tqdm(merged_df_ib.iterrows(), total=len(merged_df_ib)): if product_group not in valid_groups: merged_df_ib.loc[index, 'Kind_of_Energy__c'] = None # or set to empty string -print(merged_df_ib) + + +# Pricelist to Pricebook2 +columns_pricebook2 = ['Id', 'Name', 'Brand__c', 'Country__c'] + +df_pricelist.columns = columns_pricebook2 + +df_pricelist.insert(0, 'IsActive', 'true') +df_pricelist.insert(0, 'IsStandard', 'false') +df_pricelist.insert(0, 'Business_Type__c', 'Service') + +df_pricelist['Name'] = ( + df_pricelist['Country__c'].astype(str).fillna('').str.upper() + ' ' + + df_pricelist['Brand__c'].astype(str).fillna('').str.upper() + ' ' + + df_pricelist['Business_Type__c'].astype(str).fillna('').str.upper() +) + +df_pricelist = df_pricelist.drop('Id', axis=1) + +df_pricelistitem['Pricebook2.Name'] = ( + df_pricelistitem['Pricelist__r.Country__c'].astype(str).fillna('').str.upper() + ' ' + + df_pricelistitem['Pricelist__r.Brand__r.Name'].astype(str).fillna('').str.upper() + ' ' + + 'SERVICE' +) + +df_pricelistitem = df_pricelistitem.drop('Id', axis=1) +df_pricelistitem = df_pricelistitem.drop('PriceUnit__c', axis=1) +df_pricelistitem = df_pricelistitem.drop('Pricelist__c', axis=1) +df_pricelistitem = df_pricelistitem.drop('ValidFrom__c', axis=1) +df_pricelistitem = df_pricelistitem.drop('ValidTo__c', axis=1) +df_pricelistitem = df_pricelistitem.drop('Pricelist__r.Country__c', axis=1) +df_pricelistitem = df_pricelistitem.drop('Pricelist__r.Brand__r.Name', axis=1) + +df_pricelistitem.insert(0, 'IsActive', 'true') + +print(df_pricelistitem) + +columns_pricebookentry = ['IsActive', 'Product2.Product_Code__c', 'Product2.EAN_Product_Code__c', 'UnitPrice', 'Pricebook2.Name'] + +df_pricelistitem.columns = columns_pricebookentry # Write each DataFrame to a separate CSV file address_df.to_csv('../3_upsert_address_and_parent_location/Address.csv', index=False) parent_df.to_csv('../3_upsert_address_and_parent_location/Location.csv', index=False) child_df.to_csv('../5_upsert_child_location/Location.csv', index=False) merged_df_ib.to_csv('../7_upsert_assets/Asset.csv', index=False) -df_ibr.to_csv('../9_upsert_associated_location/AssociatedLocation.csv', index=False) +df_ibr.to_csv('../10_upsert_associated_location/AssociatedLocation.csv', index=False) +df_pricelist.to_csv('../11_insert_pricebook2/Pricebook2.csv', index=False) +df_pricelistitem.to_csv('../11_insert_pricebook2/PricebookEntry.csv', index=False) ## end mapping diff --git a/prepared_steps/2_transform_via_script/command.txt b/prepared_steps/2_transform_via_script/command.txt index aab9be0..ac834ce 100644 --- a/prepared_steps/2_transform_via_script/command.txt +++ b/prepared_steps/2_transform_via_script/command.txt @@ -1 +1 @@ -python .\LocationScript.py \ No newline at end of file +python .\TransformScript.py \ No newline at end of file