diff --git a/prepared_steps/1/export.json b/prepared_steps/1/export.json deleted file mode 100644 index 6e8a6ef..0000000 --- a/prepared_steps/1/export.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "allOrNone": true, - "objects": [ - { - "query": "SELECT Id, City__c, Country__c, GeoY__c, GeoX__c, PostalCode__c, Street__c, Extension__c, HouseNo__c, FlatNo__c, Floor__c FROM SCInstalledBaseLocation__c WHERE Country__c = 'NL' limit 1", - "externalId": "Id" - }, - { - "query": "SELECT Id, Name, CommissioningDate__c,InstallationDate__c,ProductEnergy__c, ProductUnitClass__c,ArticleNo__c,SerialNo__c, SerialNoException__c, ProductUnitType__c, InstalledBaseLocation__c FROM SCInstalledBase__c WHERE Country__c = 'NL' limit 1", - "externalId": "Id" - } - - ] -} \ No newline at end of file diff --git a/prepared_steps/1/command.txt b/prepared_steps/1_extract_data/command.txt similarity index 100% rename from prepared_steps/1/command.txt rename to prepared_steps/1_extract_data/command.txt diff --git a/prepared_steps/1_extract_data/export.json b/prepared_steps/1_extract_data/export.json new file mode 100644 index 0000000..f5d1293 --- /dev/null +++ b/prepared_steps/1_extract_data/export.json @@ -0,0 +1,53 @@ +{ + "allOrNone": true, + "useSeparatedCSVFiles": true, + "pollingQueryTimeoutMs": 1000000, + "bulkApiVersion": "2.0", + "parallelRestJobs": 2, + "objectSets": [ + { + "objects": [ + { + "query": "SELECT Id, City__c, Country__c, GeoY__c, GeoX__c, PostalCode__c, Street__c, Extension__c, HouseNo__c, FlatNo__c, Floor__c FROM SCInstalledBaseLocation__c WHERE Country__c = 'NL'", + "externalId": "Name", + "operation": "Readonly" + } + ] + },{ + "objects": [ + { + "query": "SELECT Id, Name, CommissioningDate__c,InstallationDate__c,ProductEnergy__c, ProductUnitClass__c,ArticleNo__c,SerialNo__c, SerialNoException__c, ProductUnitType__c, InstalledBaseLocation__c FROM SCInstalledBase__c WHERE Country__c = 'NL'", + "externalId": "Name", + "operation": "Readonly", + "excludedFromUpdateFields": ["InstalledBaseLocation__c"] + } + ] + },{ + "objects": [ + { + "query": "SELECT Id, Serialnumber FROM Asset WHERE Location.ParentLocation.Name LIKE '%NL'", + "externalId": "Name", + "operation": "Readonly" + } + ] + },{ + "objects": [ + { + "query": "SELECT Id, Country, CountryCode, Street, City, ParentId PostalCode FROM Address WHERE CountryCode = 'NL'", + "externalId": "Name", + "operation": "Readonly", + "excludedFields": ["ParentId"] + } + ] + + },{ + "objects": [ + { + "query": "SELECT Id, ParentLocationId, Name FROM Location WHERE Name LIKE '%NL' OR ParentLocation.Name = '%NL'", + "externalId": "Name", + "operation": "Readonly" + } + ] + } + ] +} \ No newline at end of file diff --git a/prepared_steps/2/LocationScript.py b/prepared_steps/2_transform_via_script/LocationScript.py similarity index 92% rename from prepared_steps/2/LocationScript.py rename to prepared_steps/2_transform_via_script/LocationScript.py index ab1d5b7..c090ae7 100644 --- a/prepared_steps/2/LocationScript.py +++ b/prepared_steps/2_transform_via_script/LocationScript.py @@ -5,8 +5,8 @@ country_mapping = { } # Read the input CSV file, assuming the second row is the header -read_df = pd.read_csv('../1/SCInstalledBaseLocation__c.csv', header=0, keep_default_na=False, dtype=str) -read_df_ib = pd.read_csv('../1/SCInstalledBase__c.csv', header=0, keep_default_na=False, dtype=str) +read_df = pd.read_csv('../1_extract_data/SCInstalledBaseLocation__c.csv', header=0, keep_default_na=False, dtype=str) +read_df_ib = pd.read_csv('../1_extract_data/SCInstalledBase__c.csv', header=0, keep_default_na=False, dtype=str) for row in read_df.to_dict('records'): try: # Your processing logic here @@ -154,9 +154,9 @@ print(merged_df_ib.columns) merged_df_ib.columns = ['Product2.EAN_Product_Code__c', 'FSL_1st_Ignition_Date__c', 'Id', 'InstallDate', 'Name', 'Kind_of_Energy__c', 'Kind_of_Installation__c', 'Main_Product_Group__c', 'SerialNumber', 'Serialnumber_Exception__c', 'Location.ExternalReference'] # Write each DataFrame to a separate CSV file -address_df.to_csv('../3/Address.csv', index=False) -parent_df.to_csv('../3/Location.csv', index=False) -child_df.to_csv('../5/Location.csv', index=False) -merged_df_ib.to_csv('../7/Asset.csv', index=False) +address_df.to_csv('../3_upsert_address_and_parent_location/Address.csv', index=False) +parent_df.to_csv('../3_upsert_address_and_parent_location/Location.csv', index=False) +child_df.to_csv('../5_upsert_child_location/Location.csv', index=False) +merged_df_ib.to_csv('../7_upsert_assets/Asset.csv', index=False) print('Data has been successfully split into Address.csv, Parent_Location.csv, and Child_Location.csv files with duplicate checks applied.') \ No newline at end of file diff --git a/prepared_steps/2/command.txt b/prepared_steps/2_transform_via_script/command.txt similarity index 100% rename from prepared_steps/2/command.txt rename to prepared_steps/2_transform_via_script/command.txt diff --git a/prepared_steps/3/Location.csv b/prepared_steps/3/Location.csv deleted file mode 100644 index 69e3156..0000000 --- a/prepared_steps/3/Location.csv +++ /dev/null @@ -1,2 +0,0 @@ -Name,DuplicateCheck__c,IsInventoryLocation,IsMobile,LocationType -"Plantsoen de Pas 12, 6601 BK WIJCHEN, NL",false,false,false,Site diff --git a/prepared_steps/3/command.txt b/prepared_steps/3_upsert_address_and_parent_location/command.txt similarity index 100% rename from prepared_steps/3/command.txt rename to prepared_steps/3_upsert_address_and_parent_location/command.txt diff --git a/prepared_steps/3/export.json b/prepared_steps/3_upsert_address_and_parent_location/export.json similarity index 100% rename from prepared_steps/3/export.json rename to prepared_steps/3_upsert_address_and_parent_location/export.json diff --git a/prepared_steps/4/command_dev.txt b/prepared_steps/4_fill_visitoraddress_on_parentlocation/command_dev.txt similarity index 100% rename from prepared_steps/4/command_dev.txt rename to prepared_steps/4_fill_visitoraddress_on_parentlocation/command_dev.txt diff --git a/prepared_steps/4/export.json b/prepared_steps/4_fill_visitoraddress_on_parentlocation/export.json similarity index 100% rename from prepared_steps/4/export.json rename to prepared_steps/4_fill_visitoraddress_on_parentlocation/export.json diff --git a/prepared_steps/5/command_dev.txt b/prepared_steps/5_upsert_child_location/command_dev.txt similarity index 100% rename from prepared_steps/5/command_dev.txt rename to prepared_steps/5_upsert_child_location/command_dev.txt diff --git a/prepared_steps/5/export.json b/prepared_steps/5_upsert_child_location/export.json similarity index 100% rename from prepared_steps/5/export.json rename to prepared_steps/5_upsert_child_location/export.json diff --git a/prepared_steps/6/command_dev.txt b/prepared_steps/6_build_lookup_child_to_parent_location/command_dev.txt similarity index 100% rename from prepared_steps/6/command_dev.txt rename to prepared_steps/6_build_lookup_child_to_parent_location/command_dev.txt diff --git a/prepared_steps/6/export.json b/prepared_steps/6_build_lookup_child_to_parent_location/export.json similarity index 100% rename from prepared_steps/6/export.json rename to prepared_steps/6_build_lookup_child_to_parent_location/export.json diff --git a/prepared_steps/7_upsert_assets/ValueMapping.csv b/prepared_steps/7_upsert_assets/ValueMapping.csv new file mode 100644 index 0000000..a3be9b2 --- /dev/null +++ b/prepared_steps/7_upsert_assets/ValueMapping.csv @@ -0,0 +1,2 @@ +ObjectName,FieldName,RawValue,Value +Asset,Kind_of_Energy__c,2, \ No newline at end of file diff --git a/prepared_steps/7/command_dev.txt b/prepared_steps/7_upsert_assets/command_dev.txt similarity index 100% rename from prepared_steps/7/command_dev.txt rename to prepared_steps/7_upsert_assets/command_dev.txt diff --git a/prepared_steps/7/export.json b/prepared_steps/7_upsert_assets/export.json similarity index 100% rename from prepared_steps/7/export.json rename to prepared_steps/7_upsert_assets/export.json