From 8af0ef8e78ccd8ab5652469dc0cd2068dab60927 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rene=20Ka=C3=9Feb=C3=B6hmer?= Date: Fri, 9 May 2025 15:41:55 +0200 Subject: [PATCH] savestate --- .gitignore | 5 +- __pycache__/sf_auth.cpython-313.pyc | Bin 0 -> 2176 bytes .../export.json | 2 +- .../FillServiceContractFields.py | 48 ++++++++ .../command.txt | 1 + .../InsertServiceContracts.py | 47 ++++++++ .../ValueMapping.csv | 4 + .../15_insert_servicecontract/command.txt | 1 + .../15_insert_servicecontract/export.json | 23 ++++ .../UpdateContractsSelfRerefence.py | 25 ++++ .../command.txt | 1 + .../17_update_servicecontract/command.txt | 1 + .../17_update_servicecontract/export.json | 16 +++ .../extract_via_simple_salesforce.py | 110 ++---------------- prepared_steps/1_extract_data/queries.json | 5 +- .../2_transform_via_script/TransformScript.py | 39 ++++++- sf_auth.py | 70 +++++++++++ 17 files changed, 290 insertions(+), 108 deletions(-) create mode 100644 __pycache__/sf_auth.cpython-313.pyc create mode 100644 prepared_steps/14_fill_pricebook2id_in_servicecontract/FillServiceContractFields.py create mode 100644 prepared_steps/14_fill_pricebook2id_in_servicecontract/command.txt create mode 100644 prepared_steps/15_insert_servicecontract/InsertServiceContracts.py create mode 100644 prepared_steps/15_insert_servicecontract/ValueMapping.csv create mode 100644 prepared_steps/15_insert_servicecontract/command.txt create mode 100644 prepared_steps/15_insert_servicecontract/export.json create mode 100644 prepared_steps/16_build_servicecontract_self_reference/UpdateContractsSelfRerefence.py create mode 100644 prepared_steps/16_build_servicecontract_self_reference/command.txt create mode 100644 prepared_steps/17_update_servicecontract/command.txt create mode 100644 prepared_steps/17_update_servicecontract/export.json create mode 100644 sf_auth.py diff --git a/.gitignore b/.gitignore index 7fc7838..2f4f2b6 100644 --- a/.gitignore +++ b/.gitignore @@ -27,7 +27,8 @@ SCPriceList__c.csv SCPriceListItem__c.csv SCResourceAssignment__c.csv ContractTemplates.csv -ServiceContract.csv +ServiceContract.csv* SCContract__c.csv AssociatedLocation_beforetransform.csv -AssociatedLocation.csv \ No newline at end of file +AssociatedLocation.csv* +ServiceContract_beforetransform.csv \ No newline at end of file diff --git a/__pycache__/sf_auth.cpython-313.pyc b/__pycache__/sf_auth.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b01a6309568fe1004fae72eed3cefa5f93dbe7f0 GIT binary patch literal 2176 zcmb_d&uXAR9hgfclHKhWny>O@$n%vtr8?Oy&RrSzk?Yx=i_ujmD zv){Kn`}U~_izE>xRi}RG2#C9#Cz%X@E?UhKs=qR)MJMC|zVDDhBbrhL}rJBlvth$To@u50Ug@ zdKf1^HjXDjm8fGTQ>vOFZkgKkc+<+e5cWz9!eTkLL09pLxjd?AS7$S0Jf@AG&t@)OnbWV#PL0oS z`Eq7Im^)-aBOnj$*~39B+EJ^%}ra2)DT_D75$mMY#BORFpK3}iIS259o3mjN-Xz| z4%bMHysMjrYvj`DW9hU=W|ZBfbZM2x7m2GkOJVd4s6)I|>=6v>XuAW&yY3Wk7r%7Y zC$=M~v%A)DsMd4%Wms-k)~B{xQ6zqMuo^y8!-E^qyOWhjVplk{+dTgnQ_^gy^;9TU|6CbaB#8BOC#(*uvjd`L0pTy9K20IW~XIlJ! zo7&H&1xBs`_jE#z21ft~1w{d%<+=_Dttndgj6CokJR#C!1Bhrj69SB@Yu~DXEyE?a z$)EzWHt|4)uN#b**lD2s|AcmP#}PaV@0uW)Jd|Sob1mvx-?YV;WiS4XlIrLh!CV$3 zLPguDxLqn*ye6MIo_;$e^N3FsJmOOYkG)P4JiH8;0^rbAh)jhj7Fqhl&<_JwOs8mA zHa!B1l;R4EP&iOnX7`{Q_J7 zwr0<13hd%x_hDE^yB(;#|3R{P;OP3)Z_(JsCr{#im3Ut@e&k7Xpb{N;8auSrd-n0w z#}~J{FH~a_Tak(9;n)|qKD+h#r<(vf{50NO>%N!^ID2xV9hOyjf`<5+PAVgeWa4O^9M66h22@bHvBl<3o=b`F>^4VhSxXmoR5%~VXqgFvqi@&7sv= TODAY" },{ "sobject": "ContractTemplates", - "query": "SELECT id, name, TemplateName__c, status__c, Brand__r.Name, Country__c, Runtime__c FROM sccontract__c WHERE Template__c = null AND Country__c = '{country}'" + "query": "SELECT id, name, TemplateName__c, status__c, Brand__r.Name, Country__c, Runtime__c FROM SCContract__c WHERE Template__c = null AND Country__c = '{country}'" + },{ + "sobject": "SCContract__c", + "query": "SELECT id, name, Template__c, status__c, Brand__r.Name, Country__c, Runtime__c, EndDate__c, StartDate__c, Account__c, AccountOwner__c, IoT_Registration_Status__c FROM SCContract__c WHERE Template__c != null AND EndDate__c >= TODAY AND Country__c = '{country}' limit 3" } ] } \ No newline at end of file diff --git a/prepared_steps/2_transform_via_script/TransformScript.py b/prepared_steps/2_transform_via_script/TransformScript.py index 97f4943..4ae200c 100644 --- a/prepared_steps/2_transform_via_script/TransformScript.py +++ b/prepared_steps/2_transform_via_script/TransformScript.py @@ -24,6 +24,10 @@ country_mapping = { 'RO': 'Romania' } +##--------------------------------------------------------------------------## +## Loading Data +##--------------------------------------------------------------------------## + # Read the input CSV file, assuming the second row is the header read_df = pd.read_csv('../1_extract_data/results/SCInstalledBaseLocation__c.csv', header=0, keep_default_na=False, dtype=str) read_df_ib = pd.read_csv('../1_extract_data/results/SCInstalledBase__c.csv', header=0, keep_default_na=False, dtype=str) @@ -35,6 +39,7 @@ read_df_resourceassignment = pd.read_csv('../1_extract_data/results/SCResourceAs read_df_address_iot = pd.read_csv('../1_extract_data/results/Address.csv', header=0, keep_default_na=False, dtype=str) read_df_location_iot = pd.read_csv('../1_extract_data/results/ParentLocation.csv', header=0, keep_default_na=False, dtype=str) read_df_servicecontracttemplates = pd.read_csv('../1_extract_data/results/ContractTemplates.csv', header=0, keep_default_na=False, dtype=str) +read_df_servicecontracts = pd.read_csv('../1_extract_data/results/SCContract__c.csv', header=0, keep_default_na=False, dtype=str) # Columns for reindexing reindex_columns = ['Id','City__c','Country__c','GeoY__c','GeoX__c','PostalCode__c','Street__c','Extension__c','HouseNo__c','FlatNo__c','Floor__c'] @@ -47,6 +52,7 @@ reindex_columns_resourceassignment = ['Id', 'ValidTo__c', 'ValidFrom__c', 'Count reindex_columns_address_iot = ['Id', 'Country', 'CountryCode', 'Street', 'City', 'ParentId', 'PostalCode'] reindex_columns_location_iot = ['Id', 'Name'] reindex_columns_servicecontracttemplates = ['Id', 'Name', 'TemplateName__c', 'Status__c', 'Brand__r.Name', 'Country__c', 'Runtime__c'] +reindex_columns_servicecontracts = ['Id', 'Name', 'Template__c', 'Status__c', 'Brand__r.Name', 'Country__c', 'Runtime__c', 'EndDate__c', 'StartDate__c', 'Account__c', 'AccountOwner__c', 'IoT_Registration_Status__c'] # Reindex the columns to match the desired format df = read_df.reindex(reindex_columns, axis=1) @@ -59,6 +65,7 @@ df_resourceassignment = read_df_resourceassignment.reindex(reindex_columns_resou df_address_iot = read_df_address_iot.reindex(reindex_columns_address_iot, axis=1) df_location_iot = read_df_location_iot.reindex(reindex_columns_location_iot, axis=1) df_servicecontracttemplates = read_df_servicecontracttemplates.reindex(reindex_columns_servicecontracttemplates, axis=1) +df_servicecontract = read_df_servicecontracts.reindex(reindex_columns_servicecontracts, axis=1) ##--------------------------------------------------------------------------## ## Update for IoT Addresses and Locations @@ -449,7 +456,36 @@ df_servicecontracttemplates['FSL_Sold_by__c'] = 'Service' # Convert Runtime__c (Term) to natural numbers df_servicecontracttemplates['Term'] = pd.to_numeric(df_servicecontracttemplates['Term'].str.extract('(\d+)')[0], errors='coerce') -df_servicecontracttemplates['Term'] = df_servicecontracttemplates['Term'].fillna(12).astype(int) +df_servicecontracttemplates['Term'] = df_servicecontracttemplates['Term'].fillna(0).astype(int) + +##--------------------------------------------------------------------------## +## Service Contract +##--------------------------------------------------------------------------## + +df_servicecontract['Pricebook2.Name'] = ( + df_servicecontract['Country__c'].astype(str).fillna('').str.upper() + ' ' + + df_servicecontract['Brand__r.Name'].astype(str).fillna('').str.upper() + ' ' + + "SERVICE" +) + +df_servicecontract = df_servicecontract.drop('Name', axis=1) +df_servicecontract = df_servicecontract.drop('Brand__r.Name', axis=1) + +df_servicecontract.columns = ['PKey__c', 'TemplateId__r.PKey__c', 'Status', 'BillingCountryCode', 'Term', 'EndDate', 'StartDate', 'AccountId', 'Service_Recipient__c', 'IoT_Registration_Status__c', 'Pricebook2.Name'] + +df_servicecontract['IoT_Registration_Status__c'] = df_servicecontract['IoT_Registration_Status__c'].replace('', 'Open') +df_servicecontract['Name'] = df_servicecontract['PKey__c'] +df_servicecontract['TemplateCountry__c'] = df_servicecontract['BillingCountryCode'] + +#df_servicecontract = df_servicecontract.drop('TemplateId__r.PKey__c', axis=1) + +# Convert Runtime__c (Term) to natural numbers +df_servicecontract['Term'] = pd.to_numeric(df_servicecontract['Term'].str.extract('(\d+)')[0], errors='coerce') +df_servicecontract['Term'] = df_servicecontract['Term'].fillna(0).astype(int) + +##--------------------------------------------------------------------------## +## Saving to CSV +##--------------------------------------------------------------------------## # Write each DataFrame to a separate CSV file address_df.to_csv('../4_upsert_address_and_parent_location/Address.csv', index=False) @@ -461,6 +497,7 @@ df_pricelist.to_csv('../12_insert_pricebook2_and_pricebookentries/Pricebook2.csv df_pricelistitem.to_csv('../12_insert_pricebook2_and_pricebookentries/PricebookEntry.csv', index=False) merged_df_location_iot.to_csv('../3_update_address_and_location_data_for_migration/Location.csv', index=False) df_servicecontracttemplates.to_csv('../13_insert_servicecontracttemplates_dummies/ServiceContract.csv', index=False) +df_servicecontract.to_csv('../15_insert_servicecontract/ServiceContract_beforetransform.csv', index=False) ## end mapping print('Data has been successfully transformed and saved to CSV files.') \ No newline at end of file diff --git a/sf_auth.py b/sf_auth.py new file mode 100644 index 0000000..36536ed --- /dev/null +++ b/sf_auth.py @@ -0,0 +1,70 @@ +import os +from dotenv import load_dotenv, find_dotenv +from simple_salesforce import Salesforce + +def get_credentials(context): + """ + Get credentials for a given context from the .env file + + Args: + context (str): Context name (e.g., 'qa2', 'prod') + + Returns: + dict: Credentials dictionary with username, password, and security_token + """ + context = context.upper() + + # Initialize credentials dictionary + credentials = { + 'USERNAME': None, + 'PASSWORD': None, + 'SECURITY_TOKEN': None + } + + if context != 'PROD': + credentials['DOMAIN'] = 'test' + + # Load the .env file + env_file = find_dotenv(".env") + load_dotenv(env_file) + + # Load all environment variables + env_vars = os.environ + + for key, value in env_vars.items(): + if f'{context}_SF_' in key: + credential_key = key.split(f'{context}_SF_')[-1].upper() + credentials[credential_key] = value + + return credentials + +def get_sf_connection(context): + """ + Create Salesforce connection based on context + + Args: + context (str): Context name (e.g., 'qa2', 'prod') + + Returns: + Salesforce: Authenticated Salesforce connection + """ + credentials = get_credentials(context) + + if not all(credentials.values()): + raise ValueError(f"Missing credentials for context: {context}") + + if context.lower() == 'prod': + return Salesforce( + username=credentials['USERNAME'], + password=credentials['PASSWORD'], + security_token=credentials['SECURITY_TOKEN'], + version='62.0' + ) + else: + return Salesforce( + username=credentials['USERNAME'], + password=credentials['PASSWORD'], + security_token=credentials['SECURITY_TOKEN'], + domain=credentials['DOMAIN'], + version='62.0' + ) \ No newline at end of file