From c3be184dbf6bf6cf23091b9fa211a121539ba2e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20S=C3=A1nchez?= Date: Mon, 11 Nov 2024 15:58:16 -0300 Subject: [PATCH] CPU optimizations --- .gitignore | 3 +-- changelog.txt | 3 +++ exchange_wrapper.py | 62 ++++++++++++++++++++++++++++++--------------- main.py | 2 +- profits/db_read.py | 25 ------------------ todo.txt | 2 +- trader.py | 50 +++++++++++++++++++++--------------- 7 files changed, 77 insertions(+), 70 deletions(-) delete mode 100644 profits/db_read.py diff --git a/.gitignore b/.gitignore index 574164b..1c19c56 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,6 @@ credentials.py DCAv2.code-workspace profits/my_database.db profits/profits_database.db -profits/db_read.py utils/certs/ utils/__pycache__/ utils/close.py @@ -21,4 +20,4 @@ upload_mainnet.sh utils/data/binance.db utils/data/okex.db utils/data/gateio.db -utils/data/kucoin.db +utils/data/kucoin.db \ No newline at end of file diff --git a/changelog.txt b/changelog.txt index 2d0a225..61b5e43 100755 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,6 @@ +2024.11.11: +. Refactored boost check to optimize CPU usage. + 2024.11.10: . Removed a double-summing bug in new_so_routine. diff --git a/exchange_wrapper.py b/exchange_wrapper.py index 500274e..ca28bc9 100755 --- a/exchange_wrapper.py +++ b/exchange_wrapper.py @@ -65,6 +65,28 @@ class broker: return 1 + def get_trades_timestamps(self,pair,timespan,no_retries=False): + ''' + Returns the timestamps of the last trades from the database for the boosting algorithm + ''' + + retries = self.retries + while retries>0: + try: + database_connection = sqlite3.connect("profits/profits_database.db") + database_cursor = database_connection.cursor() + database_cursor.execute(f"SELECT * FROM profits_table WHERE timestamp >= {time.time()-timespan} ORDER BY timestamp DESC") + rows = database_cursor.fetchall() + return [item[0] for item in rows if item[1]==pair] + except Exception as e: + self.logger.log_this(f"Exception in preload_timestamps: {e}") + if no_retries: + break + retries-=1 + time.sleep(self.wait_time) + return [] + + def write_profit_to_db(self,dataset,no_retries=False): ''' dataset format: (timestamp,pair,amount,exchange_name,order_id,order_history) @@ -87,27 +109,27 @@ class broker: return 1 - def return_last_n_deals_timestamps(self, pair, amount, no_retries=False): - ''' - Returns a list containing the last n deals timestamps. - ''' + # def return_last_n_deals_timestamps(self, pair, amount, no_retries=False): + # ''' + # Returns a list containing the last n deals timestamps. + # ''' - retries = self.retries - while retries>0: - try: - database_connection = sqlite3.connect("profits/profits_database.db") - database_cursor = database_connection.cursor() - database_cursor.execute(f"SELECT * FROM profits_table WHERE pair = '{pair}' ORDER BY timestamp DESC LIMIT {amount};") - rows = database_cursor.fetchall() - database_connection.close() - return [item[0] for item in rows] - except Exception as e: - self.logger.log_this(f"Exception in return_last_n_deals_timestamps: {e}",1) - if no_retries: - break - retries-=1 - time.sleep(self.wait_time) - return [] + # retries = self.retries + # while retries>0: + # try: + # database_connection = sqlite3.connect("profits/profits_database.db") + # database_cursor = database_connection.cursor() + # database_cursor.execute(f"SELECT * FROM profits_table WHERE pair = '{pair}' ORDER BY timestamp DESC LIMIT {amount};") + # rows = database_cursor.fetchall() + # database_connection.close() + # return [item[0] for item in rows] + # except Exception as e: + # self.logger.log_this(f"Exception in return_last_n_deals_timestamps: {e}",1) + # if no_retries: + # break + # retries-=1 + # time.sleep(self.wait_time) + # return [] def check_for_duplicate_profit_in_db(self,order,no_retries=False): diff --git a/main.py b/main.py index 53fb532..0085691 100644 --- a/main.py +++ b/main.py @@ -22,7 +22,7 @@ In case the permissions of the certificate changes, reset them this way: # ll /etc/letsencrypt/ ''' -version = "2024.11.10" +version = "2024.11.11" ''' Color definitions. If you want to change them, check the reference at https://en.wikipedia.org/wiki/ANSI_escape_code#Colors diff --git a/profits/db_read.py b/profits/db_read.py deleted file mode 100644 index dec96d9..0000000 --- a/profits/db_read.py +++ /dev/null @@ -1,25 +0,0 @@ -import sqlite3 -import time -import json - -# Connect to the SQLite database -conn = sqlite3.connect('profits_database.db') -cursor = conn.cursor() - -# Execute a SELECT query to retrieve data from the database -cursor.execute('SELECT * FROM profits_table') - -# Fetch all rows from the result set -rows = cursor.fetchall() - -# Process the fetched rows -#for row in rows: - #human_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(row[0]))) - #print(human_time,row[1],round(row[2],2),row[3]) # Or do whatever processing you need - -#data = json.loads(rows[-3][-1]) -print([item[0] for item in rows[-4:]]) -print(f"Number of entries: {len(rows)}") - -# Close the connection -conn.close() diff --git a/todo.txt b/todo.txt index 4648ca6..75ec6d0 100755 --- a/todo.txt +++ b/todo.txt @@ -2,7 +2,7 @@ Mandatory: ========= 0. Mobile app. 1. Stats webpage. -2. Instead of giving a list of order_ids to each trader, give a list of the open orders and that's it (for easier future development, partial order fills for example) +2. Instead of providing a list of order_ids to each trader, provide a list of the open orders for easier future development, including partial order fills. 3. Maintain local orderbooks for each trading pair, which enables: 3a. Smart order pricing: Prioritization of fill speed over instant profit or vice versa 4. Consolidate vocabulary (trader, pair and bot; instance & trader) diff --git a/trader.py b/trader.py index fa923d6..22cb30e 100755 --- a/trader.py +++ b/trader.py @@ -75,6 +75,17 @@ class trader: self.broker.logger.log_this(f"Exception: No old_long file. {e}",1,self.pair) self.profit_filename = f"profits/{self.base}{self.quote}.profits" self.log_filename = f"logs/{self.base}{self.quote}.log" + + self.boosted_deals_range = 4 + self.boosted_time_range = 3600 + self.boosted_amount = .01 + if "boosted_deals_range" in self.config_dict: + self.boosted_deals_range = self.config_dict["boosted_deals_range"] + if "boosted_time_range" in self.config_dict: + self.boosted_time_range = self.config_dict["boosted_time_range"] + if "boosted_amount" in self.config_dict: + self.boosted_amount = self.config_dict["boosted_amount"] + self.deals_timestamps = self.broker.get_trades_timestamps(self.pair,self.boosted_time_range) self.stop_when_profit = False self.status_dict["pause_reason"] = "Initialization" @@ -535,7 +546,7 @@ class trader: self.pause = False self.status_dict["pause_reason"] = "" return None,None - return optimal_order_size,amount_of_so + return optimal_order_size,amount_of_so def fetch_free_base(self,currency: str = ""): @@ -790,6 +801,9 @@ class trader: self.pause = True #To stop the main thread to iterate through this bot's orders (just in case) self.status_dict["pause_reason"] = "take_profit_routine - order handling" #start_trader will set this flag to False again once it starts + #Add the timestamp to the deals cache + self.deals_timestamps.append(time.time()) + #Let's do some type checking first if self.tp_order is None: self.status_dict["pause_reason"] = time.strftime(f"[%Y/%m/%d %H:%M:%S] | {self.pair} | TP order is None") @@ -1207,6 +1221,16 @@ class trader: return 0 + def check_boosted(self): + ''' + Checks if the trader qualifies for boost: + The last n deals must be within the last t seconds + ''' + + return len(self.deals_timestamps)>=self.boosted_deals_range and self.deals_timestamps[-1]-self.boosted_time_range<=self.deals_timestamps[-4] + + + def get_tp_level(self, order_index: int = 0) -> float: ''' Returns the correct take profit percentage, according to the strategy (config_dict["tp_mode"]): @@ -1219,27 +1243,11 @@ class trader: tp_level = 1 boost_percentage = 0 - #BOOST ROUTINE: If the trader closed certain amount of deals within the last t timespan, raise the take profit level by x% - #Default values - boosted_deals_range = 4 - boosted_time_range = 3600 - boosted_amount = .01 - - #Load config values (if present) - if "boosted_deals_range" in self.config_dict: - boosted_deals_range = self.config_dict["boosted_deals_range"] - if "boosted_time_range" in self.config_dict: - boosted_time_range = self.config_dict["boosted_time_range"] - if "boosted_amount" in self.config_dict: - boosted_amount = self.config_dict["boosted_amount"] - + #BOOST ROUTINE: If the trader closed certain amount of deals within the last t seconds, raise the take profit level by x% self.is_boosted = False - - if not self.is_short: - last_deals_timestamps = self.broker.return_last_n_deals_timestamps(self.pair,boosted_deals_range) - if len(last_deals_timestamps)==boosted_deals_range and all(item >= time.time()-boosted_time_range for item in last_deals_timestamps): - self.is_boosted = True - boost_percentage = boosted_amount + if not self.is_short and self.check_boosted(): + self.is_boosted = True + boost_percentage = self.boosted_amount if self.is_short or self.config_dict["tp_mode"]==0: #Fixed take profit percentage tp_level = self.config_dict["tp_level"]