CPU optimizations

.gitignore

@@ -5,7 +5,6 @@ credentials.py
 DCAv2.code-workspace
 profits/my_database.db
 profits/profits_database.db
-profits/db_read.py
 utils/certs/
 utils/__pycache__/
 utils/close.py
@@ -21,4 +20,4 @@ upload_mainnet.sh
 utils/data/binance.db
 utils/data/okex.db
 utils/data/gateio.db
-utils/data/kucoin.db
+utils/data/kucoin.db

changelog.txt

@@ -1,3 +1,6 @@
+2024.11.11:
+. Refactored boost check to optimize CPU usage.
+
 2024.11.10:
 . Removed a double-summing bug in new_so_routine.
 

exchange_wrapper.py

@@ -65,6 +65,28 @@ class broker:
             return 1
 
     
+    def get_trades_timestamps(self,pair,timespan,no_retries=False):
+        '''
+        Returns the timestamps of the last trades from the database for the boosting algorithm
+        '''
+        
+        retries = self.retries
+        while retries>0:
+            try:
+                database_connection = sqlite3.connect("profits/profits_database.db")
+                database_cursor = database_connection.cursor()         
+                database_cursor.execute(f"SELECT * FROM profits_table WHERE timestamp >= {time.time()-timespan} ORDER BY timestamp DESC")
+                rows = database_cursor.fetchall()
+                return [item[0] for item in rows if item[1]==pair]
+            except Exception as e:
+                self.logger.log_this(f"Exception in preload_timestamps: {e}")
+                if no_retries:
+                    break
+                retries-=1
+                time.sleep(self.wait_time) 
+        return []
+        
+        
     def write_profit_to_db(self,dataset,no_retries=False):
         '''
         dataset format: (timestamp,pair,amount,exchange_name,order_id,order_history)
@@ -87,27 +109,27 @@ class broker:
         return 1
 
 
-    def return_last_n_deals_timestamps(self, pair, amount, no_retries=False):
+    # def return_last_n_deals_timestamps(self, pair, amount, no_retries=False):
-        '''
+    #     '''
-        Returns a list containing the last n deals timestamps.
+    #     Returns a list containing the last n deals timestamps.
-        '''
+    #     '''
         
-        retries = self.retries
+    #     retries = self.retries
-        while retries>0:
+    #     while retries>0:
-            try:
+    #         try:
-                database_connection = sqlite3.connect("profits/profits_database.db")
+    #             database_connection = sqlite3.connect("profits/profits_database.db")
-                database_cursor = database_connection.cursor()                
+    #             database_cursor = database_connection.cursor()                
-                database_cursor.execute(f"SELECT * FROM profits_table WHERE pair = '{pair}' ORDER BY timestamp DESC LIMIT {amount};")
+    #             database_cursor.execute(f"SELECT * FROM profits_table WHERE pair = '{pair}' ORDER BY timestamp DESC LIMIT {amount};")
-                rows = database_cursor.fetchall()
+    #             rows = database_cursor.fetchall()
-                database_connection.close()
+    #             database_connection.close()
-                return [item[0] for item in rows]
+    #             return [item[0] for item in rows]
-            except Exception as e:
+    #         except Exception as e:
-                self.logger.log_this(f"Exception in return_last_n_deals_timestamps: {e}",1)
+    #             self.logger.log_this(f"Exception in return_last_n_deals_timestamps: {e}",1)
-                if no_retries:
+    #             if no_retries:
-                    break
+    #                 break
-                retries-=1
+    #             retries-=1
-                time.sleep(self.wait_time)
+    #             time.sleep(self.wait_time)
-        return []
+    #     return []
 
 
     def check_for_duplicate_profit_in_db(self,order,no_retries=False):

main.py

@@ -22,7 +22,7 @@ In case the permissions of the certificate changes, reset them this way:
 # ll /etc/letsencrypt/
 '''
 
-version = "2024.11.10"
+version = "2024.11.11"
 
 '''
 Color definitions. If you want to change them, check the reference at https://en.wikipedia.org/wiki/ANSI_escape_code#Colors

profits/db_read.py

@@ -1,25 +0,0 @@
-import sqlite3
-import time
-import json
-
-# Connect to the SQLite database
-conn = sqlite3.connect('profits_database.db')
-cursor = conn.cursor()
-
-# Execute a SELECT query to retrieve data from the database
-cursor.execute('SELECT * FROM profits_table')
-
-# Fetch all rows from the result set
-rows = cursor.fetchall()
-
-# Process the fetched rows
-#for row in rows:
-    #human_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(row[0])))
-    #print(human_time,row[1],round(row[2],2),row[3])  # Or do whatever processing you need
-
-#data = json.loads(rows[-3][-1])
-print([item[0] for item in rows[-4:]])
-print(f"Number of entries: {len(rows)}")
-
-# Close the connection
-conn.close()

todo.txt

@@ -2,7 +2,7 @@ Mandatory:
 =========
 0. Mobile app.
 1. Stats webpage.
-2. Instead of giving a list of order_ids to each trader, give a list of the open orders and that's it (for easier future development, partial order fills for example)
+2. Instead of providing a list of order_ids to each trader, provide a list of the open orders for easier future development, including partial order fills.
 3. Maintain local orderbooks for each trading pair, which enables:
     3a. Smart order pricing: Prioritization of fill speed over instant profit or vice versa
 4. Consolidate vocabulary (trader, pair and bot; instance & trader)

trader.py

@@ -75,6 +75,17 @@ class trader:
                 self.broker.logger.log_this(f"Exception: No old_long file. {e}",1,self.pair)
         self.profit_filename = f"profits/{self.base}{self.quote}.profits"
         self.log_filename = f"logs/{self.base}{self.quote}.log"
+        
+        self.boosted_deals_range = 4
+        self.boosted_time_range = 3600
+        self.boosted_amount = .01
+        if "boosted_deals_range" in self.config_dict:
+            self.boosted_deals_range = self.config_dict["boosted_deals_range"]
+        if "boosted_time_range" in self.config_dict:
+            self.boosted_time_range = self.config_dict["boosted_time_range"]
+        if "boosted_amount" in self.config_dict:
+            self.boosted_amount = self.config_dict["boosted_amount"]        
+        self.deals_timestamps = self.broker.get_trades_timestamps(self.pair,self.boosted_time_range)
 
         self.stop_when_profit = False
         self.status_dict["pause_reason"] = "Initialization"        
@@ -535,7 +546,7 @@ class trader:
             self.pause = False
             self.status_dict["pause_reason"] = ""
             return None,None
-        return optimal_order_size,amount_of_so            
+        return optimal_order_size,amount_of_so
 
 
     def fetch_free_base(self,currency: str = ""):
@@ -790,6 +801,9 @@ class trader:
         self.pause = True                                       #To stop the main thread to iterate through this bot's orders (just in case)
         self.status_dict["pause_reason"] = "take_profit_routine - order handling"     #start_trader will set this flag to False again once it starts
         
+        #Add the timestamp to the deals cache
+        self.deals_timestamps.append(time.time())
+        
         #Let's do some type checking first
         if self.tp_order is None:
             self.status_dict["pause_reason"] = time.strftime(f"[%Y/%m/%d %H:%M:%S] | {self.pair} | TP order is None")
@@ -1207,6 +1221,16 @@ class trader:
         return 0
 
 
+    def check_boosted(self):
+        '''
+        Checks if the trader qualifies for boost:
+        The last n deals must be within the last t seconds
+        '''
+        
+        return len(self.deals_timestamps)>=self.boosted_deals_range and self.deals_timestamps[-1]-self.boosted_time_range<=self.deals_timestamps[-4]
+            
+
+
     def get_tp_level(self, order_index: int = 0) -> float:
         '''
         Returns the correct take profit percentage, according to the strategy (config_dict["tp_mode"]):
@@ -1219,27 +1243,11 @@ class trader:
         tp_level = 1
         boost_percentage = 0
         
-        #BOOST ROUTINE: If the trader closed certain amount of deals within the last t timespan, raise the take profit level by x%
+        #BOOST ROUTINE: If the trader closed certain amount of deals within the last t seconds, raise the take profit level by x%
-        #Default values
-        boosted_deals_range = 4
-        boosted_time_range = 3600
-        boosted_amount = .01
-        
-        #Load config values (if present)
-        if "boosted_deals_range" in self.config_dict:
-            boosted_deals_range = self.config_dict["boosted_deals_range"]
-        if "boosted_time_range" in self.config_dict:
-            boosted_time_range = self.config_dict["boosted_time_range"]
-        if "boosted_amount" in self.config_dict:
-            boosted_amount = self.config_dict["boosted_amount"]
-        
         self.is_boosted = False
-        
+        if not self.is_short and self.check_boosted():
-        if not self.is_short:
+            self.is_boosted = True
-            last_deals_timestamps = self.broker.return_last_n_deals_timestamps(self.pair,boosted_deals_range)
+            boost_percentage = self.boosted_amount
-            if len(last_deals_timestamps)==boosted_deals_range and all(item >= time.time()-boosted_time_range for item in last_deals_timestamps):
-                self.is_boosted = True
-                boost_percentage = boosted_amount
             
         if self.is_short or self.config_dict["tp_mode"]==0:                 #Fixed take profit percentage
             tp_level = self.config_dict["tp_level"]