statistics_server:

. Added guards when calculating averages
. Fixed useless variable assignment in fetch_full_log
. Streamlined profit queries
. Optimized last_n_lines
. Cleaned up db connection code

changelog.txt

@@ -6,6 +6,7 @@
 . Decorator for api endpoints error handling
 . Input validation in api parameters
 . credentials.py import fallback
+. open_orders prefiltered before hitting the traders
 
 2026.06.03:
 . Fixed tp mode 2 non-functional

main.py

@@ -336,9 +336,12 @@ def main_routine():
                 pairs_to_fetch.append(instance.status.get_pair())
             
         open_orders = broker.fetch_open_orders(pairs_to_fetch)
+        open_orders_by_pair = {}
+        for order in open_orders:
+            open_orders_by_pair.setdefault(order["symbol"], []).append(order)
         with traders_lock:        
             for instance in running_traders:
-                future = executor.submit(instance.check_status, open_orders)
+                future = executor.submit(instance.check_status, open_orders_by_pair)
                 futures.append(future)
                 online_pairs.append(f"{instance.base}{instance.quote}")
         

trader.py

@@ -36,10 +36,11 @@ class trader:
         self.base,self.quote = base_quote.split("/")
         self.status = StatusHandler(broker, self.base, self.quote)        
         self.market = self.broker.fetch_market(base_quote)
-        self.market_load_time = int(time.time())
+        now = time.time()
+        self.market_load_time = int(now)
         self.market_reload_period = 86400       #Market reload period in seconds
-        self.status.set_start_time(int(time.time()))
-        self.last_time_seen = time.time()
+        self.status.set_start_time(int(now))
+        self.last_time_seen = now
         
         self.base_diff_check = False        
         self.min_base_difference = .1   #Percentage difference between base in the tp order and its adjusted amount that triggers a recheck.
@@ -316,8 +317,9 @@ class trader:
             self.status.set_is_paused(self.pause)
             self.status.set_is_short(self.config.get_is_short())
             #self.status.set_no_of_safety_orders(self.config.get_no_of_safety_orders())
-            self.status.set_deal_uptime(int(time.time()) - self.status.get_deal_start_time())
-            self.status.set_total_uptime(int(time.time()) - self.status.get_start_time())
+            now = int(time.time())
+            self.status.set_deal_uptime(now - self.status.get_deal_start_time())
+            self.status.set_total_uptime(now - self.status.get_start_time())
             self.status.set_tp_mode(self.config.get_tp_mode())
             self.status.set_profit_table(self.config.get_tp_table())
             self.status.set_autoswitch(self.config.get_autoswitch())
@@ -1099,7 +1101,7 @@ class trader:
         return (base_left*price)+self.status.get_quote_spent()>=old_target
         
 
-    def check_status(self,open_orders: list) -> int:         #Should I change the order? Check the SO first?
+    def check_status(self,open_orders: dict) -> int:         #Should I change the order? Check the SO first?
         '''
         Main routine. It checks for closed orders and proceeds accordingly.
         '''
@@ -1128,7 +1130,8 @@ class trader:
 
         #Extract ids from order list
         self.status.set_pause_reason("filtering open orders")
-        open_orders_list = [order for order in open_orders if order["symbol"]==self.status.get_pair()]
+        #open_orders_list = [order for order in open_orders if order["symbol"]==self.status.get_pair()]
+        open_orders_list = open_orders.get(self.status.get_pair(),[])
         open_orders_ids = [order["id"] for order in open_orders_list]
 
         self.status.set_pause_reason("check if tp_order is valid")
@@ -1246,10 +1249,11 @@ class trader:
         self.status.set_status_string(self.generate_status_strings())
         
         #Wrap up
-        self.status.set_deal_uptime(int(time.time()) - self.status.get_deal_start_time())
-        self.status.set_total_uptime(int(time.time()) - self.status.get_start_time())
+        now = int(time.time())
+        self.status.set_deal_uptime(now - self.status.get_deal_start_time())
+        self.status.set_total_uptime(now - self.status.get_start_time())
         self.update_status(False)
-        self.last_time_seen = int(time.time())
+        self.last_time_seen = now
         
         return 0
 

utils/statistics_server_v3.py

@@ -16,12 +16,7 @@ _local_storage = threading.local()
 def get_db_connection():
     current_time = time.time()
     
-    if not hasattr(_local_storage, 'connection') or not hasattr(_local_storage, 'created_at') or (current_time - _local_storage.created_at) > 3600:  # Reconnect every hour
-        if hasattr(_local_storage, 'connection'):
-            try:
-                _local_storage.connection.close()
-            except:
-                pass
+    if not hasattr(_local_storage, 'connection'):
         _local_storage.connection = sqlite3.connect(profits_database, check_same_thread=False)
         _local_storage.connection.row_factory = sqlite3.Row
         _local_storage.created_at = current_time
@@ -121,9 +116,11 @@ def profit_report():
 
     #Projection calculation
     days_in_month = calendar.monthrange(datetime.date.today().year, datetime.date.today().month)[1]
-    daily_combined_media = (last_30_days[0][1]/30+last_7_days[0][1]/7)/2
-    current_amount = last_n_months_rows[-1][1]
-    days_past_this_month = int(last_60_days_rows[-1][0][8:10])
+    daily_30_avg = last_30_days[0][1]/30 if last_30_days else 0
+    daily_7_avg = last_7_days[0][1]/7 if last_7_days else 0
+    daily_combined_media = (daily_30_avg + daily_7_avg)/2 if (daily_30_avg or daily_7_avg) else 0
+    current_amount = last_n_months_rows[-1][1] if last_n_months_rows else 0
+    days_past_this_month = int(last_60_days_rows[-1][0][8:10]) if last_60_days_rows else 0
     
     #Per exchange
     binance_amount = 0
@@ -150,18 +147,16 @@ def profit_report():
     
     last_60_days_result = {row[0]: round(row[1],2) for row in last_60_days_rows}
     last_18_months_result = {row[0]: round(row[1],2) for row in last_n_months_rows}
-    last_30_days_average = last_30_days[0][1]/30
-    last_7_days_average = last_7_days[0][1]/7
     this_month_projection = current_amount + daily_combined_media*(days_in_month-days_past_this_month)
-    binance_percentage = binance_amount/total_amount*100
-    gateio_percentage = gateio_amount/total_amount*100
-    kucoin_percentage = kucoin_amount/total_amount*100
-    okex_percentage = okex_amount/total_amount*100
+    binance_percentage = binance_amount/total_amount*100 if total_amount else 0
+    gateio_percentage = gateio_amount/total_amount*100 if total_amount else 0
+    kucoin_percentage = kucoin_amount/total_amount*100 if total_amount else 0
+    okex_percentage = okex_amount/total_amount*100 if total_amount else 0
 
     return {"Last 60 days": last_60_days_result, 
             "Last 18 months": last_18_months_result, 
-            "Last 30 days average": last_30_days_average, 
-            "Last 7 days average": last_7_days_average, 
+            "Last 30 days average": daily_30_avg, 
+            "Last 7 days average": daily_7_avg, 
             "This month projection": this_month_projection,
             "Binance": binance_amount,
             "Binance percentage": binance_percentage,
@@ -182,22 +177,15 @@ def query_total_profit(pair=None, start_date=0):
 
     if pair is None:
         query = "SELECT SUM(amount) AS total_profit FROM profits_table WHERE timestamp >= ?"
-        with db_cursor() as cursor:
-            cursor.execute(query, (start_date,))
-            query_result = cursor.fetchall()
-        return query_result[0][0]
+        params = (start_date,)
     else:
-        query = """SELECT pair, SUM(amount) AS total_profit
-            FROM profits_table
-            WHERE timestamp >= ?
-            GROUP BY pair;"""        
-        with db_cursor() as cursor:
-            cursor.execute(query, (start_date,))
-            query_result = cursor.fetchall()
-        for item in query_result:
-            if item[0].replace("/","")==pair:
-                return item[1]
-    return 0
+        query = """SELECT pair, SUM(amount) AS total_profit FROM profits_table WHERE timestamp >= ? AND pair = ?"""        
+        params = (start_date, pair)
+    
+    with db_cursor() as cursor:
+        cursor.execute(query, params)
+        result = cursor.fetchone()
+        return result[0] if result[0] else 0
 
 
 def daily_and_monthly_totals() -> tuple[float, float]:
@@ -317,20 +305,20 @@ def last_n_lines(file_name,width,amount=4,full_log=False):
     file_contents = []
     result = []
 
-    with open(file_name) as f:
-        file_contents = f.readlines()
-    
     if full_log:
+        with open(file_name) as f:
+            file_contents = f.readlines()        
         for line in file_contents:
             result.append(line.strip())
         return result,len(file_contents)
 
-    for line in file_contents[::-1][:amount]:
+    contents,amount_of_lines = tail_log(file_name,amount)
+    for line in contents:
         trimmed = line.strip()
         result.append(trimmed[:width])
         if len(trimmed)>width:
             result.append(trimmed[width:width*2])
-    return result[:amount],len(file_contents)
+    return result,amount_of_lines
 
 
 def tail_log(filename, lines=200):
@@ -423,15 +411,14 @@ def fetch_full_log():
     It trims the full log to 200 lines, to avoid sending too much data to the client.
     '''
     if not "X-API-KEY" in request.headers or not request.headers.get("X-API-KEY") in get_valid_keys():
-        return jsonify({'Error': 'API key invalid'}), 401   
+        return jsonify({'Error': 'API key invalid'}), 401
     try:
         exchange_name = request.args.get("exchange_name")
-        width = 0
         last_lines, amount_of_lines = tail_log(f"../logs/{exchange_name}.log", 200)
         return jsonify({"line": last_lines[-200:], "amount_of_lines": amount_of_lines})
     except Exception as e:
         print(e)
-        return {"line": [""]*width,"amount_of_lines": 0}
+        return {"line": [""]*10,"amount_of_lines": 0}
 
 
 @stats_api.route("/fetch_log")
@@ -522,19 +509,28 @@ def get_averages():
     if not "X-API-KEY" in request.headers or not request.headers.get("X-API-KEY") in get_valid_keys():
         return jsonify({'Error': 'API key invalid'}), 401
     try:
-        daily_totals = query_daily_totals()
-        val_30 = 0
-        val_7 = 0
-        recent_days = sorted(daily_totals.keys(), reverse=True)[:30]  
-        acc_30 = [daily_totals[date] for date in recent_days[:30]]
-        acc_7 = [daily_totals[date] for date in recent_days[:7]]
-        length_30 = min(30,len(acc_30))           #Last 30 days
-        length_7 = min(7,len(acc_7))             #Last 7 days
-        for _ in range(length_30):
-            val_30 += acc_30.pop()
-        for _ in range(length_7):
-            val_7 += acc_7.pop()
-        return jsonify({"30_day": val_30/length_30, "7_day": val_7/length_7})
+        with db_cursor() as cursor:
+            cursor.execute("""SELECT
+                        COALESCE(SUM(CASE WHEN timestamp >= ? THEN amount END), 0) AS sum_30d,
+                        COALESCE(SUM(CASE WHEN timestamp >= ? THEN amount END), 0) AS sum_7d,
+                        FROM profits_table""",
+                        (time.time()-30*86400, time.time()-7*86400))
+            row = cursor.fetchone()
+            sum_30d, sum_7d = float(row["sum_30d"]), float(row["sum_7d"])
+            return jsonify({"30_day": sum_30d/30, "7_day": sum_7d/7})
+        # daily_totals = query_daily_totals()
+        # val_30 = 0
+        # val_7 = 0
+        # recent_days = sorted(daily_totals.keys(), reverse=True)[:30]  
+        # acc_30 = [daily_totals[date] for date in recent_days[:30]]
+        # acc_7 = [daily_totals[date] for date in recent_days[:7]]
+        # length_30 = min(30,len(acc_30))           #Last 30 days
+        # length_7 = min(7,len(acc_7))             #Last 7 days
+        # for _ in range(length_30):
+        #     val_30 += acc_30.pop()
+        # for _ in range(length_7):
+        #     val_7 += acc_7.pop()
+        # return jsonify({"30_day": val_30/length_30, "7_day": val_7/length_7})
     except Exception as e:
         print(e)
         return jsonify({'Error': 'Halp'})