diff --git a/changelog.txt b/changelog.txt index 2395a21..9bbc7b2 100755 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,14 @@ +2025.08.14: +. Refactored gib_so_size. +. Refactored seconds_to_time. +. Refactored linear_space. +. Refactored dca_cost_calculator. +. Refactored return_optimal_order_size. +. Minor refactor in generate_status_strings. +. Optimized imports. +. Deal_order_history now only stores the important parts of the orders to save some RAM. +. Removed deprecated "profit_to_file" method. + 2025.08.12: . Default "check_slippage" value now True. . Removed capitalization from exchange name when sending trader quit notification. diff --git a/config_handler.py b/config_handler.py index e5ed869..b4c61a2 100644 --- a/config_handler.py +++ b/config_handler.py @@ -1,5 +1,5 @@ -import json -import time +from time import time +from json import dumps, load class ConfigHandler: ''' @@ -41,7 +41,7 @@ class ConfigHandler: #Loads from disk the config file (if it exists) if self.load_from_file()==1: #If the config file does not exist, write a new one with the default values and sign it with timestamp. - self.config_dictionary["generated_at"] = int(time.time()) + self.config_dictionary["generated_at"] = int(time()) self.save_to_file() if config_dict is not None: self.config_dictionary = {**self.config_dictionary, **config_dict} @@ -315,7 +315,7 @@ class ConfigHandler: # return 1 try: with open(file_path, "w") as f: - f.write(json.dumps(self.config_dictionary, indent=4)) + f.write(dumps(self.config_dictionary, indent=4)) return 0 except Exception as e: self.broker.logger.log_this(f"Error saving config to file: {file_path}: {e}",1,self.get_pair()) @@ -329,7 +329,7 @@ class ConfigHandler: # return 1 try: with open(file_path, "r") as f: - self.set_config({**self.default_config_dictionary, **json.load(f)}) + self.set_config({**self.default_config_dictionary, **load(f)}) return 0 except Exception as e: self.broker.logger.log_this(f"Config file does not exist or is not readable: {e}",1,self.get_pair()) diff --git a/exchange_wrapper.py b/exchange_wrapper.py index 7927975..f0d203c 100755 --- a/exchange_wrapper.py +++ b/exchange_wrapper.py @@ -1,8 +1,8 @@ -import json import time -import requests import credentials import sqlite3 +from requests import get as requests_get +from json import load, dumps from copy import deepcopy @@ -290,7 +290,7 @@ class Broker: def reload_config_file(self): try: with open(self.config_filename) as f: - self.broker_config = json.load(f) + self.broker_config = load(f) except Exception as e: self.logger.log_this(f"Exception while reading the config file: {e}",1) @@ -340,9 +340,9 @@ class Broker: try: if backup: with open(f"{self.exchange}.bak","w") as c: - c.write(json.dumps(self.broker_config, indent=4)) + c.write(dumps(self.broker_config, indent=4)) with open(f"{self.config_filename}","w") as f: - f.write(json.dumps(self.broker_config, indent=4)) + f.write(dumps(self.broker_config, indent=4)) return 0 except Exception as e: self.logger.log_this(f"Problems writing the config file. Exception: {e}",1) @@ -1115,7 +1115,7 @@ class Logger: send_text = f"https://api.telegram.org/bot{tg_credentials['token']}/sendMessage?chat_id={tg_credentials['chatid']}&parse_mode=Markdown&text={message}" output = None if self.broker_config["telegram"] or ignore_config: - output = requests.get(send_text,timeout=5).json() #5 seconds timeout. This could also be a tunable. + output = requests_get(send_text,timeout=5).json() #5 seconds timeout. This could also be a tunable. if not output["ok"]: self.log_this(f"Error in send_tg_message: {output}") return 1 diff --git a/main.py b/main.py index 6884e5d..b341504 100644 --- a/main.py +++ b/main.py @@ -1,9 +1,9 @@ -import datetime -import json -import os -import sys import time import logging +from sys import argv +from os import _exit as os_exit +from json import load +from datetime import date from threading import Thread from waitress import serve @@ -16,7 +16,7 @@ import exchange_wrapper import trader -version = "2025.08.12" +version = "2025.08.14" ''' Color definitions. If you want to change them, check the reference at https://en.wikipedia.org/wiki/ANSI_escape_code#Colors @@ -42,14 +42,9 @@ def seconds_to_time(total_seconds: float) -> str: str: The formatted string ''' - time_delta = datetime.timedelta(seconds=total_seconds) - - hours = time_delta.seconds//3600 - remainder = time_delta.seconds%3600 - minutes = remainder//60 - seconds = remainder%60 - - return f"{time_delta.days}:{hours:02d}:{minutes:02d}:{seconds:02d}" + days = int(total_seconds // 86400) + h, m, sec = int((total_seconds % 86400) // 3600), int((total_seconds % 3600) // 60), int(total_seconds % 60) + return f"{days}:{h:02d}:{m:02d}:{sec:02d}" def time_to_unix(year: str, month: str, day: str) -> int: @@ -66,7 +61,7 @@ def time_to_unix(year: str, month: str, day: str) -> int: ''' try: - return int(time.mktime(datetime.date(int(year), int(month), int(day)).timetuple())) + return int(time.mktime(date(int(year), int(month), int(day)).timetuple())) except Exception as e: broker.logger.log_this(f"{e}") return 0 @@ -1603,7 +1598,7 @@ def unwrapped_load_old_long(base,quote): #Load the file try: with open(f"{base}{quote}.oldlong") as ol: - old_long = json.load(ol) + old_long = load(ol) except Exception as e: broker.logger.log_this(f"Exception while loading old_long file: {e}",1,f"{base}/{quote}") return jsonify({"Error": "old_long file of that pair does not exist."}) @@ -1640,7 +1635,7 @@ def unwrapped_view_old_long(base,quote,from_file): try: if int(from_file)==1: with open(f"{base}{quote}.oldlong") as ol: - old_long = json.load(ol) + old_long = load(ol) return jsonify(old_long) for instance in running_traders: if f"{base}/{quote}"==instance.config.get_pair(): @@ -2407,16 +2402,16 @@ if __name__=="__main__": #Loading config file print(time.strftime("[%Y/%m/%d %H:%M:%S] | Loading config file...")) try: - with open(sys.argv[1]) as f: - read_config = json.load(f) + with open(argv[1]) as f: + read_config = load(f) except Exception as e: print(e) print("Wrong syntax. Correct syntax is 'python3 main.py xxxxx.json (--first_start)', xxxxx.json being the config file.") - os._exit(1) + os_exit(1) #Check for import or load import_mode = True - if "--first_start" in sys.argv: + if "--first_start" in argv: import_mode = False print(time.strftime("[%Y/%m/%d %H:%M:%S] | Initializing in FIRST START MODE, press enter to start...")) else: @@ -2427,11 +2422,11 @@ if __name__=="__main__": exchange = set_exchange(read_config) if exchange is None: print("Error initializing exchange. Check spelling and/or the exchange configuration file.") - os._exit(1) + os_exit(1) #Creating the broker object print(time.strftime(f"[%Y/%m/%d %H:%M:%S] | Connecting to {str(exchange)}...")) - broker = exchange_wrapper.Broker(exchange,read_config,sys.argv[1]) #Also passes the config filename + broker = exchange_wrapper.Broker(exchange,read_config,argv[1]) #Also passes the config filename #Declaring some variables running_traders = [] @@ -2465,7 +2460,7 @@ if __name__=="__main__": toggle = input(f"This will initialize {len(broker.get_pairs())} instances, proceed? (Y/n) ") if toggle not in ["Y","y",""]: broker.logger.log_this("Aborting initialization",2) - os._exit(1) + os_exit(1) #broker.logger.log_this(f"Initializing {len(broker.get_pairs())} instances",2) for x in broker.get_pairs(): symbol = broker.get_symbol(x) @@ -2477,7 +2472,7 @@ if __name__=="__main__": toggle = input(f"This will import {len(broker.get_pairs())} instances, proceed? (Y/n) ") if toggle not in ["Y","y",""]: broker.logger.log_this("Aborting import",2) - os._exit(1) + os_exit(1) #broker.logger.log_this(f"Importing {len(broker.get_pairs())} instances",2) for x in broker.get_pairs(): symbol = broker.get_symbol(x) diff --git a/status_handler.py b/status_handler.py index e758cd1..6b1f6ee 100644 --- a/status_handler.py +++ b/status_handler.py @@ -1,5 +1,5 @@ -import json -import time +from time import strftime +from json import dumps, load class StatusHandler: ''' @@ -389,22 +389,40 @@ class StatusHandler: def update_deal_order_history(self, new_deal: dict): # if not isinstance(new_deal, dict): # self.broker.logger.log_this(f"value provided is not a dict",1,self.get_pair()) - self.status_dictionary["deal_order_history"].append(new_deal) + self.status_dictionary["deal_order_history"].append(self.strip_order(new_deal)) return 0 + def strip_order(self, order): + try: + stripped_order = {"id": order["id"], + "symbol": order["symbol"], + "type": order["type"], + "side": order["side"], + "price": float(order["price"]), + "amount": float(order["amount"]), + "filled": float(order["filled"]), + "cost": float(order["cost"]), + "remaining": float(order["remaining"]), + "timestamp": order["timestamp"], + "fees": order["fees"]} + return stripped_order + except Exception as e: + self.broker.logger.log_this(f"Error stripping order: {e}",2) + return order + def save_to_file(self, file_path = None, is_backup = False): if file_path is None: file_path = self.status_file_path if is_backup: try: - with open(time.strftime(f"{file_path}_%Y-%m-%d_%H:%M:%S.json"), "w") as f: - f.write(json.dumps(self.status_dictionary, indent=4)) + with open(strftime(f"{file_path}_%Y-%m-%d_%H:%M:%S.json"), "w") as f: + f.write(dumps(self.status_dictionary, indent=4)) except Exception as e: self.broker.logger.log_this(f"Error creating status backup file: {e}",1) try: with open(file_path, "w") as f: - f.write(json.dumps(self.status_dictionary, indent=4)) + f.write(dumps(self.status_dictionary, indent=4)) return 0 except Exception as e: self.broker.logger.log_this(f"Error saving status to file: {file_path}: {e}",1) @@ -415,7 +433,7 @@ class StatusHandler: file_path = self.status_file_path try: with open(file_path, "r") as f: - self.status_dictionary = {**self.default_status_dictionary, **json.load(f)} + self.status_dictionary = {**self.default_status_dictionary, **load(f)} return 0 except Exception as e: self.broker.logger.log_this(f"Error loading status from file: {file_path}: {e}",1) diff --git a/todo.txt b/todo.txt index 16ff8c2..e3ec0fc 100755 --- a/todo.txt +++ b/todo.txt @@ -12,6 +12,7 @@ Mandatory: 6. API documentation. 7. Implement api key hashing. 8. Dockerize. +9. Cache generated status strings, only recalculate when prices change. Would be nice to have: diff --git a/trader.py b/trader.py index 522d878..0f13353 100755 --- a/trader.py +++ b/trader.py @@ -1,7 +1,6 @@ -import csv -import json import time -import os +from os import path, remove +from json import dumps, load from config_handler import ConfigHandler from status_handler import StatusHandler @@ -22,9 +21,10 @@ class trader: self.broker = broker self.config = ConfigHandler(pair,broker) - self.base,self.quote = self.config.get_pair().split("/") + base_quote = self.config.get_pair() + self.base,self.quote = base_quote.split("/") self.status = StatusHandler(broker, self.base, self.quote) - self.market = self.broker.fetch_market(self.config.get_pair()) + self.market = self.broker.fetch_market(base_quote) self.market_load_time = int(time.time()) self.market_reload_period = 86400 #Market reload period in seconds self.status.set_start_time(int(time.time())) @@ -34,13 +34,14 @@ class trader: #Check if there is an old_long file. If so, load it. try: with open(f"status/{self.base}{self.quote}.oldlong") as ol: - self.status.set_old_long(json.load(ol)) + self.status.set_old_long(load(ol)) except Exception as e: - self.broker.logger.log_this(f"Exception: No old_long file. {e}",1,self.config.get_pair()) + self.broker.logger.log_this(f"Exception: No old_long file. {e}",1,base_quote) + self.profit_filename = f"profits/{self.base}{self.quote}.profits" self.log_filename = f"logs/{self.base}{self.quote}.log" - self.deals_timestamps = self.broker.get_trades_timestamps(self.config.get_pair(),self.config.get_boosted_time_range()) + self.deals_timestamps = self.broker.get_trades_timestamps(base_quote,self.config.get_boosted_time_range()) self.status.set_pause_reason("Initialization") if is_import: @@ -320,10 +321,10 @@ class trader: :param scalar: float :return: float ''' - total = order_size - for i in range(1,amount_of_so+1): - total+=self.gib_so_size(order_size,i,scalar) - return total + r = scalar * 100 + if abs(r - 1.0) < 1e-6: + return order_size * (amount_of_so + 1) + return order_size * (r * (r**amount_of_so - 1) / (r - 1)) + order_size def base_add_calculation(self, base_currency_amount: float, max_so: int = 100): @@ -358,35 +359,17 @@ class trader: :param scalar: float :return: float ''' - total_size = float(min_size) - - #Calculate optimal step size - self.broker.logger.log_this("Calculating optimal step size...",2,self.config.get_pair()) - #step = self.get_step_size() - #if step is None: - # step = min_size - #if step==0: - # step = min_size - #self.broker.logger.log_this(f"Step size is {step}",2,self.config.get_pair()) - - divisor = 10 - while divisor>0: - #step = self.broker.amount_to_precision(self.config.get_pair(),min_size/divisor) - step = min_size/divisor - if step!=0: #When using amount_to_precision, this comes handy. - break - divisor-=1 - - #if step==0: - # step = self.broker.amount_to_precision(self.config.get_pair(),min_size) - previous_size = 0 - self.broker.logger.log_this(f"Calculating optimal order size ...",2,self.config.get_pair()) - while True: #This loop should have a safeguard - total_cost = self.dca_cost_calculator(total_size,amount_of_safety_orders,scalar) - if total_cost>=amount: - return previous_size - previous_size = total_size - total_size+=step + low, high = min_size, amount + best = 0.0 + while high - low > min_size / 10: + mid = (low + high) / 2 + cost = self.dca_cost_calculator(mid, amount_of_safety_orders, scalar) + if cost <= amount: + best = mid + low = mid + else: + high = mid + return best def parse_fees(self, order: dict) -> tuple: @@ -570,7 +553,7 @@ class trader: }) try: with open(f"status/{self.base}{self.quote}.oldlong","w") as s: - s.write(json.dumps(self.status.get_old_long(),indent=4)) + s.write(dumps(self.status.get_old_long(),indent=4)) except Exception as e: self.broker.logger.log_this(f"Exception while saving old_long file: {e}",1,self.config.get_pair()) @@ -607,7 +590,7 @@ class trader: self.broker.logger.log_this("Can't find old long info on status_dict, searching for oldlong file",1,self.config.get_pair()) try: with open(f"status/{self.base}{self.quote}.oldlong") as f: - self.status.set_old_long(json.load(f)) + self.status.set_old_long(load(f)) except Exception as e: #self.write_to_log(time.strftime(f"[%Y/%m/%d %H:%M:%S] | {self.config.get_pair()} | Can't find old long file")) self.broker.logger.log_this(f"Can't file oldlong file. Exception: {e}",1,self.config.get_pair()) @@ -638,11 +621,11 @@ class trader: return 1 #Rewrite config file (if it exists) - if os.path.isfile(f"configs/{self.base}{self.quote}.bak") and os.path.isfile(f"configs/{self.base}{self.quote}.json"): + if path.isfile(f"configs/{self.base}{self.quote}.bak") and path.isfile(f"configs/{self.base}{self.quote}.json"): with open(f"configs/{self.base}{self.quote}.bak") as c: - old_config = json.load(c) + old_config = load(c) with open(f"configs/{self.base}{self.quote}.json","w") as c: - c.write(json.dumps(old_config, indent=4)) + c.write(dumps(old_config, indent=4)) if self.config.load_from_file()==1: self.config.reset_to_default() else: @@ -651,9 +634,9 @@ class trader: self.config.save_to_file() #Remove old_long file (if it exists) - if os.path.isfile(f"status/{self.base}{self.quote}.oldlong"): + if path.isfile(f"status/{self.base}{self.quote}.oldlong"): self.broker.logger.log_this("Removing old_long file...",2,self.config.get_pair()) - os.remove(f"status/{self.base}{self.quote}.oldlong") + remove(f"status/{self.base}{self.quote}.oldlong") #Set up a few variables self.status.set_fees_paid_in_quote(0) @@ -697,7 +680,6 @@ class trader: profit = already_received_quote + market_tp_order["cost"] - self.status.get_old_long()["quote_spent"] - self.status.get_old_long()["fees_paid_in_quote"] - fees_paid #Add profits to file and send telegram notifying profits - self.profit_to_file(profit,market_tp_order["id"]) self.profit_to_db(profit,market_tp_order["id"],self.broker.get_write_order_history()) self.broker.logger.log_this(f"Switch successful. Profit: {round(profit,2)} {self.quote}",0,self.config.get_pair()) self.broker.logger.log_this(f"Sell price: {market_tp_order['price']} {self.quote}",0,self.config.get_pair()) @@ -769,7 +751,6 @@ class trader: # Write the profit to file and send telegram message if profit>0: #Negative profits are not saved because the cleanup takes care of the unsold base currency (the notorious small change issue that plagues some exchanges) - self.profit_to_file(profit,filled_order["id"]) self.profit_to_db(profit,filled_order["id"],self.broker.get_write_order_history()) else: #For logging purposes self.broker.logger.log_this(f"NEGATIVE PROFIT - Total amount of base: {self.status.get_base_bought()}, base in the order: {filled_order['amount']}, base filled: {filled_order['filled']}, base 'profit': {base_profit}",1,self.config.get_pair()) @@ -1170,7 +1151,9 @@ class trader: ''' Returns a D:HH:MM:SS representation of total_seconds ''' - return f"{int(total_seconds / 86400)}:" + '%02d:%02d:%02d' % (int(total_seconds % 86400 / 3600), int(total_seconds % 3600 / 60), int(total_seconds % 60)) + days = int(total_seconds // 86400) + h, m, sec = int((total_seconds % 86400) // 3600), int((total_seconds % 3600) // 60), int(total_seconds % 60) + return f"{days}:{h:02d}:{m:02d}:{sec:02d}" def adjust_base(self): @@ -1212,28 +1195,14 @@ class trader: return 1 - def profit_to_file(self, amount: float, orderid: str) -> int: - ''' - Saves the profit to the corresponding profit file - DEPRECATED. Use profit_to_db instead. - ''' - try: - with open(self.profit_filename,"a") as profit_file: - profit_writer = csv.writer(profit_file, delimiter=",") - profit_writer.writerow([time.strftime("%Y-%m-%d"), amount, orderid]) - except Exception as e: - self.broker.logger.log_this(f"Exception in profit_to_file: {e}",1,self.config.get_pair()) - return 0 - - def profit_to_db(self, amount: float, orderid: str, write_deal_order_history: bool = False) -> int: ''' Saves the profit to the db in the format (pair,timestamp,profit,exchange_name,order_id,order_history) ''' - retries = self.broker.get_retries() + retries = 5 #Hardcoded because it's not an API call while retries>0: try: - order_history = json.dumps(self.status.get_deal_order_history()) if write_deal_order_history else "" + order_history = dumps(self.status.get_deal_order_history()) if write_deal_order_history else "" dataset = (time.time(),self.config.get_pair(),amount,self.broker.get_exchange_name(),str(orderid),order_history) #Write profit to cache self.broker.write_profit_to_cache(dataset) @@ -1241,7 +1210,7 @@ class trader: except Exception as e: self.broker.logger.log_this(f"Exception while writing profit: {e}",1,self.config.get_pair()) retries-=1 - time.sleep(self.broker.get_wait_time()) + time.sleep(.1) #Shorter wait time since it's not an API call return 1 @@ -1286,10 +1255,7 @@ class trader: Returns the correct safety order size depending on the number Scaling factor example: 5% = 0.0105 ''' - order_size = starting_order_size - for _ in range(so_number): - order_size = order_size*scaling_factor*100 - return order_size + return starting_order_size * (scaling_factor*100)**so_number def clip_value(self,value,lower_limit,upper_limit): @@ -1334,13 +1300,9 @@ class trader: - This is the only piece of code needed from Numpy - Only executed when calculating the safety order table, so there's no need for outstanding performance. ''' - result = [start] - if amount in [0,1]: - return result - step = (start-stop)/(amount-1) - for _ in range(1,amount): - result.append(result[-1]-step) - return result + + step = (stop - start) / (amount - 1) + return [start + i * step for i in range(amount)] def switch_quote_currency(self, new_quote: str) -> int: @@ -1426,7 +1388,7 @@ class trader: if self.config.get_is_short() and self.status.get_old_long()!={}: try: with open(f"status/{self.base}{self.quote}.oldlong","w") as c: - c.write(json.dumps(self.status.get_old_long(), indent=4)) + c.write(dumps(self.status.get_old_long(), indent=4)) except Exception as e: self.broker.logger.log_this(f"Exception while writing new old_long file: {e}",1,self.config.get_pair()) @@ -1512,9 +1474,9 @@ class trader: percentage_to_profit = 100 pct_to_profit_str = "XX.XX" - if self.status.get_price()!=0: - diff = abs(self.status.get_take_profit_price()-self.status.get_price()) - percentage_to_profit = diff/self.status.get_price()*100 + if mid_price!=0: + diff = abs(high_price-mid_price) + percentage_to_profit = diff/mid_price*100 #Formatting (on-screen percentage not longer than 4 digits) pct_to_profit_str = "{:.2f}".format(percentage_to_profit) @@ -1525,7 +1487,7 @@ class trader: line3 = "" if self.status.get_base_bought()!=0: - line3 = draw_line(self.status.get_price(),self.status.get_next_so_price(),self.status.get_take_profit_price(),self.status.get_quote_spent()/self.status.get_base_bought()) + line3 = draw_line(mid_price,low_price,high_price,self.status.get_quote_spent()/self.status.get_base_bought()) p = "*PAUSED*" if self.pause==True else "" low_boundary_color = red price_color = white @@ -1535,9 +1497,9 @@ class trader: price_color = white pair_color = yellow if self.status.get_old_long()!={}: - if self.status.get_price()>self.status.get_old_long()["tp_price"]: + if mid_price>self.status.get_old_long()["tp_price"]: price_color = bright_green - if self.status.get_take_profit_price()>self.status.get_old_long()["tp_price"]: + if high_price>self.status.get_old_long()["tp_price"]: target_price_color = bright_green #Set percentage's color @@ -1559,7 +1521,7 @@ class trader: if old_target-self.status.get_quote_spent()>0 and base_left>0 and minimum_switch_price