diff --git a/changelog.txt b/changelog.txt index d3dd01a..2686f22 100755 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,7 @@ +2024.11.26: +. Implemented deals cache to reduce db load until the new database service is implemented. +. Added a new API endpoint: /get_deals_cache. + 2024.11.25: . Implemented a short log list: In order to avoid constant log file queries, a list of the last few log entries is stored in memory and it's returned via /get_log_list API endpoint. diff --git a/exchange_wrapper.py b/exchange_wrapper.py index dea00e8..d1075ff 100755 --- a/exchange_wrapper.py +++ b/exchange_wrapper.py @@ -25,7 +25,8 @@ class broker: self.logger = logger(self.read_config) self.write_order_history = True #This should be a toggle in config_file - self.database_connection = sqlite3.connect("profits/profits_database.db") + self.profits_database_filename = "profits/profits_database.db" + self.database_connection = sqlite3.connect(self.profits_database_filename) self.database_cursor = self.database_connection.cursor() self.database_cursor.execute(''' CREATE TABLE IF NOT EXISTS profits_table ( @@ -41,7 +42,29 @@ class broker: self.database_connection.close() self.exchange.load_markets() + + #Populates deals cache + self.deals_cache_length = 10 + self.deals_list = self.preload_deals(amount_to_preload=self.deals_cache_length) + + def preload_deals(self,amount_to_preload=10): + ''' + Reads the last n deals from the database and returns them in a list + ''' + connection = sqlite3.connect(self.profits_database_filename) + cursor = connection.cursor() + cursor.execute(f"SELECT * FROM profits_table WHERE exchange_name = ? ORDER BY timestamp DESC LIMIT ?", (self.get_exchange_name(), amount_to_preload)) + result = cursor.fetchall() + connection.close() + + return [(row[0],row[1],row[2],row[3],row[4],"") for row in result] + + + def get_deals_cache(self): + return self.deals_list + + def all_markets(self,no_retries=False): retries = self.retries while retries>0: @@ -73,7 +96,7 @@ class broker: retries = self.retries while retries>0: try: - database_connection = sqlite3.connect("profits/profits_database.db") + database_connection = sqlite3.connect(self.profits_database_filename) database_cursor = database_connection.cursor() database_cursor.execute(f"SELECT * FROM profits_table WHERE timestamp >= {time.time()-timespan} ORDER BY timestamp") rows = database_cursor.fetchall() @@ -87,6 +110,16 @@ class broker: return [] + def write_profit_to_cache(self,dataset): + ''' + dataset format: (timestamp,pair,amount,exchange_name,order_id,order_history) + ''' + + self.deals_list.insert(0,(dataset[0],dataset[1],dataset[2],dataset[3],dataset[4],"")) + self.deals_list = self.deals_list[:self.deals_cache_length] + return 0 + + def write_profit_to_db(self,dataset,no_retries=False): ''' dataset format: (timestamp,pair,amount,exchange_name,order_id,order_history) @@ -94,7 +127,7 @@ class broker: retries = self.retries while retries>0: try: - database_connection = sqlite3.connect("profits/profits_database.db") + database_connection = sqlite3.connect(self.profits_database_filename) database_cursor = database_connection.cursor() database_cursor.execute('INSERT INTO profits_table VALUES(?, ?, ?, ?, ?, ?)', dataset) database_connection.commit() @@ -118,7 +151,7 @@ class broker: retries = self.retries while retries>0: try: - database_connection = sqlite3.connect("profits/profits_database.db") + database_connection = sqlite3.connect(self.profits_database_filename) database_cursor = database_connection.cursor() database_cursor.execute(f"SELECT * FROM profits_table WHERE pair = '{order['symbol']}' ORDER BY timestamp DESC LIMIT 1;") rows = database_cursor.fetchall() @@ -324,7 +357,7 @@ class broker: retries = self.retries while retries>0: try: - if self.read_config["exchange"]=="binance": + if self.get_exchange_name()=="binance": a = self.exchange.fetch_last_prices(pair_list) return {x: a[x]["price"] for x in a.keys()} else: @@ -465,7 +498,7 @@ class broker: pairs = [] try: orders = [] - if self.read_config["exchange"]=="binance": + if self.get_exchange_name()=="binance": orders = self.get_opened_orders_binance(pairs) else: orders = self.get_opened_orders() @@ -487,7 +520,7 @@ class broker: pairs = [] try: #id_list = [] - if self.read_config["exchange"]=="binance": + if self.get_exchange_name()=="binance": return self.get_opened_orders_binance(pairs) return self.get_opened_orders() #else: @@ -625,7 +658,7 @@ class broker: pair = symbol while retries>0: try: - if self.read_config["exchange"]=="gateio" and side=="buy" and not amount_in_base: + if self.get_exchange_name()=="gateio" and side=="buy" and not amount_in_base: new_order = self.exchange.create_market_buy_order_with_cost(pair, size) else: order_book = self.get_order_book(symbol) diff --git a/main.py b/main.py index 099febc..e947082 100644 --- a/main.py +++ b/main.py @@ -22,7 +22,7 @@ In case the permissions of the certificate changes, reset them this way: # ll /etc/letsencrypt/ ''' -version = "2024.11.25" +version = "2024.11.26" ''' Color definitions. If you want to change them, check the reference at https://en.wikipedia.org/wiki/ANSI_escape_code#Colors @@ -1102,6 +1102,20 @@ def get_log_list(): return jsonify({'Error': 'API key invalid'}), 401 +@base_api.route("/get_deals_cache", methods=['GET']) +def get_deals_cache(): + ''' + GET request + + Parameters: + None + ''' + + if "X-API-KEY" in request.headers and request.headers.get("X-API-KEY") in valid_keys: + return unwrapped_get_deals_cache() + return jsonify({'Error': 'API key invalid'}), 401 + + @base_api.route("/trader_time", methods=['GET']) def trader_time(): ''' @@ -2020,6 +2034,20 @@ def unwrapped_get_log_list(): return jsonify({"Logs": broker.logger.get_log_list()}) +def unwrapped_get_deals_cache(): + ''' + Retrieves the last n entries from the broker's logger. + This list is kept on memory, to avoid having to read the log file every time. + + Parameters: + None + + Returns: + jsonify: A jsonified dictionary containing the last n entries from the log file. + ''' + return jsonify({"Deals": broker.get_deals_cache()}) + + def unwrapped_call_wait_time(wait_time): ''' Modifies the time between some API calls and retries. diff --git a/todo.txt b/todo.txt index 42a4e3f..983ce56 100755 --- a/todo.txt +++ b/todo.txt @@ -7,8 +7,7 @@ Mandatory: 3. Consolidate vocabulary (trader, pair and bot; instance & trader) 4. Base add for short traders. 5. Proper handling of order price too high/low in OKX (rare, it happens when under heavy volatility). -6. Keep a copy of the instance's last n log entries on RAM, to speed up querying. -7. Do the same for the last n deals. Load a few from the db at instance initialization. +6. Redo all the database code. It should be a separate service (running locally or not) that all instances can access. Would be nice to have: diff --git a/trader.py b/trader.py index 8a5e61e..45e0257 100755 --- a/trader.py +++ b/trader.py @@ -1327,6 +1327,8 @@ class trader: try: order_history = json.dumps(self.status_dict["deal_order_history"]) if write_deal_order_history else "" dataset = (time.time(),self.pair,amount,self.broker.get_exchange_name(),str(orderid),order_history) + #Write profit to cache + self.broker.write_profit_to_cache(dataset) return self.broker.write_profit_to_db(dataset) except Exception as e: self.broker.logger.log_this(f"Exception while writing profit: {e}",1,self.pair)