From e43c98997b0de6a8da989f68d97f1d4f7fb7bb6f Mon Sep 17 00:00:00 2001 From: kboz Date: Fri, 24 Apr 2026 00:55:26 +0200 Subject: feat(core): implement wallet trade tracking and sqlite market caching - Transitioned primary ESI networking and dataset initialization to `asyncio` and `aiohttp` for non-blocking I/O. - Introduced `aiosqlite` utilizing an in-memory database to significantly speed up arbitrage querying. - Implemented character wallet tracking (added `esi-wallet.read_character_wallet.v1` scope) to automatically deduct filled market volumes internally instead of waiting for full region polling. - Exposed `/settings` endpoint allowing dynamic tweaking of threshold variables (cargo size, limits, tax rate, ship speeds) directly from the React UI. - Decoupled auto-waypoint submission during route planning; users now push routes selectively via a new "Set in Game" frontend action and `/set_waypoints` backend logic. - Optimized order evaluations and triplet generation limits through `LegMetrics` to lower CPU footprint drastically on large dataset passes. - Migrated React frontend completely to Server-Sent Events (SSE), cleanly ditching HTTP polling. --- CHANGELOG.md | 12 + backend/src/arbitrage.py | 303 +++++++++++++++++--------- backend/src/esi/api.py | 336 ++++++++++++++++++++-------- backend/src/esi/shared_flow.py | 1 + backend/src/graph.py | 231 +++++++++++++++----- backend/src/main.py | 37 +++- backend/src/manager.py | 483 ++++++++++++++++++++++++++++++++++++++--- backend/src/market.py | 255 ++++++++++++++++------ backend/src/pathfinder.py | 61 ++---- backend/src/route.py | 330 +++++++++++++++++++++++----- frontend/src/App.css | 9 - frontend/src/App.js | 8 +- frontend/src/Info.js | 5 +- frontend/src/InfoDisplay.js | 2 +- frontend/src/Options.js | 170 +++++++++++++-- frontend/src/Path.js | 5 +- readme.md | 38 ++-- scripts/setup_auth.py | 2 + 18 files changed, 1777 insertions(+), 511 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c3b3c23..2e05e7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,12 +8,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- Implemented `WalletTransaction` tracking and `get_wallet_transactions` in `backend/src/esi/api.py` to reconcile and deduct filled orders locally without requiring full region re-polls. +- Requested `esi-wallet.read_character_wallet.v1` scope via SSO `scripts/setup_auth.py` to enable route confirmation. +- Introduced an in-memory SQLite database via `aiosqlite` in `backend/src/market.py` for efficient arbitrage querying and volume state updates. +- Added `GET /settings`, `POST /settings`, and `POST /set_waypoints` FastAPI endpoints in `backend/src/main.py`. +- Developed `LegMetrics` and graph caching mechanism in `backend/src/graph.py` to dramatically speed up route optimization. +- Upgraded the React frontend `Options.js` to allow real-time UI configuration of cargo size, speeds, risk values, and profit thresholds. - `scripts/haul.sh` CLI for unified backend/frontend dependency checking, running, and EVE data bootstrapping. - `scripts/setup_auth.py` and `scripts/setup_eve_data.py` scripts to fetch OAuth credentials and base CSV game data. - `.env.example` to ease SSO configuration. - Custom Copilot Chat prompt for documenting changes under `.github/prompts`. ### Changed +- Migrated all primary ESI networking and dataset initialization in `backend/src/` to `asyncio` and `aiohttp` for non-blocking I/O. +- Redesigned `create_route` no longer auto-sets waypoints; players must now use the explicit "Set in Game" button to push route locations to the client. +- Switched frontend components (`Path.js`, `Info.js`) from HTTP polling to strictly using Server-Sent Events (`useSSE`) for real-time dataset rendering. +- Rewrote the arbitrage shortlist capabilities using SQLite queries ranking to limit calculation matrix overhead strictly. +- Overhauled triplet routing optimizations (`backend/src/route.py`) to reduce duplicate calculations when evaluating top candidates. - Refactored `backend/src/manager.py` to fix graph instantiations, updated ship properties to feature larger cargo, lower subwarp/warp speed, and accurately lower ship ISK value. - Re-aligned route and item capital checks across `backend/src/manager.py` and `backend/src/pathfinder.py` by transitioning the `Ship` attribute references from dataclasses to dictionaries. - Reduced initial max capital layout cap to `20,000,000` in `backend/src/arbitrage.py` to be more cautious about early-game up-front ISK demands. @@ -22,6 +33,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Adjusted `.gitignore` to track standard Python and Node virtual environment directories properly. ### Fixed +- Addressed ESI rate limit errors by enforcing an active semaphore limit and 429 backoff mechanism inside ESI client requests. - Fixed uninitialized class name alias by swapping `Graph` to `NavigationGraph` across references. - Remedied route step structure by moving away from hardcoded dataclass references (`RouteStep`) to generic dictionaries in `backend/src/route.py`. - Checked and resolved printing of `snipe` variable references within `backend/src/manager.py` loops arrays versus strings. diff --git a/backend/src/arbitrage.py b/backend/src/arbitrage.py index c1a0459..2226a17 100644 --- a/backend/src/arbitrage.py +++ b/backend/src/arbitrage.py @@ -2,15 +2,22 @@ import logging from dataclasses import dataclass from typing import Optional, List -from market import Order +from market import Market logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) -TAX_RATE = 0.08 -PROFIT_THRESHOLD = 100_000 -MAX_CAPITAL = 20_000_000 -SNIPE_PROFIT_THRESHOLD = 20_000_000 +ARBITRAGE_CANDIDATE_LIMIT = 30_000 +ARBITRAGE_TYPE_LIMIT = 250 +ARBITRAGE_STATION_PAIR_LIMIT = 40 + + +@dataclass(frozen=True) +class ShortlistConfig: + candidate_limit: int = ARBITRAGE_CANDIDATE_LIMIT + type_limit: int = ARBITRAGE_TYPE_LIMIT + station_pair_limit: int = ARBITRAGE_STATION_PAIR_LIMIT + rank_mode: str = "and" @dataclass class Trade: @@ -28,114 +35,196 @@ class Trade: gross_profit: float net_profit: Optional[float] = None -def arbitrage(orders: List[Order], cargo_space: int) -> List[Trade]: - sell_orders = sorted( - (order for order in orders if not order.is_buy_order), - key=lambda x: x.price +async def arbitrage( + market: Market, + cargo_space: int, + tax_rate: float, + profit_threshold: float, + max_capital: float, + shortlist: Optional[ShortlistConfig] = None, +) -> List[Trade]: + if not market.db_conn: + return [] + + shortlist = shortlist or ShortlistConfig() + if shortlist.rank_mode not in {"and", "or"}: + raise ValueError(f"Unsupported shortlist rank mode: {shortlist.rank_mode}") + + rank_operator = "AND" if shortlist.rank_mode == "and" else "OR" + + query = f""" + WITH buy_orders AS ( + SELECT * FROM orders WHERE is_buy_order = 1 + ), + sell_orders AS ( + SELECT * FROM orders WHERE is_buy_order = 0 + ), + raw_trades AS ( + SELECT + s.location_id as from_station, + b.location_id as to_station, + s.order_id as from_order_id, + b.order_id as to_order_id, + s.item_name, + s.type_id, + s.item_cargo_volume, + s.price as from_price, + b.price as to_price, + MIN( + s.volume_remain, + b.volume_remain, + CAST({cargo_space} / NULLIF(s.item_cargo_volume, 0) AS INTEGER), + CAST({max_capital} / NULLIF(s.price, 0) AS INTEGER), + CAST({max_capital} / NULLIF(b.price, 0) AS INTEGER) + ) as quantity + FROM sell_orders s + JOIN buy_orders b ON s.type_id = b.type_id + WHERE + b.price * (1 - {tax_rate}) >= s.price + AND s.item_cargo_volume > 0 + AND s.price > 0 AND b.price > 0 + ), + candidate_trades AS ( + SELECT + *, + quantity * (to_price * (1 - {tax_rate}) - from_price) as gross_profit, + quantity * item_cargo_volume as cargo, + (quantity * (to_price * (1 - {tax_rate}) - from_price)) / NULLIF(quantity * item_cargo_volume, 0) as profit_density, + (quantity * (to_price * (1 - {tax_rate}) - from_price)) / NULLIF(quantity * from_price, 0) as roi + FROM raw_trades + WHERE quantity > 0 AND quantity * (to_price * (1 - {tax_rate}) - from_price) >= {profit_threshold} + ), + ranked_trades AS ( + SELECT + *, + ROW_NUMBER() OVER ( + PARTITION BY type_id + ORDER BY gross_profit DESC, roi DESC, profit_density DESC + ) as type_rank, + ROW_NUMBER() OVER ( + PARTITION BY from_station, to_station + ORDER BY gross_profit DESC, roi DESC, profit_density DESC + ) as station_pair_rank + FROM candidate_trades ) - buy_orders = sorted( - (order for order in orders if order.is_buy_order), - key=lambda x: x.price, reverse=True - ) - - logger.info("Initial sell orders: %d, buy orders: %d", len(sell_orders), len(buy_orders)) + SELECT + from_station, + to_station, + from_order_id, + to_order_id, + item_name, + type_id, + item_cargo_volume, + from_price, + to_price, + quantity, + cargo, + gross_profit + FROM ranked_trades + WHERE (type_rank <= {shortlist.type_limit} + {rank_operator} station_pair_rank <= {shortlist.station_pair_limit}) + ORDER BY gross_profit DESC, roi DESC, profit_density DESC + LIMIT {shortlist.candidate_limit} + """ - sell_orders = filter_orders_for_cargo(sell_orders, cargo_space) - buy_orders = filter_orders_for_cargo(buy_orders, cargo_space) - logger.info("After cargo filter - sell orders: %d, buy orders: %d", len(sell_orders), len(buy_orders)) - - sell_orders = filter_orders_for_capital_risk(sell_orders, MAX_CAPITAL) - buy_orders = filter_orders_for_capital_risk(buy_orders, MAX_CAPITAL) - logger.info("After capital risk filter - sell orders: %d, buy orders: %d", len(sell_orders), len(buy_orders)) - - trades = create_trades(sell_orders, buy_orders) - return trades - -def create_trades(sell_orders: List[Order], buy_orders: List[Order]) -> List[Trade]: trades = [] - grouped_sell_orders = {} - grouped_buy_orders = {} - - for order in sell_orders: - grouped_sell_orders.setdefault(order.type_id, []).append(order) - for order in buy_orders: - grouped_buy_orders.setdefault(order.type_id, []).append(order) - - for type_id in grouped_sell_orders.keys() & grouped_buy_orders.keys(): - for sell_order in grouped_sell_orders[type_id]: - for buy_order in grouped_buy_orders[type_id]: - effective_buy_price = buy_order.price * (1 - TAX_RATE) - if effective_buy_price < sell_order.price: - continue - - quantity = min(sell_order.volume_remain, buy_order.volume_remain) - gross_profit = quantity * (effective_buy_price - sell_order.price) - - if gross_profit < PROFIT_THRESHOLD: - continue - - cargo_volume = quantity * sell_order.item_cargo_volume + async with market.db_conn.execute(query) as cursor: + async for row in cursor: + from_station, to_station, from_order_id, to_order_id, item_name, type_id, item_cargo_volume, from_price, to_price, quantity, cargo_volume, gross_profit = row + + trades.append(Trade( + from_station=from_station, + to_station=to_station, + from_order_id=from_order_id, + to_order_id=to_order_id, + item_name=item_name, + type_id=type_id, + item_cargo_volume=item_cargo_volume, + from_price=from_price, + to_price=to_price, + quantity=quantity, + cargo=cargo_volume, + gross_profit=gross_profit + )) + + logger.info( + "Generated %d arbitrage candidates after shortlist (limit=%d, type_limit=%d, station_pair_limit=%d, rank_mode=%s)", + len(trades), + shortlist.candidate_limit, + shortlist.type_limit, + shortlist.station_pair_limit, + shortlist.rank_mode, + ) + return trades - trade = Trade( - from_station=sell_order.location_id, - to_station=buy_order.location_id, - from_order_id=sell_order.order_id, - to_order_id=buy_order.order_id, - item_name=sell_order.item_name, - type_id=sell_order.type_id, - item_cargo_volume=sell_order.item_cargo_volume, - from_price=sell_order.price, - to_price=buy_order.price, +async def snipe(market: Market, cargo_space: int, current_region: int, tax_rate: float, max_capital: float, snipe_profit_threshold: float) -> Optional[Trade]: + if not market.db_conn: + return None + + query = f""" + WITH buy_orders AS ( + SELECT * FROM orders WHERE is_buy_order = 1 + ), + sell_orders AS ( + SELECT * FROM orders WHERE is_buy_order = 0 AND region_id = {current_region} + ) + SELECT + s.location_id as from_station, + b.location_id as to_station, + s.order_id as from_order_id, + b.order_id as to_order_id, + s.item_name, + s.type_id, + s.item_cargo_volume, + s.price as from_price, + b.price as to_price, + MIN( + s.volume_remain, + b.volume_remain, + CAST({cargo_space} / NULLIF(s.item_cargo_volume, 0) AS INTEGER), + CAST({max_capital} / NULLIF(s.price, 0) AS INTEGER), + CAST({max_capital} / NULLIF(b.price, 0) AS INTEGER) + ) as quantity + FROM sell_orders s + JOIN buy_orders b ON s.type_id = b.type_id + WHERE + b.price * (1 - {tax_rate}) >= s.price + AND s.item_cargo_volume > 0 + AND s.price > 0 AND b.price > 0 + """ + + best_trade = None + max_profit = 0 + + async with market.db_conn.execute(query) as cursor: + async for row in cursor: + from_station, to_station, from_order_id, to_order_id, item_name, type_id, item_cargo_volume, from_price, to_price, quantity = row + + if quantity is None or quantity <= 0: + continue + + effective_buy_price = to_price * (1 - tax_rate) + gross_profit = quantity * (effective_buy_price - from_price) + + if gross_profit > max_profit: + cargo_volume = quantity * item_cargo_volume + + best_trade = Trade( + from_station=from_station, + to_station=to_station, + from_order_id=from_order_id, + to_order_id=to_order_id, + item_name=item_name, + type_id=type_id, + item_cargo_volume=item_cargo_volume, + from_price=from_price, + to_price=to_price, quantity=quantity, cargo=cargo_volume, gross_profit=gross_profit ) - trades.append(trade) - return trades - -def filter_orders_for_cargo(orders: List[Order], cargo_space: int) -> List[Order]: - filtered_orders = [] - for order in orders: - item_volume = order.item_cargo_volume - if item_volume == 0: - continue - max_quantity = int(cargo_space // item_volume) - volume_remain = min(order.volume_remain, max_quantity) - if volume_remain > 0: - order = Order(**vars(order)) - order.volume_remain = volume_remain - filtered_orders.append(order) - return filtered_orders - -def filter_orders_for_capital_risk(orders: List[Order], max_amount: int) -> List[Order]: - filtered_orders = [] - for order in orders: - price = order.price - if price == 0: - continue - max_quantity = int(max_amount // price) - volume_remain = min(order.volume_remain, max_quantity) - if volume_remain > 0: - order = Order(**vars(order)) - order.volume_remain = volume_remain - filtered_orders.append(order) - return filtered_orders + max_profit = gross_profit -def snipe(orders: List[Order], cargo_space: int, current_region: int) -> Optional[Trade]: - trades = arbitrage(orders, cargo_space) - - order_regions = {order.order_id: order.region_id for order in orders} - - region_trades = [ - trade for trade in trades - if order_regions.get(trade.from_order_id) == current_region - ] - - region_trades.sort(key=lambda x: x.gross_profit, reverse=True) - - logger.info("Reviewing potential snipes for region %d: %d trades found", current_region, len(region_trades)) - - if region_trades and region_trades[0].gross_profit > SNIPE_PROFIT_THRESHOLD: - return region_trades[0] - else: - return None + if max_profit > snipe_profit_threshold and best_trade: + return best_trade + return None diff --git a/backend/src/esi/api.py b/backend/src/esi/api.py index 862b4bf..d8bed26 100644 --- a/backend/src/esi/api.py +++ b/backend/src/esi/api.py @@ -6,9 +6,12 @@ import datetime import json import logging import os +from dataclasses import asdict, dataclass, field import queue import threading import time +import asyncio +import aiohttp from pathlib import Path from typing import Any @@ -30,7 +33,35 @@ ERROR_LIMIT_THRESHOLD = 50 # ESI Error limit threshold REQUEST_RATE_LIMIT = 0.05 # Time in seconds between requests to avoid rate limiting -def get_location() -> int: +@dataclass +class WaypointSubmissionResult: + requested_count: int + success_count: int = 0 + applied_waypoints: list[int] = field(default_factory=list) + failed_waypoints: list[int] = field(default_factory=list) + errors: list[str] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + data = asdict(self) + data["success"] = self.requested_count > 0 and self.success_count == self.requested_count + return data + + +@dataclass(frozen=True) +class WalletTransaction: + transaction_id: int + date: datetime.datetime + location_id: int + type_id: int + unit_price: float + quantity: int + client_id: int + is_buy: bool + is_personal: bool + journal_ref_id: int + + +async def get_location() -> int: """ Gets the current location of the character. @@ -55,42 +86,48 @@ def get_location() -> int: "datasource": DATASOURCE, } - response = requests.get(url, params=params, headers=headers) - logger.debug(f"Request URL: {response.url}") - logger.debug(f"Response Status Code: {response.status_code}") + async with aiohttp.ClientSession() as session: + async with session.get(url, params=params, headers=headers) as response: + logger.debug(f"Request URL: {response.url}") + logger.debug(f"Response Status Code: {response.status}") - if response.status_code == 401: - logger.info("Access token expired, refreshing token.") - access_token(force_refresh=True) - headers["Authorization"] = f"Bearer {access_token()}" - response = requests.get(url, params=params, headers=headers) + if response.status == 401: + logger.info("Access token expired, refreshing token.") + # Using run_in_executor for synchronous access_token + loop = asyncio.get_running_loop() + await loop.run_in_executor(None, access_token, True) + headers["Authorization"] = f"Bearer {access_token()}" + async with session.get(url, params=params, headers=headers) as retry_response: + response = retry_response - if int(response.headers.get("X-Esi-Error-Limit-Remain", 100)) < ERROR_LIMIT_THRESHOLD: - logger.warning("Approaching ESI Error Limit.") - raise Exception("ESI Error Limit approaching threshold.") + if int(response.headers.get("X-Esi-Error-Limit-Remain", 100)) < ERROR_LIMIT_THRESHOLD: + logger.warning("Approaching ESI Error Limit.") + raise Exception("ESI Error Limit approaching threshold.") - response.raise_for_status() - data: dict[str, Any] = response.json() + response.raise_for_status() + data = await response.json() - location_id = data.get("station_id") or data.get("solar_system_id") - if not location_id: - logger.error("Location ID not found in response.") - raise ValueError("Unable to determine character's location.") + location_id = data.get("station_id") or data.get("solar_system_id") + if not location_id: + logger.error("Location ID not found in response.") + raise ValueError("Unable to determine character's location.") - logger.info(f"Character location ID: {location_id}") - return location_id + logger.info(f"Character location ID: {location_id}") + return location_id -def set_waypoints(waypoints: list[int]) -> None: +async def set_waypoints(waypoints: list[int]) -> dict[str, Any]: """ Sets the waypoints for the character's autopilot. Args: waypoints: A list of waypoint IDs to set. """ + result = WaypointSubmissionResult(requested_count=len(waypoints)) + if not waypoints: logger.warning("No waypoints provided.") - return + return result.to_dict() url = f"{ESI_BASE_URL}/ui/autopilot/waypoint/" @@ -101,25 +138,58 @@ def set_waypoints(waypoints: list[int]) -> None: **USER_AGENT_HEADER, } - for index, waypoint in enumerate(waypoints): - params = { - "add_to_beginning": False, - "clear_other_waypoints": index == 0, - "destination_id": waypoint, - "datasource": DATASOURCE, - } - - try: - response = requests.post(url, params=params, headers=headers) - response.raise_for_status() - logger.info(f"Waypoint {waypoint} set successfully.") - time.sleep(1) # Rate limiting - except requests.HTTPError as e: - logger.error(f"Failed to set waypoint {waypoint}: {e}") - continue - - -def open_market_window(type_id: int) -> None: + async with aiohttp.ClientSession() as session: + for index, waypoint in enumerate(waypoints): + params = { + "add_to_beginning": "false", + "clear_other_waypoints": "true" if index == 0 else "false", + "destination_id": waypoint, + "datasource": DATASOURCE, + } + waypoint_headers = dict(headers) + waypoint_error = None + + for attempt in range(2): + try: + async with session.post(url, params=params, headers=waypoint_headers) as response: + if response.status == 401 and attempt == 0: + logger.info("Access token expired while setting waypoint %s, refreshing token.", waypoint) + loop = asyncio.get_running_loop() + refreshed_token = await loop.run_in_executor(None, access_token, True) + waypoint_headers["Authorization"] = f"Bearer {refreshed_token}" + continue + + if response.status == 401: + waypoint_error = f"Waypoint {waypoint} unauthorized after token refresh" + logger.error(waypoint_error) + break + + response.raise_for_status() + logger.info(f"Waypoint {waypoint} set successfully.") + result.success_count += 1 + result.applied_waypoints.append(waypoint) + await asyncio.sleep(1) # Rate limiting + waypoint_error = None + break + except (aiohttp.ClientError, TypeError, ValueError) as error: + waypoint_error = f"Failed to set waypoint {waypoint}: {error}" + logger.error(waypoint_error) + break + except Exception as error: + waypoint_error = f"Unexpected error while setting waypoint {waypoint}: {error}" + logger.exception(waypoint_error) + break + else: + waypoint_error = f"Failed to set waypoint {waypoint}: retries exhausted" + + if waypoint_error is not None: + result.failed_waypoints.append(waypoint) + result.errors.append(waypoint_error) + + return result.to_dict() + + +async def open_market_window(type_id: int) -> None: """ Opens the market window for a specific item type in the game client. @@ -140,17 +210,93 @@ def open_market_window(type_id: int) -> None: "datasource": DATASOURCE, } - try: - response = requests.post(url, params=params, headers=headers) - response.raise_for_status() - logger.info(f"Market window opened for type ID {type_id}.") - except requests.HTTPError as e: - logger.error(f"Failed to open market window for type ID {type_id}: {e}") + async with aiohttp.ClientSession() as session: + try: + async with session.post(url, params=params, headers=headers) as response: + response.raise_for_status() + logger.info(f"Market window opened for type ID {type_id}.") + except aiohttp.ClientError as e: + logger.error(f"Failed to open market window for type ID {type_id}: {e}") -def get_region_orders(region_id: int) -> tuple[list[dict[str, Any]], str]: +async def get_wallet_transactions(from_id: int | None = None) -> list[WalletTransaction]: """ - Retrieves all market orders for a given region. + Retrieves recent wallet transactions for the configured character. + + Args: + from_id: Optional transaction ID cursor. + + Returns: + A list of normalized wallet transactions. + """ + character_id = os.getenv('character_id') + if not character_id: + logger.error("Character ID not found in environment variables.") + raise EnvironmentError("Character ID not set in environment variables.") + + url = f"{ESI_BASE_URL}/characters/{character_id}/wallet/transactions/" + headers = { + "Accept": "application/json", + "Cache-Control": "no-cache", + "Authorization": f"Bearer {access_token()}", + **USER_AGENT_HEADER, + } + params = { + "datasource": DATASOURCE, + } + if from_id is not None: + params["from_id"] = from_id + + async with aiohttp.ClientSession() as session: + for attempt in range(2): + async with session.get(url, params=params, headers=headers) as response: + if response.status == 401 and attempt == 0: + logger.info("Access token expired while loading wallet transactions, refreshing token.") + loop = asyncio.get_running_loop() + refreshed_token = await loop.run_in_executor(None, access_token, True) + headers["Authorization"] = f"Bearer {refreshed_token}" + continue + + if response.status == 403: + error_body = await response.text() + logger.error("Wallet transaction access denied: %s", error_body) + raise PermissionError( + "Wallet transaction access denied. Re-run scripts/setup_auth.py to grant esi-wallet.read_character_wallet.v1." + ) + + if int(response.headers.get("X-Esi-Error-Limit-Remain", 100)) < ERROR_LIMIT_THRESHOLD: + logger.warning("Approaching ESI Error Limit while loading wallet transactions.") + raise Exception("ESI Error Limit approaching threshold.") + + response.raise_for_status() + payload = await response.json() + transactions: list[WalletTransaction] = [] + for entry in payload: + issued_at = datetime.datetime.fromisoformat(entry["date"].replace("Z", "+00:00")) + transactions.append( + WalletTransaction( + transaction_id=entry["transaction_id"], + date=issued_at, + location_id=entry["location_id"], + type_id=entry["type_id"], + unit_price=entry["unit_price"], + quantity=entry["quantity"], + client_id=entry["client_id"], + is_buy=entry["is_buy"], + is_personal=entry["is_personal"], + journal_ref_id=entry["journal_ref_id"], + ) + ) + + logger.info("Retrieved %d wallet transactions.", len(transactions)) + return transactions + + return [] + + +async def get_region_orders(region_id: int) -> tuple[list[dict[str, Any]], str]: + """ + Retrieves all market orders for a given region using asyncio and aiohttp. Args: region_id: The ID of the region. @@ -158,44 +304,60 @@ def get_region_orders(region_id: int) -> tuple[list[dict[str, Any]], str]: Returns: A tuple containing the list of orders and the expiry time. """ - def request_orders(region: int, page: int, q: queue.Queue) -> None: - url = f"{ESI_BASE_URL}/markets/{region}/orders/" - params = { - 'datasource': DATASOURCE, - 'order_type': 'all', - 'page': page, - } - headers = USER_AGENT_HEADER - - try: - response = requests.get(url, params=params, headers=headers) - response.raise_for_status() - logger.debug(f"Retrieved page {page} for region {region}.") - q.put(response) - except requests.HTTPError as e: - logger.error(f"Error fetching page {page} for region {region}: {e}") - - order_queue = queue.Queue() - request_orders(region_id, 1, order_queue) # Get the first page - first_response = order_queue.get() - pages = int(first_response.headers.get('X-Pages', '1')) - expires = first_response.headers.get('Expires', '') - - if pages > 1: - threads = [] - for page in range(2, pages + 1): - thread = threading.Thread(target=request_orders, args=(region_id, page, order_queue)) - threads.append(thread) - thread.start() - time.sleep(REQUEST_RATE_LIMIT) # Rate limiting - - for thread in threads: - thread.join() - - results: list[dict[str, Any]] = first_response.json() - while not order_queue.empty(): - response = order_queue.get() - results.extend(response.json()) + url_template = f"{ESI_BASE_URL}/markets/{{region}}/orders/" + headers = USER_AGENT_HEADER + + async def fetch_page(session: aiohttp.ClientSession, region: int, page: int, sem: asyncio.Semaphore) -> tuple[list[dict[str, Any]], str, int] | None: + async with sem: + params = { + 'datasource': DATASOURCE, + 'order_type': 'all', + 'page': page, + } + for attempt in range(3): + try: + async with session.get(url_template.format(region=region), params=params, headers=headers) as response: + if response.status == 429: + logger.warning(f"Rate limited (429) on page {page} for region {region}. Retrying...") + await asyncio.sleep(2 * (attempt + 1)) + continue + + response.raise_for_status() + logger.debug(f"Retrieved page {page} for region {region}.") + + pages = int(response.headers.get('X-Pages', '1')) + expires = response.headers.get('Expires', '') + data = await response.json() + return data, expires, pages + except aiohttp.ClientError as e: + logger.error(f"Error fetching page {page} for region {region} (attempt {attempt + 1}): {e}") + if attempt == 2: + return None + await asyncio.sleep(1) + return None + + results: list[dict[str, Any]] = [] + expires = '' + sem = asyncio.Semaphore(15) # Bound concurrency to 15 active requests + + async with aiohttp.ClientSession() as session: + # Fetch page 1 to get the total number of pages + first_page_result = await fetch_page(session, region_id, 1, sem) + if not first_page_result: + return [], '' + + data, expires, total_pages = first_page_result + results.extend(data) + + if total_pages > 1: + tasks = [ + fetch_page(session, region_id, page, sem) + for page in range(2, total_pages + 1) + ] + pages_data = await asyncio.gather(*tasks) + for page_result in pages_data: + if page_result: + results.extend(page_result[0]) # Filter out orders with zero remaining volume results = [order for order in results if order.get('volume_remain', 0) > 0] diff --git a/backend/src/esi/shared_flow.py b/backend/src/esi/shared_flow.py index 1d89fb1..00e1b79 100644 --- a/backend/src/esi/shared_flow.py +++ b/backend/src/esi/shared_flow.py @@ -32,6 +32,7 @@ def print_auth_url(client_id: str, code_challenge: str = "") -> None: "esi-assets.read_assets.v1", "esi-location.read_location.v1", "esi-location.read_online.v1", + "esi-wallet.read_character_wallet.v1", "esi-ui.open_window.v1", "esi-ui.write_waypoint.v1", ] diff --git a/backend/src/graph.py b/backend/src/graph.py index 9e4534a..951bd45 100644 --- a/backend/src/graph.py +++ b/backend/src/graph.py @@ -1,17 +1,24 @@ import logging -import os -import pickle +import functools +from dataclasses import dataclass import networkx as nx from math import sqrt, log -from pathlib import Path from typing import Any, Optional +from tqdm import tqdm from loader import load_map_data, load_star_gate_connections logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) -PATHS_FILE = Path(__file__).resolve().parent.parent / 'cache' / 'path_cache.pkl' +LOCAL_SYSTEM_HUB_POSITION = (0.0, 0.0, 0.0) + + +@dataclass(frozen=True) +class LegMetrics: + weight: float + time: float + risk: float class NavigationGraph: @@ -33,7 +40,6 @@ class NavigationGraph: self._add_intra_system_connections() self._load_star_gate_connections() self._add_time_and_risk(ship) - self.paths = self._load_paths() def shortest_path(self, source: int, target: int) -> Optional[list[int]]: """ @@ -54,6 +60,7 @@ class NavigationGraph: logger.warning(f"No path found between {source} and {target}") return None + @functools.lru_cache(maxsize=100000) def shortest_path_length(self, source: int, target: int) -> Optional[float]: """ Returns the length of the shortest path between two stations. @@ -66,11 +73,75 @@ class NavigationGraph: The length of the shortest path, or None if no path exists. """ try: - return self.paths[source][target] - except KeyError: + return nx.shortest_path_length(self.graph, source=source, target=target, weight="weight") + except nx.NetworkXNoPath: logger.warning(f"No path length found between {source} and {target}") return None + def build_trade_leg_metrics(self, ship_location: Optional[int], trades: list[Any]) -> dict[tuple[int, int], Optional[LegMetrics]]: + """ + Builds weighted travel metrics for the legs used by a list of trades. + + Args: + ship_location: The current ship location. + trades: The trade opportunities that need pricing. + + Returns: + A mapping of (source, target) pairs to weighted travel metrics. + """ + if ship_location is None: + return {} + + source_targets: dict[int, set[int]] = {ship_location: set()} + for trade in trades: + source_targets.setdefault(ship_location, set()).add(trade.from_station) + source_targets.setdefault(trade.from_station, set()).add(trade.to_station) + + return self.build_leg_metrics(source_targets) + + def build_leg_metrics(self, source_targets: dict[int, set[int]]) -> dict[tuple[int, int], Optional[LegMetrics]]: + """ + Computes weighted leg metrics for a batch of sources and targets. + + Args: + source_targets: Mapping of a source node to the targets needed from it. + + Returns: + A mapping of (source, target) pairs to weighted travel metrics. + """ + leg_metrics: dict[tuple[int, int], Optional[LegMetrics]] = {} + + for source, targets in tqdm( + source_targets.items(), + total=len(source_targets), + desc="Building leg metrics", + ): + if source not in self.graph: + for target in targets: + leg_metrics[(source, target)] = None if source != target else LegMetrics(0.0, 0.0, 0.0) + continue + + try: + weights, paths = nx.single_source_dijkstra(self.graph, source=source, weight="weight") + except nx.NodeNotFound: + for target in targets: + leg_metrics[(source, target)] = None if source != target else LegMetrics(0.0, 0.0, 0.0) + continue + + for target in targets: + if source == target: + leg_metrics[(source, target)] = LegMetrics(0.0, 0.0, 0.0) + continue + + path = paths.get(target) + if path is None: + leg_metrics[(source, target)] = None + continue + + leg_metrics[(source, target)] = self._summarize_path(path, weights[target]) + + return leg_metrics + def formatted_route(self, path: list[int]) -> list[dict[str, Any]]: """ Formats the route for the ESI API and frontend. @@ -81,15 +152,30 @@ class NavigationGraph: Returns: A list of dictionaries containing location information. """ - combined_path = self._combine_warp_gates(path) + formatted_route, _ = self.formatted_route_with_offsets(path) + return formatted_route + + def formatted_route_with_offsets(self, path: list[int]) -> tuple[list[dict[str, Any]], list[int]]: + """ + Formats the route and returns the raw-path offset for each display step. + + Args: + path: A list of station IDs representing the raw path. + + Returns: + A tuple of formatted route steps and their starting offsets in the raw path. + """ + combined_path, raw_offsets = self._combine_warp_gates(path) formatted_route = [] + formatted_offsets = [] - for index, location_id in enumerate(combined_path): - location_info = self._get_location_info(location_id, index) + for location_id, raw_offset in zip(combined_path, raw_offsets): + location_info = self._get_location_info(location_id, len(formatted_route)) if location_info is not None: formatted_route.append(location_info) + formatted_offsets.append(raw_offset) - return formatted_route + return formatted_route, formatted_offsets def _get_location_info(self, location_id: int, index: int) -> Optional[dict[str, Any]]: """ @@ -128,7 +214,28 @@ class NavigationGraph: logger.warning(f"Location ID {location_id} not found in stations or solar systems") return None - def _combine_warp_gates(self, path: list[int]) -> list[int]: + def _summarize_path(self, path: list[int], weight: float) -> LegMetrics: + """ + Summarizes the time and risk of a chosen weighted path. + + Args: + path: The chosen weighted path. + weight: The total weighted cost of the path. + + Returns: + The weighted, time, and risk totals for that path. + """ + transport_time = 0.0 + risk = 0.0 + + for u, v in zip(path, path[1:]): + edge = self.graph[u][v] + transport_time += edge["time"] + risk += edge["risk"] + + return LegMetrics(weight=weight, time=transport_time, risk=risk) + + def _combine_warp_gates(self, path: list[int]) -> tuple[list[int], list[int]]: """ Combines warp gates in the path to reduce redundant entries. @@ -139,24 +246,28 @@ class NavigationGraph: A list of location IDs with combined warp gates. """ combined_path = [] + raw_offsets = [] - for i, (current_id, next_id) in enumerate(zip(path, path[1:])): + for index, (current_id, next_id) in enumerate(zip(path, path[1:])): current_station = self.stations.get(current_id) next_station = self.stations.get(next_id) if current_station and current_station['is_station']: combined_path.append(current_id) + raw_offsets.append(index) elif (current_station and next_station and not current_station['is_station'] and not next_station['is_station'] and current_station['solar_system_id'] == next_station['solar_system_id']): combined_path.append(current_station['solar_system_id']) + raw_offsets.append(index) last_station = self.stations.get(path[-1]) if last_station and last_station['is_station']: combined_path.append(path[-1]) + raw_offsets.append(len(path) - 1) - return combined_path + return combined_path, raw_offsets def _add_intra_system_connections(self) -> None: """ @@ -167,10 +278,16 @@ class NavigationGraph: system_id = station_data['solar_system_id'] stations_by_system.setdefault(system_id, []).append(station_id) - for system_stations in stations_by_system.values(): - for i in range(len(system_stations)): - for j in range(i + 1, len(system_stations)): - self.graph.add_edge(system_stations[i], system_stations[j]) + for system_id, system_stations in stations_by_system.items(): + self.graph.add_node(system_id) + for station_id in system_stations: + self.graph.add_edge(system_id, station_id) + + # Keep direct same-system edges so weighted pricing uses the real + # station-to-station distance instead of an artificial hub detour. + for index, from_station in enumerate(system_stations): + for to_station in system_stations[index + 1:]: + self.graph.add_edge(from_station, to_station) def _load_star_gate_connections(self) -> None: """ @@ -196,28 +313,40 @@ class NavigationGraph: self.graph[u][v]['risk'] = risk self.graph[u][v]['weight'] = weight - def _calculate_time(self, from_station: int, to_station: int, ship: dict[str, Any]) -> float: + def _calculate_time(self, from_node: int, to_node: int, ship: dict[str, Any]) -> float: """ - Calculates the time to travel between two stations. + Calculates the time to travel between two nodes. Args: - from_station: The starting station ID. - to_station: The destination station ID. + from_node: The starting node ID (station or system hub). + to_node: The destination node ID (station or system hub). ship: A dictionary containing ship parameters. Returns: - The time in seconds to travel between the two stations. + The time in seconds to travel between the two nodes. """ max_warp_speed = ship.get("max_warp_speed", 1) max_subwarp_speed = ship.get("max_subwarp_speed", 1) - # Inter-system travel (using stargates) - if self.stations[from_station]['solar_system_id'] != self.stations[to_station]['solar_system_id']: - return 9.0 # Fixed time for stargate jumps - - # Intra-system travel - position_from = self.stations[from_station]['position'] - position_to = self.stations[to_station]['position'] + # Helper to get node position + def get_node_position(node_id): + if node_id in self.stations: + return self.stations[node_id]['position'] + elif node_id in self.solar_systems: + # Solar-system rows use universe coordinates while stations and + # stargates use local system coordinates. Synthetic hub nodes + # must stay in the local frame to avoid impossible warp lengths. + return LOCAL_SYSTEM_HUB_POSITION + return (0.0, 0.0, 0.0) + + # Both are stations: check if it's an inter-system jump + if from_node in self.stations and to_node in self.stations: + if self.stations[from_node]['solar_system_id'] != self.stations[to_node]['solar_system_id']: + return 9.0 # Fixed time for stargate jumps + + # Intra-system travel (station <-> hub or station <-> system) + position_from = get_node_position(from_node) + position_to = get_node_position(to_node) warp_dist = sqrt(sum((position_from[i] - position_to[i]) ** 2 for i in range(3))) time_in_warp = self._calculate_time_in_warp(warp_dist, max_warp_speed, max_subwarp_speed) @@ -260,20 +389,31 @@ class NavigationGraph: total_time = cruise_time + accel_time + decel_time return total_time - def _calculate_risk(self, to_station: int, gankers_areas: dict[str, float]) -> float: + def _calculate_risk(self, to_node: int, gankers_areas: dict[str, float]) -> float: """ - Calculates the risk of traveling to a station. + Calculates the risk of traveling to a node. Args: - to_station: The destination station ID. + to_node: The destination node ID (station or system hub). gankers_areas: A dictionary mapping solar system names to risk factors. Returns: A risk score as a float. """ - station_data = self.stations[to_station] - security = station_data.get('security', 1.0) - solar_system_name = station_data.get('solar_system_name', '') + # Determine security and solar_system_name based on node type + if to_node in self.stations: + station_data = self.stations[to_node] + security = station_data.get('security', 1.0) + solar_system_name = station_data.get('solar_system_name', '') + elif to_node in self.solar_systems: + system_data = self.solar_systems[to_node] + # Can default to high-sec if system info doesn't have security, + # but usually it's known, actually we'll default to 1.0 + security = system_data.get('security', 1.0) + solar_system_name = system_data.get('item_name', '') + else: + security = 1.0 + solar_system_name = '' # Base risk calculation based on security status risk = 0.00004 + (1 - security) * 0.0004 @@ -285,22 +425,3 @@ class NavigationGraph: risk += gankers_areas.get(solar_system_name, 0) return risk - - def _load_paths(self) -> dict[int, dict[int, float]]: - """ - Loads or generates the shortest path lengths between all pairs. - - Returns: - A dictionary of dictionaries containing path lengths. - """ - if PATHS_FILE.exists(): - with PATHS_FILE.open("rb") as f: - logger.info("Loading cached paths from file") - return pickle.load(f) - else: - logger.info("Generating path lengths and caching to file") - dist = dict(nx.all_pairs_dijkstra_path_length(self.graph)) - PATHS_FILE.parent.mkdir(parents=True, exist_ok=True) - with PATHS_FILE.open("wb") as f: - pickle.dump(dist, f) - return dist diff --git a/backend/src/main.py b/backend/src/main.py index c1a76f5..fef58b0 100644 --- a/backend/src/main.py +++ b/backend/src/main.py @@ -3,6 +3,9 @@ import json import logging from typing import Any, Optional +from dotenv import load_dotenv +load_dotenv() + from fastapi import BackgroundTasks, FastAPI from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import StreamingResponse @@ -21,7 +24,7 @@ async def start_manager() -> None: while True: try: logger.info("Starting manager update") - await asyncio.to_thread(manager.update) + await manager.update() logger.info("Manager update completed") except Exception as error: logger.exception("An error occurred during manager update: %s", error) @@ -64,7 +67,7 @@ async def read_route_info() -> Any: @app.post("/create_route") async def create_route(background_tasks: BackgroundTasks) -> dict: """Initiates route creation in the background.""" - background_tasks.add_task(manager.create_route) + asyncio.create_task(manager.create_route()) return {"status": "Route creation initiated"} @app.get("/data_stream") @@ -84,11 +87,33 @@ async def data_stream() -> StreamingResponse: @app.get("/logs") async def get_logs() -> Any: """Retrieves logs from the manager.""" - return manager.log_queue + return manager.log @app.post("/open_market_window/{type_id}") -async def open_market_window(type_id: int) -> Any: +async def open_market_window(type_id: int) -> dict: """Opens a market window for a given `type_id` using the ESI API.""" - result = esi.open_market_window(type_id) + await esi.open_market_window(type_id) logger.info("Opened market window for type_id %s", type_id) - return result + return {"status": "opened"} + +@app.get("/settings") +async def get_settings() -> dict: + """Returns the current ship settings.""" + return manager.ship + +@app.post("/settings") +async def update_settings(settings: dict) -> dict: + """Updates the ship settings.""" + for key, value in settings.items(): + if key in manager.ship: + manager.ship[key] = value + # Re-initialize the graph with new settings + from graph import NavigationGraph + manager.graph = NavigationGraph(manager.ship) + return {"status": "success", "settings": manager.ship} + +@app.post("/set_waypoints") +async def set_waypoints_endpoint() -> dict: + """Uses the ESI API to push the calculated route as in-game waypoints""" + return await manager.set_waypoints() + diff --git a/backend/src/manager.py b/backend/src/manager.py index 09d920e..7e86949 100644 --- a/backend/src/manager.py +++ b/backend/src/manager.py @@ -1,38 +1,88 @@ -import threading +import asyncio +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from typing import Any +from time import perf_counter from graph import NavigationGraph -from esi.api import get_location, set_waypoints, open_market_window +from esi.api import WalletTransaction, get_location, get_wallet_transactions, open_market_window, set_waypoints as push_waypoints from market import Market -from arbitrage import arbitrage, snipe -from pathfinder import pathfinder +from arbitrage import ShortlistConfig, Trade, arbitrage, snipe +from pathfinder import calculate_net_profit, filter_order_matches_low_profit, filter_order_matches_not_in_graph import route +PRIMARY_SHORTLIST = ShortlistConfig() +FALLBACK_SHORTLIST = ShortlistConfig(candidate_limit=80_000, type_limit=600, station_pair_limit=80) +WALLET_POLL_INTERVAL_SECONDS = 60.0 +TRADE_CONFIRMATION_TTL = timedelta(hours=6) +PRICE_TOLERANCE_RATIO = 0.002 +MIN_PRICE_TOLERANCE = 0.01 +CLOCK_SKEW_TOLERANCE = timedelta(minutes=5) + + +@dataclass +class TrackedTradeConfirmation: + trade: Trade + created_at: datetime + expires_at: datetime + buy_remaining: int + sell_remaining: int + + @classmethod + def from_trade(cls, trade: Trade, created_at: datetime) -> "TrackedTradeConfirmation": + return cls( + trade=trade, + created_at=created_at, + expires_at=created_at + TRADE_CONFIRMATION_TTL, + buy_remaining=trade.quantity, + sell_remaining=trade.quantity, + ) + + def is_complete(self) -> bool: + return self.buy_remaining <= 0 and self.sell_remaining <= 0 + + def is_expired(self, now: datetime) -> bool: + return now >= self.expires_at + + class Manager: def __init__(self): self.ship = { - "location": get_location(), + "location": None, "cargo": 4000, "max_warp_speed": 4.55, # AU/s "max_subwarp_speed": 166.9, # m/s "gankers_areas": {"Ahbazon": 0.5}, #["Miroitem", "Rancer", "Huola", "Ahbazon", "Uedama", "Sivala", "Tama", "Onatoh"], "player_cost": 15_000_000 / 3600, # isk/second "risk_cost": 60_000_000, # isk - "ship_cost": 5_500_000 # isk + "ship_cost": 5_500_000, # isk + "tax_rate": 8.0, # percentage % + "profit_threshold": 100_000, + "max_capital": 20_000_000, + "snipe_profit_threshold": 20_000_000 } self.graph = NavigationGraph(self.ship) self.route = [] self.route_info = {} + self.route_path: list[int] = [] + self.route_path_offsets: list[int] = [] + self.active_route_trades: list[Trade] = [] + self.active_trade_confirmations: list[TrackedTradeConfirmation] = [] + self.pending_trade_confirmations: list[TrackedTradeConfirmation] = [] self.market = Market() self.updating_orders = False self.log = ["Test Log"] + self.last_wallet_poll_at = 0.0 + self.last_seen_wallet_transaction_id = 0 + self.wallet_scope_error_logged = False - def route(self): + def get_route(self): return self.route - def route_info(self): + def get_route_info(self): return self.route_info def get_log(self): @@ -40,7 +90,349 @@ class Manager: self.log = [] return result - def snipe_callback(self, updated_regions): + def build_trade_confirmations(self, trades: list[Trade]) -> list[TrackedTradeConfirmation]: + created_at = datetime.now(UTC) + return [TrackedTradeConfirmation.from_trade(trade, created_at) for trade in trades] + + def clear_active_route(self) -> None: + self.route = [] + self.route_info = {} + self.route_path = [] + self.route_path_offsets = [] + self.active_route_trades = [] + self.active_trade_confirmations = [] + + def trim_active_route(self, display_start_index: int) -> None: + if display_start_index <= 0: + return + + self.route = self.route[display_start_index:] + + if display_start_index < len(self.route_path_offsets): + raw_start_index = self.route_path_offsets[display_start_index] + self.route_path = self.route_path[raw_start_index:] + self.route_path_offsets = [ + offset - raw_start_index + for offset in self.route_path_offsets[display_start_index:] + ] + else: + self.route_path = [] + self.route_path_offsets = [] + + def refresh_route_info(self) -> None: + if not self.route or not self.active_route_trades or not self.route_path: + self.route_info = {} + return + + self.route_info = route.calculate_route_info( + self.route, + self.active_route_trades, + self.graph, + self.ship, + metric_path=self.route_path, + ) or {} + + def prune_trade_confirmations(self) -> None: + now = datetime.now(UTC) + self.active_trade_confirmations = [ + confirmation + for confirmation in self.active_trade_confirmations + if not confirmation.is_complete() + ] + + retained_pending: list[TrackedTradeConfirmation] = [] + expired_confirmations = 0 + for confirmation in self.pending_trade_confirmations: + if confirmation.is_complete(): + continue + if confirmation.is_expired(now): + expired_confirmations += 1 + continue + retained_pending.append(confirmation) + + self.pending_trade_confirmations = retained_pending + if expired_confirmations: + self.log.append(f"Dropped {expired_confirmations} stale unconfirmed trade confirmations") + + def move_active_confirmations_to_pending(self) -> None: + outstanding_confirmations = [ + confirmation + for confirmation in self.active_trade_confirmations + if not confirmation.is_complete() + ] + if not outstanding_confirmations: + self.active_trade_confirmations = [] + return + + now = datetime.now(UTC) + for confirmation in outstanding_confirmations: + confirmation.expires_at = max(confirmation.expires_at, now + TRADE_CONFIRMATION_TTL) + + self.pending_trade_confirmations.extend(outstanding_confirmations) + self.active_trade_confirmations = [] + self.log.append(f"Waiting for ESI confirmation on {len(outstanding_confirmations)} route trade(s)") + + def wallet_transaction_matches( + self, + transaction: WalletTransaction, + confirmation: TrackedTradeConfirmation, + ) -> bool: + trade = confirmation.trade + expected_location = trade.from_station if transaction.is_buy else trade.to_station + expected_price = trade.from_price if transaction.is_buy else trade.to_price + remaining_quantity = confirmation.buy_remaining if transaction.is_buy else confirmation.sell_remaining + + if remaining_quantity <= 0: + return False + if transaction.type_id != trade.type_id or transaction.location_id != expected_location: + return False + if transaction.date < confirmation.created_at - CLOCK_SKEW_TOLERANCE: + return False + if transaction.date > confirmation.expires_at: + return False + + price_tolerance = max(MIN_PRICE_TOLERANCE, expected_price * PRICE_TOLERANCE_RATIO) + return abs(transaction.unit_price - expected_price) <= price_tolerance + + def apply_wallet_transaction( + self, + transaction: WalletTransaction, + confirmations: list[TrackedTradeConfirmation], + order_adjustments: dict[int, int], + ) -> int: + remaining_quantity = transaction.quantity + + for confirmation in confirmations: + if not self.wallet_transaction_matches(transaction, confirmation): + continue + + remaining_attr = "buy_remaining" if transaction.is_buy else "sell_remaining" + order_id = confirmation.trade.from_order_id if transaction.is_buy else confirmation.trade.to_order_id + available_quantity = getattr(confirmation, remaining_attr) + applied_quantity = min(available_quantity, remaining_quantity) + if applied_quantity <= 0: + continue + + setattr(confirmation, remaining_attr, available_quantity - applied_quantity) + order_adjustments[order_id] = order_adjustments.get(order_id, 0) + applied_quantity + remaining_quantity -= applied_quantity + if remaining_quantity == 0: + break + + return transaction.quantity - remaining_quantity + + async def reconcile_trade_confirmations(self) -> None: + self.prune_trade_confirmations() + if not (self.active_trade_confirmations or self.pending_trade_confirmations): + return + + if perf_counter() - self.last_wallet_poll_at < WALLET_POLL_INTERVAL_SECONDS: + return + + self.last_wallet_poll_at = perf_counter() + + try: + transactions = await get_wallet_transactions() + self.wallet_scope_error_logged = False + except PermissionError as exc: + if not self.wallet_scope_error_logged: + self.log.append(str(exc)) + self.wallet_scope_error_logged = True + return + except Exception as exc: + self.log.append(f"Failed to poll wallet transactions: {exc}") + return + + if not transactions: + return + + highest_transaction_id = max(transaction.transaction_id for transaction in transactions) + new_transactions = [ + transaction + for transaction in transactions + if transaction.transaction_id > self.last_seen_wallet_transaction_id + ] + self.last_seen_wallet_transaction_id = max(self.last_seen_wallet_transaction_id, highest_transaction_id) + + if not new_transactions: + return + + order_adjustments: dict[int, int] = {} + matched_transactions = 0 + matched_units = 0 + confirmations = self.active_trade_confirmations + self.pending_trade_confirmations + + for transaction in sorted(new_transactions, key=lambda item: (item.date, item.transaction_id)): + applied_quantity = self.apply_wallet_transaction(transaction, confirmations, order_adjustments) + if applied_quantity > 0: + matched_transactions += 1 + matched_units += applied_quantity + + self.prune_trade_confirmations() + + if not order_adjustments: + return + + summary = await self.market.apply_order_volume_adjustments(order_adjustments) + self.log.append( + f"Confirmed {matched_units:,} routed units across {matched_transactions:,} wallet transaction(s); " + f"updated {summary['affected_orders']:,} local market orders" + ) + if summary["ignored_orders"]: + self.log.append( + f"Skipped {summary['ignored_orders']:,} confirmed order adjustments because newer market data was already present" + ) + + def extract_route_waypoints(self) -> tuple[list[int], str | None]: + if not self.route: + return [], "No route exists" + + if len(self.route) <= 1: + return [], "No destination waypoints available" + + waypoints = [] + invalid_entries = [] + for step in self.route[1:]: + location_id = step.get("location_id") if isinstance(step, dict) else None + if isinstance(location_id, int): + waypoints.append(location_id) + else: + invalid_entries.append(step) + + if invalid_entries: + return [], "Current route contains invalid waypoint entries" + + if not waypoints: + return [], "No destination waypoints available" + + return waypoints, None + + async def set_waypoints(self) -> dict[str, Any]: + waypoints, error_message = self.extract_route_waypoints() + if error_message: + self.log.append(f"Waypoint push skipped: {error_message}") + return { + "success": False, + "status": "error", + "message": error_message, + "requested_count": 0, + "success_count": 0, + "applied_waypoints": [], + "failed_waypoints": [], + "errors": [], + } + + self.log.append(f"Submitting {len(waypoints)} waypoint(s) to EVE...") + result = await push_waypoints(waypoints) + requested_count = result.get("requested_count", len(waypoints)) + success_count = result.get("success_count", 0) + failed_waypoints = result.get("failed_waypoints", []) + + if result.get("success"): + message = f"Set {success_count} waypoint(s) in game" + status = "success" + elif success_count > 0: + message = f"Set {success_count}/{requested_count} waypoint(s) in game" + status = "partial" + else: + message = "Failed to set waypoints in game" + status = "error" + + self.log.append(message) + if failed_waypoints: + self.log.append(f"Failed waypoint IDs: {', '.join(str(waypoint) for waypoint in failed_waypoints)}") + if result.get("errors"): + self.log.append(f"Waypoint error: {result['errors'][0]}") + + return { + **result, + "status": status, + "message": message, + } + + def summarize_trades(self, trades): + if not trades: + return { + "count": 0, + "sources": 0, + "pairs": 0, + "triplets": 0, + } + + return { + "count": len(trades), + "sources": len({trade.from_station for trade in trades}), + "pairs": len({(trade.from_station, trade.to_station) for trade in trades}), + "triplets": len({(self.ship["location"], trade.from_station, trade.to_station) for trade in trades}), + } + + def describe_trade(self, trade): + if not trade: + return "No scored candidates" + + from_station = self.graph.stations.get(trade.from_station, {}).get("item_name", trade.from_station) + to_station = self.graph.stations.get(trade.to_station, {}).get("item_name", trade.to_station) + net_profit = trade.net_profit if trade.net_profit is not None else 0 + return ( + f"{trade.item_name} {from_station} -> {to_station} " + f"gross {trade.gross_profit:,.0f} ISK, net {net_profit:,.0f} ISK" + ) + + async def run_route_pass(self, label, shortlist): + self.log.append(f"{label}: fetching market data and shortlisting arbitrage opportunities...") + stage_start = perf_counter() + order_matches = await arbitrage( + self.market, + self.ship["cargo"], + self.ship["tax_rate"] / 100.0, + self.ship["profit_threshold"], + self.ship["max_capital"], + shortlist=shortlist, + ) + shortlist_duration = perf_counter() - stage_start + shortlist_summary = self.summarize_trades(order_matches) + self.log.append( + f"{label}: shortlisted {shortlist_summary['count']:,} candidates across {shortlist_summary['pairs']:,} station pairs " + f"and {shortlist_summary['sources']:,} source stations in {shortlist_duration:.2f}s" + ) + + self.log.append(f"{label}: preparing graph leg metrics and estimating net profit...") + stage_start = perf_counter() + graph_valid_matches = filter_order_matches_not_in_graph(order_matches, self.graph) + graph_valid_summary = self.summarize_trades(graph_valid_matches) + graph_rejects = len(order_matches) - len(graph_valid_matches) + leg_metrics = self.graph.build_trade_leg_metrics(self.ship["location"], graph_valid_matches) + scored_matches = calculate_net_profit(graph_valid_matches, self.ship["location"], leg_metrics) + path_missing = sum(1 for trade in scored_matches if trade.net_profit is None) + profitable_matches = filter_order_matches_low_profit(scored_matches) + positive_count = len(profitable_matches) + scored_count = len(scored_matches) - path_missing + non_positive = max(scored_count - positive_count, 0) + profitable_duration = perf_counter() - stage_start + profitable_summary = self.summarize_trades(profitable_matches) + best_candidate = max( + (trade for trade in scored_matches if trade.net_profit is not None), + key=lambda trade: trade.net_profit, + default=None, + ) + self.log.append( + f"{label}: retained {profitable_summary['count']:,} profitable candidates across {profitable_summary['triplets']:,} unique triplets " + f"from {graph_valid_summary['pairs']:,} graph-valid station pairs in {profitable_duration:.2f}s" + ) + self.log.append( + f"{label}: rejected {graph_rejects:,} candidates not in graph, {path_missing:,} candidates without a path, " + f"and {non_positive:,} candidates with non-positive net profit" + ) + self.log.append( + f"{label}: cached {len(leg_metrics):,} weighted travel legs from {graph_valid_summary['sources']:,} trade sources plus current location" + ) + self.log.append(f"{label}: best scored candidate {self.describe_trade(best_candidate)}") + + return profitable_matches, leg_metrics + + async def snipe_callback(self, updated_regions): + if not self.ship["location"]: + return if self.ship["location"] in self.graph.solar_systems: current_region = self.graph.solar_systems[self.ship["location"]]["region_id"] @@ -48,7 +440,7 @@ class Manager: current_region = self.graph.stations[self.ship["location"]]["region_id"] if current_region in updated_regions: - potential_snipe = snipe(self.market.orders, self.ship["cargo"], current_region) + potential_snipe = await snipe(self.market, self.ship["cargo"], current_region, self.ship["tax_rate"] / 100.0, self.ship["max_capital"], self.ship["snipe_profit_threshold"]) if potential_snipe: self.log.append("----------------") self.log.append(f"Snipe: {potential_snipe.item_name} orders for {potential_snipe.gross_profit} ISK") @@ -56,54 +448,87 @@ class Manager: print("snipe possible", potential_snipe.gross_profit, "ISK", potential_snipe.item_name) - open_market_window(potential_snipe.type_id) + await open_market_window(potential_snipe.type_id) - def update(self): + async def update(self): self.log = self.log[-100:] - location = get_location() - self.ship["location"] = location + location = await get_location() + if not self.ship["location"]: + self.ship["location"] = location + self.graph = NavigationGraph(self.ship) + else: + self.ship["location"] = location + + await self.reconcile_trade_confirmations() if len(self.route) == 1: # Only the last step is left, so we are at the destination self.log.append("Arrived at Destination") - self.route = [] + self.move_active_confirmations_to_pending() + self.clear_active_route() return for i, step in enumerate(self.route[:2]): if i != 0 and step["location_id"] == location: self.log.append("Arrived at " + step["location"]) - self.route = self.route[i:] + self.trim_active_route(i) + self.refresh_route_info() break if not self.updating_orders: self.updating_orders = True - def target_function(log): + async def target_function(): try: - updated_regions = self.market.update_orders() + updated_regions = await self.market.update_orders() if updated_regions: - log.append(f"Updated {len(updated_regions)} market regions") - self.snipe_callback(updated_regions) + self.log.append(f"Updated {len(updated_regions)} market regions") + await self.snipe_callback(updated_regions) finally: self.updating_orders = False - update_thread = threading.Thread(target=target_function, args=(self.log,)) - update_thread.start() + asyncio.create_task(target_function()) + + + async def create_route(self): + if self.ship["location"] is None: + self.ship["location"] = await get_location() + self.graph = NavigationGraph(self.ship) + if self.route or self.active_route_trades or self.active_trade_confirmations: + self.log.append("Discarded previous active route and its unconfirmed settlement tracking") + self.clear_active_route() - def create_route(self): - order_matches = arbitrage(self.market.orders, self.ship["cargo"]) - order_matches = pathfinder(order_matches, self.graph, self.ship) - self.route, self.route_info = route.route(order_matches, self.graph, self.ship) + route_start = perf_counter() + + order_matches, leg_metrics = await self.run_route_pass("Primary pass", PRIMARY_SHORTLIST) + if not order_matches: + self.log.append("Primary pass found no profitable candidates; retrying with a wider shortlist...") + order_matches, leg_metrics = await self.run_route_pass("Fallback pass", FALLBACK_SHORTLIST) + + self.log.append("Optimizing route...") + unique_triplets = route.count_unique_triplets(order_matches, self.ship["location"]) + self.log.append(f"Evaluating up to {min(unique_triplets, route.MAX_TRADES_TO_CONSIDER):,} unique triplets for route optimization") + stage_start = perf_counter() + ( + self.route, + self.route_info, + self.active_route_trades, + self.route_path, + self.route_path_offsets, + ) = route.route(order_matches, self.graph, self.ship, leg_metrics) + route_duration = perf_counter() - stage_start + total_duration = perf_counter() - route_start if self.route: - set_waypoints([r["location_id"] for r in self.route[1:]]) + self.active_trade_confirmations = self.build_trade_confirmations(self.active_route_trades) print(self.route) + self.log.append(f"Route complete in {total_duration:.2f}s ({route_duration:.2f}s optimizer)") else: - self.route = [] - self.route_info = {} + self.clear_active_route() print("No profitable route found yet.") + self.log.append(f"No profitable route found yet after {total_duration:.2f}s") \ No newline at end of file diff --git a/backend/src/market.py b/backend/src/market.py index 311b181..045b07e 100644 --- a/backend/src/market.py +++ b/backend/src/market.py @@ -3,7 +3,9 @@ import logging from dataclasses import dataclass, asdict from datetime import datetime from pathlib import Path -from typing import List, Dict +from typing import List, Dict, Any +import aiosqlite +import asyncio from loader import load_items from esi.api import get_region_orders @@ -46,89 +48,211 @@ class Order: class Market: """ - Manages market orders, including fetching, updating, and caching. + Manages market orders, including fetching, updating, and caching via in-memory SQLite for high performance. """ def __init__(self): self.orders: List[Order] = [] self.expires_times: Dict[int, datetime] = {} + self.db_conn: aiosqlite.Connection | None = None + self._lock = asyncio.Lock() + + async def init_db(self): + """Initializes the in-memory SQLite database.""" + if not self.db_conn: + self.db_conn = await aiosqlite.connect(':memory:') + await self.db_conn.execute(''' + CREATE TABLE IF NOT EXISTS orders ( + order_id INTEGER PRIMARY KEY, + region_id INTEGER, + location_id INTEGER, + type_id INTEGER, + is_buy_order BOOLEAN, + price REAL, + issued TEXT, + volume_remain INTEGER, + item_name TEXT, + item_cargo_volume REAL + ) + ''') + await self.db_conn.execute('CREATE INDEX IF NOT EXISTS idx_arbitrage ON orders(type_id, is_buy_order, region_id)') + await self.db_conn.commit() - def get_orders(self) -> List[Order]: + async def get_orders(self) -> List[Order]: """Retrieves the current list of market orders.""" - self.update_orders() + await self.update_orders() return self.orders - - def update_orders(self) -> List[int]: + + async def get_all_orders_from_db(self) -> List[Order]: + """Retrieves all orders from the database as Order objects (mostly for fallback/testing).""" + if not self.db_conn: + return [] + + async with self.db_conn.execute('SELECT * FROM orders') as cursor: + rows = await cursor.fetchall() + orders = [] + for r in rows: + orders.append(Order( + order_id=r[0], + region_id=r[1], + location_id=r[2], + type_id=r[3], + is_buy_order=bool(r[4]), + price=r[5], + issued=datetime.fromisoformat(r[6]), + volume_remain=r[7], + item_name=r[8], + item_cargo_volume=r[9] + )) + return orders + + async def update_orders(self) -> List[int]: """ Updates market orders by fetching new data for expired regions. Returns: A list of region IDs that were updated. """ - updated_orders: List[Order] = [] - regions_updated: List[int] = [] - - for region_id in REGION_IDS: - if self.check_expired(region_id): - logger.info(f"Downloading orders for region {region_id}") - orders = self.download_orders(region_id) - updated_orders.extend(orders) - regions_updated.append(region_id) - else: - # Include existing orders for this region - updated_orders.extend( - [order for order in self.orders if order.region_id == region_id] + if not self.db_conn: + await self.init_db() + + async with self._lock: + regions_updated: List[int] = [] + + for region_id in REGION_IDS: + if self.check_expired(region_id): + logger.info(f"Downloading orders for region {region_id}") + orders = await self.download_orders(region_id) + self.orders = [o for o in self.orders if o.region_id != region_id] + self.orders.extend(orders) + + # Update SQLite DB + await self.db_conn.execute('DELETE FROM orders WHERE region_id = ?', (region_id,)) + + # Bulk insert + insert_data = [ + (o.order_id, o.region_id, o.location_id, o.type_id, o.is_buy_order, + o.price, o.issued.isoformat(), o.volume_remain, o.item_name, o.item_cargo_volume) + for o in orders + ] + await self.db_conn.executemany(''' + INSERT OR REPLACE INTO orders ( + order_id, region_id, location_id, type_id, is_buy_order, + price, issued, volume_remain, item_name, item_cargo_volume + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ''', insert_data) + await self.db_conn.commit() + regions_updated.append(region_id) + + return regions_updated + + async def apply_order_volume_adjustments(self, order_adjustments: Dict[int, int]) -> Dict[str, int]: + if not order_adjustments: + return { + 'affected_orders': 0, + 'updated_orders': 0, + 'removed_orders': 0, + 'ignored_orders': 0, + 'consumed_units': 0, + } + + if not self.db_conn: + await self.init_db() + + async with self._lock: + order_index = {order.order_id: order for order in self.orders} + update_rows: list[tuple[int, int]] = [] + delete_rows: list[tuple[int]] = [] + updated_orders = 0 + removed_orders = 0 + ignored_orders = 0 + consumed_units = 0 + + for order_id, requested_quantity in order_adjustments.items(): + if requested_quantity <= 0: + continue + + order = order_index.get(order_id) + if order is None: + ignored_orders += 1 + continue + + applied_quantity = min(order.volume_remain, requested_quantity) + if applied_quantity <= 0: + ignored_orders += 1 + continue + + order.volume_remain -= applied_quantity + consumed_units += applied_quantity + + if order.volume_remain <= 0: + delete_rows.append((order_id,)) + removed_orders += 1 + else: + update_rows.append((order.volume_remain, order_id)) + updated_orders += 1 + + if update_rows: + await self.db_conn.executemany( + 'UPDATE orders SET volume_remain = ? WHERE order_id = ?', + update_rows, + ) + + if delete_rows: + await self.db_conn.executemany( + 'DELETE FROM orders WHERE order_id = ?', + delete_rows, ) - self.orders = updated_orders - return regions_updated + if update_rows or delete_rows: + await self.db_conn.commit() + + self.orders = [order for order in self.orders if order.volume_remain > 0] + + return { + 'affected_orders': updated_orders + removed_orders, + 'updated_orders': updated_orders, + 'removed_orders': removed_orders, + 'ignored_orders': ignored_orders, + 'consumed_units': consumed_units, + } def check_expired(self, region_id: int) -> bool: """ Checks if the cached orders for a region have expired. - - Args: - region_id: The ID of the region to check. - - Returns: - True if the orders have expired or are not cached, False otherwise. """ current_time = datetime.utcnow() expiry_time = self.expires_times.get(region_id) return expiry_time is None or current_time >= expiry_time - def download_orders(self, region_id: int) -> List[Order]: + async def download_orders(self, region_id: int) -> List[Order]: """ Downloads orders for a given region and updates the expiry time. - - Args: - region_id: The ID of the region to download orders for. - - Returns: - A list of Order instances for the region. """ - raw_orders, expiry = get_region_orders(region_id) - expiry_time = datetime.strptime(expiry, "%a, %d %b %Y %H:%M:%S %Z") - self.expires_times[region_id] = expiry_time + raw_orders, expiry = await get_region_orders(region_id) + if expiry: + expiry_time = datetime.strptime(expiry, "%a, %d %b %Y %H:%M:%S %Z") + self.expires_times[region_id] = expiry_time + else: + self.expires_times[region_id] = datetime.utcnow() # Fallback orders = self.add_item_info_to_orders(raw_orders) - return orders + return list(orders) - def add_item_info_to_orders(self, raw_orders: List[Dict]) -> List[Order]: + def add_item_info_to_orders(self, raw_orders: List[Dict[str, Any]]) -> List[Order]: """ Enhances raw order data with item information and converts them into Order instances. - - Args: - raw_orders: A list of raw order dictionaries from the API. - - Returns: - A list of Order instances with additional item information. """ items = load_items() orders: List[Order] = [] for raw_order in raw_orders: item_info = items.get(raw_order['type_id'], {}) + # Handle different datetime formats from ESI if missing timezone + issued_str = raw_order['issued'] + if issued_str.endswith('Z'): + issued_str = issued_str[:-1] + '+00:00' + order = Order( order_id=raw_order['order_id'], region_id=raw_order['region_id'], @@ -136,7 +260,7 @@ class Market: type_id=raw_order['type_id'], is_buy_order=raw_order['is_buy_order'], price=raw_order['price'], - issued=datetime.strptime(raw_order['issued'], "%Y-%m-%dT%H:%M:%S%z"), + issued=datetime.fromisoformat(issued_str), volume_remain=raw_order['volume_remain'], item_name=item_info.get('type_name', f"Not found: {raw_order['type_id']}"), item_cargo_volume=item_info.get('volume', 1.0), @@ -145,23 +269,19 @@ class Market: return orders - def cache_orders(self) -> None: + async def cache_orders(self) -> None: """Caches the current list of orders to a JSON file.""" try: with ORDERS_CACHE.open("w", encoding="utf-8") as f: - # Serialize orders to dictionaries and handle datetime serialization orders_data = [self._serialize_order(order) for order in self.orders] json.dump(orders_data, f, ensure_ascii=False, indent=4) logger.info("Orders cached successfully.") except IOError as e: logger.error(f"Error caching orders: {e}") - def load_orders(self) -> List[Order]: + async def load_orders(self) -> List[Order]: """ Loads orders from the cache file. - - Returns: - A list of Order instances loaded from the cache. """ try: with ORDERS_CACHE.open("r", encoding="utf-8") as f: @@ -174,30 +294,27 @@ class Market: return self.orders - def _serialize_order(self, order: Order) -> Dict: + def _serialize_order(self, order: Order) -> Dict[str, Any]: """ Serializes an Order instance to a dictionary suitable for JSON serialization. - - Args: - order: The Order instance to serialize. - - Returns: - A dictionary representation of the Order. """ order_dict = asdict(order) - # Convert datetime objects to ISO format strings order_dict['issued'] = order.issued.isoformat() return order_dict - def _deserialize_order(self, data: Dict) -> Order: + def _deserialize_order(self, data: Dict[str, Any]) -> Order: """ Deserializes a dictionary into an Order instance. - - Args: - data: The dictionary representation of an Order. - - Returns: - An Order instance. """ - data['issued'] = datetime.fromisoformat(data['issued']) - return Order(**data) + return Order( + order_id=data['order_id'], + region_id=data['region_id'], + location_id=data['location_id'], + type_id=data['type_id'], + is_buy_order=data['is_buy_order'], + price=data['price'], + issued=datetime.fromisoformat(data['issued']), + volume_remain=data['volume_remain'], + item_name=data['item_name'], + item_cargo_volume=data['item_cargo_volume'] + ) diff --git a/backend/src/pathfinder.py b/backend/src/pathfinder.py index 4ce2df3..a08d47c 100644 --- a/backend/src/pathfinder.py +++ b/backend/src/pathfinder.py @@ -1,13 +1,17 @@ import logging from dataclasses import dataclass +from typing import Optional + from tqdm import tqdm from arbitrage import Trade -from graph import NavigationGraph as Graph +from graph import NavigationGraph as Graph, LegMetrics logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) +LegMetricLookup = dict[tuple[int, int], Optional[LegMetrics]] + @dataclass class Ship: location: int @@ -19,7 +23,7 @@ class Ship: risk_cost: float -def pathfinder(order_matches: list[Trade], graph: Graph, ship: Ship) -> list[Trade]: +def pathfinder(order_matches: list[Trade], graph: Graph, ship: Ship, leg_metrics: Optional[LegMetricLookup] = None) -> list[Trade]: """ Processes order matches by filtering and calculating net profit. @@ -33,11 +37,11 @@ def pathfinder(order_matches: list[Trade], graph: Graph, ship: Ship) -> list[Tra """ order_matches = filter_order_matches_not_in_graph(order_matches, graph) logger.info("After filtering order matches not in graph: %d", len(order_matches)) - - order_matches = filter_order_matches_same_region(order_matches, graph) - logger.info("After filtering order matches to same region: %d", len(order_matches)) - order_matches = calculate_net_profit(order_matches, graph, ship) + if leg_metrics is None: + leg_metrics = graph.build_trade_leg_metrics(ship["location"], order_matches) + + order_matches = calculate_net_profit(order_matches, ship["location"], leg_metrics) order_matches = filter_order_matches_low_profit(order_matches) logger.info("After filtering low profit order matches: %d", len(order_matches)) @@ -60,42 +64,7 @@ def filter_order_matches_not_in_graph(order_matches: list[Trade], graph: Graph) if order_match.from_station in graph.graph.nodes and order_match.to_station in graph.graph.nodes ] -def filter_order_matches_same_region(order_matches: list[Trade], graph: Graph) -> list[Trade]: - """ - Filters out order matches where the from_region and to_region are different. - Keeps only order matches within the same region. - - Args: - order_matches: List of trade opportunities. - graph: Navigation graph. - - Returns: - Filtered list of trade opportunities where both stations are in the same region. - """ - filtered_order_matches = [] - - for order_match in order_matches: - from_station = order_match.from_station - to_station = order_match.to_station - - from_station_data = graph.stations.get(from_station) - to_station_data = graph.stations.get(to_station) - - if not from_station_data or not to_station_data: - continue # Skip if station data is missing - - from_solar_system_id = from_station_data["solar_system_id"] - to_solar_system_id = to_station_data["solar_system_id"] - - from_region = graph.solar_systems[from_solar_system_id]["region_id"] - to_region = graph.solar_systems[to_solar_system_id]["region_id"] - - if from_region == to_region: - filtered_order_matches.append(order_match) - - return filtered_order_matches - -def calculate_net_profit(order_matches: list[Trade], graph: Graph, ship: Ship) -> list[Trade]: +def calculate_net_profit(order_matches: list[Trade], ship_location: int, leg_metrics: LegMetricLookup) -> list[Trade]: """ Calculates the net profit for each order match by subtracting transport cost from gross profit. @@ -108,12 +77,12 @@ def calculate_net_profit(order_matches: list[Trade], graph: Graph, ship: Ship) - List of trade opportunities with net_profit calculated. """ logger.info("Calculating estimated profits") - for order_match in tqdm(order_matches): + for order_match in tqdm(order_matches, desc="Scoring trades"): try: # Calculate transport cost from ship location to from_station - cost_to_source = graph.shortest_path_length(ship["location"], order_match.from_station) + cost_to_source = leg_metrics.get((ship_location, order_match.from_station)) # Calculate transport cost from from_station to to_station - cost_to_destination = graph.shortest_path_length(order_match.from_station, order_match.to_station) + cost_to_destination = leg_metrics.get((order_match.from_station, order_match.to_station)) # Check if paths exist if cost_to_source is None or cost_to_destination is None: @@ -121,7 +90,7 @@ def calculate_net_profit(order_matches: list[Trade], graph: Graph, ship: Ship) - order_match.net_profit = None continue - total_transport_cost = cost_to_source + cost_to_destination + total_transport_cost = cost_to_source.weight + cost_to_destination.weight order_match.net_profit = order_match.gross_profit - total_transport_cost except Exception as e: logger.error(f"Error calculating net profit for order match {order_match}: {e}") diff --git a/backend/src/route.py b/backend/src/route.py index 2bcae7e..8379e6d 100644 --- a/backend/src/route.py +++ b/backend/src/route.py @@ -1,10 +1,11 @@ +import heapq import logging -from dataclasses import dataclass, field +from collections import defaultdict from typing import Optional from tqdm import tqdm -from graph import NavigationGraph as Graph +from graph import NavigationGraph as Graph, LegMetrics from pathfinder import Ship from arbitrage import Trade from typing import Any @@ -16,8 +17,198 @@ MAX_TRADES_TO_CONSIDER = 20_000 MAX_CAPITAL = 100_000_000 Route = list[dict[str, Any]] +RoutePlan = tuple[Optional[Route], Optional[dict], list[Trade], list[int], list[int]] +LegMetricLookup = dict[tuple[int, int], Optional[LegMetrics]] -def route(trades: list[Trade], graph: Graph, ship: Ship) -> tuple[Optional[Route], Optional[dict]]: + +def outstanding_trades_for_locations(location_ids: list[int], trades: list[Trade]) -> list[Trade]: + remaining_locations = set(location_ids) + return [trade for trade in trades if trade.to_station in remaining_locations] + + +def calculate_peak_capital(location_ids: list[int], trades: list[Trade], ship_cost: float) -> float: + outstanding_trades = outstanding_trades_for_locations(location_ids, trades) + if not outstanding_trades: + return ship_cost + + remaining_locations = set(location_ids) + committed_capital = ship_cost + sum( + trade.from_price * trade.quantity + for trade in outstanding_trades + if trade.from_station not in remaining_locations + ) + peak_capital = committed_capital + + for location_id in location_ids: + committed_capital -= sum( + trade.from_price * trade.quantity + for trade in outstanding_trades + if trade.to_station == location_id + ) + committed_capital = max(ship_cost, committed_capital) + committed_capital += sum( + trade.from_price * trade.quantity + for trade in outstanding_trades + if trade.from_station == location_id + ) + peak_capital = max(peak_capital, committed_capital) + + return peak_capital + + +def summarize_graph_path(graph: Graph, path: list[int]) -> LegMetrics: + weight = 0.0 + transport_time = 0.0 + risk = 0.0 + + for source, target in zip(path, path[1:]): + edge = graph.graph[source][target] + weight += edge["weight"] + transport_time += edge["time"] + risk += edge["risk"] + + return LegMetrics(weight=weight, time=transport_time, risk=risk) + + +def summarize_location_sequence(graph: Graph, location_ids: list[int]) -> Optional[tuple[float, float]]: + if len(location_ids) < 2: + return 0.0, 0.0 + + transport_time = 0.0 + risk = 0.0 + + for source, target in zip(location_ids, location_ids[1:]): + if source == target: + continue + + if graph.graph.has_edge(source, target): + leg_metrics = LegMetrics( + weight=graph.graph[source][target]["weight"], + time=graph.graph[source][target]["time"], + risk=graph.graph[source][target]["risk"], + ) + else: + path = graph.shortest_path(source, target) + if path is None: + return None + leg_metrics = summarize_graph_path(graph, path) + + transport_time += leg_metrics.time + risk += leg_metrics.risk + + return transport_time, risk + + +def build_route_info( + location_ids: list[int], + trades: list[Trade], + ship: Ship, + transport_time: float, + risk: float, +) -> Optional[dict]: + outstanding_trades = outstanding_trades_for_locations(location_ids, trades) + if not outstanding_trades: + return None + + capital = calculate_peak_capital(location_ids, outstanding_trades, ship.get("ship_cost", 0.0)) + gross_profit = sum(trade.gross_profit for trade in outstanding_trades) + net_profit = gross_profit - risk * capital + profit_rate = net_profit / transport_time if transport_time > 0 else 0.0 + + return { + "profit_rate": profit_rate, + "risk": risk, + "capital": capital, + "transport_time": transport_time, + "gross_profit": gross_profit, + "net_profit": net_profit, + } + + +def extract_route_location_ids(route_steps: Route) -> list[int]: + return [ + step["location_id"] + for step in route_steps + if isinstance(step, dict) and isinstance(step.get("location_id"), int) + ] + + +def calculate_route_info( + route_steps: Route, + trades: list[Trade], + graph: Graph, + ship: Ship, + metric_path: Optional[list[int]] = None, +) -> Optional[dict]: + location_ids = [ + location_id + for location_id in extract_route_location_ids(route_steps) + ] + if not location_ids: + return None + + if metric_path is not None: + if len(metric_path) < 2: + transport_time = 0.0 + risk = 0.0 + else: + path_metrics = summarize_graph_path(graph, metric_path) + transport_time = path_metrics.time + risk = path_metrics.risk + else: + path_metrics = summarize_location_sequence(graph, location_ids) + if path_metrics is None: + return None + + transport_time, risk = path_metrics + + return build_route_info(location_ids, trades, ship, transport_time, risk) + + +def trade_capital_efficiency(trade: Trade) -> float: + invested_capital = trade.from_price * trade.quantity + return trade.gross_profit / invested_capital if invested_capital > 0 else 0.0 + + +def trade_profit_density(trade: Trade) -> float: + return trade.gross_profit / trade.cargo if trade.cargo > 0 else 0.0 + + +def count_unique_triplets(trades: list[Trade], ship_location: Optional[int]) -> int: + if ship_location is None: + return 0 + + return len({(ship_location, trade.from_station, trade.to_station) for trade in trades}) + + +def build_ranked_triplets(trades: list[Trade], ship_location: int) -> list[tuple[int, int, int]]: + ranked_triplets: list[tuple[int, int, int]] = [] + seen_triplets: set[tuple[int, int, int]] = set() + + for trade in sorted(trades, key=trade_capital_efficiency, reverse=True): + triplet = (ship_location, trade.from_station, trade.to_station) + if triplet in seen_triplets: + continue + + seen_triplets.add(triplet) + ranked_triplets.append(triplet) + + return ranked_triplets + + +def build_station_pair_index(trades: list[Trade]) -> dict[tuple[int, int], list[Trade]]: + trades_by_station_pair: dict[tuple[int, int], list[Trade]] = defaultdict(list) + + for trade in trades: + trades_by_station_pair[(trade.from_station, trade.to_station)].append(trade) + + for station_pair_trades in trades_by_station_pair.values(): + station_pair_trades.sort(key=trade_profit_density, reverse=True) + + return dict(trades_by_station_pair) + + +def route(trades: list[Trade], graph: Graph, ship: Ship, leg_metrics: Optional[LegMetricLookup] = None) -> RoutePlan: """ Given a list of trades, find the best route starting from the ship's location. @@ -29,92 +220,109 @@ def route(trades: list[Trade], graph: Graph, ship: Ship) -> tuple[Optional[Route Returns: A tuple containing the best route and route information. """ - trades.sort( - key=lambda trade: trade.gross_profit / (trade.from_price * trade.quantity) if trade.from_price * trade.quantity > 0 else 0, - reverse=True - ) + if not trades: + return None, None, [], [], [] + + ranked_triplets = build_ranked_triplets(trades, ship["location"]) + trades_by_station_pair = build_station_pair_index(trades) + + if leg_metrics is None: + leg_metrics = graph.build_trade_leg_metrics(ship["location"], trades) best_profit_rate = 0.0 best_route: Optional[Route] = None best_route_info: Optional[dict] = None + best_route_stations: Optional[tuple[int, int, int]] = None + best_optimized_trades: Optional[list[Trade]] = None - seen_routes = set() - - for trade in tqdm(trades[:MAX_TRADES_TO_CONSIDER], desc="Evaluating trades"): - route_stations = (ship["location"], trade.from_station, trade.to_station) - - if route_stations in seen_routes: + for route_stations in tqdm(ranked_triplets[:MAX_TRADES_TO_CONSIDER], desc="Evaluating triplets"): + optimized_trades = select_trades(route_stations, trades_by_station_pair, ship) + if not optimized_trades: continue - seen_routes.add(route_stations) - optimized_trades = select_trades(list(route_stations), trades, ship) - total_profit = sum(t.gross_profit for t in optimized_trades) - - if not optimized_trades or total_profit <= 0: - continue + source_leg = leg_metrics.get((route_stations[0], route_stations[1])) + destination_leg = leg_metrics.get((route_stations[1], route_stations[2])) - try: - path = graph.shortest_path(ship["location"], trade.from_station)[:-1] - path += graph.shortest_path(trade.from_station, trade.to_station) - except Exception as e: - logger.error(f"Error finding path: {e}") + if source_leg is None or destination_leg is None: continue - risk = sum(graph.graph[u][v]['risk'] for u, v in zip(path, path[1:])) - transport_time = sum(graph.graph[u][v]['time'] for u, v in zip(path, path[1:])) + risk = source_leg.risk + destination_leg.risk + transport_time = source_leg.time + destination_leg.time - capital = ship.get('ship_cost', 0.0) + sum(t.from_price * t.quantity for t in optimized_trades) - - if transport_time == 0: - logger.warning("Transport time is zero, skipping this route to avoid division by zero.") + candidate_route_info = build_route_info(list(route_stations), optimized_trades, ship, transport_time, risk) + if candidate_route_info is None: continue - net_profit = total_profit - risk * capital - profit_rate = net_profit / transport_time + profit_rate = candidate_route_info["profit_rate"] if profit_rate > best_profit_rate: logger.info( f"New best profit rate: {profit_rate * 3600:.2f} per hour, " - f"Risk: {risk:.4f}, Capital: {capital:.2f}, " - f"Transport time: {transport_time:.2f} sec, Gross profit: {total_profit:.2f}" + f"Risk: {risk:.4f}, Capital: {candidate_route_info['capital']:.2f}, " + f"Transport time: {transport_time:.2f} sec, Gross profit: {candidate_route_info['gross_profit']:.2f}" ) - formatted_route = graph.formatted_route(path) - final_route = set_actions(formatted_route, optimized_trades) - best_route = final_route - best_route_info = { - "profit_rate": profit_rate, - "risk": risk, - "capital": capital, - "transport_time": transport_time, - "gross_profit": total_profit, - "net_profit": net_profit, - } + best_route_info = candidate_route_info best_profit_rate = profit_rate + best_route_stations = route_stations + best_optimized_trades = optimized_trades + + if best_route_stations is None or best_optimized_trades is None: + return None, None, [], [], [] + + path = reconstruct_route_path(graph, best_route_stations) + if path is None: + logger.error("Unable to reconstruct best route path for stations %s", best_route_stations) + return None, None, [], [], [] + + formatted_route, raw_offsets = graph.formatted_route_with_offsets(path) + best_route = set_actions(formatted_route, best_optimized_trades) + best_route_info = calculate_route_info(best_route, best_optimized_trades, graph, ship, metric_path=path) or best_route_info - return best_route, best_route_info + return best_route, best_route_info, best_optimized_trades, path, raw_offsets -def select_trades(stations: list[int], trades: list[Trade], ship: Ship) -> list[Trade]: +def reconstruct_route_path(graph: Graph, route_stations: tuple[int, int, int]) -> Optional[list[int]]: + source_path = graph.shortest_path(route_stations[0], route_stations[1]) + if source_path is None: + return None + + destination_path = graph.shortest_path(route_stations[1], route_stations[2]) + if destination_path is None: + return None + + path = source_path[:-1] + path.extend(destination_path) + return path + + +def select_trades(route_stations: tuple[int, int, int], trades_by_station_pair: dict[tuple[int, int], list[Trade]], ship: Ship) -> list[Trade]: """ - Given a list of stations, find the optimal trades to add. + Given a triplet of stations, find the optimal trades to add. Args: - stations: List of station IDs. - trades: List of Trade instances. + route_stations: Ordered station triplet for the route. + trades_by_station_pair: Trade instances grouped and pre-sorted by ordered station pair. ship: Ship parameters. Returns: A list of optimized Trade instances. """ - filtered_trades = [ - trade for trade in trades - if trade.from_station in stations and trade.to_station in stations and - stations.index(trade.from_station) < stations.index(trade.to_station) + candidate_pairs: list[tuple[int, int]] = [] + for index, from_station in enumerate(route_stations): + for to_station in route_stations[index + 1:]: + station_pair = (from_station, to_station) + if station_pair not in candidate_pairs: + candidate_pairs.append(station_pair) + + pair_groups = [ + trades_by_station_pair[station_pair] + for station_pair in candidate_pairs + if station_pair in trades_by_station_pair ] - filtered_trades.sort( - key=lambda trade: trade.gross_profit / trade.cargo if trade.cargo > 0 else 0, - reverse=True - ) + if not pair_groups: + return [] + + filtered_trades = heapq.merge(*pair_groups, key=trade_profit_density, reverse=True) used_orders = {} optimized_trades = [] @@ -154,7 +362,7 @@ def select_trades(stations: list[int], trades: list[Trade], ship: Ship) -> list[ to_price=trade.to_price, quantity=max_quantity, cargo=trade.item_cargo_volume * max_quantity, - gross_profit=(trade.to_price - trade.from_price) * max_quantity, + gross_profit=trade.gross_profit * (max_quantity / trade.quantity) if trade.quantity > 0 else 0.0, net_profit=None # Will be calculated later if needed ) @@ -198,6 +406,8 @@ def set_actions(route: Route, trades: list[Trade]) -> Route: }) for step in route: - step["actions"].sort(key=lambda action: (action["action_type"], action["item"], action["price"])) + step["actions"].sort( + key=lambda action: ({"sell": 0, "buy": 1}[action["action_type"]], action["item"], action["price"]) + ) return route diff --git a/frontend/src/App.css b/frontend/src/App.css index cc8ccba..19cf08b 100644 --- a/frontend/src/App.css +++ b/frontend/src/App.css @@ -3,15 +3,6 @@ background: url(topo.svg), #111; } - -.App-logo { - height: 40vmin; - max-height: 256px; - pointer-events: none; - - mix-blend-mode: lighten; -} - .App-header { background-color: #282c34; min-height: 100vh; diff --git a/frontend/src/App.js b/frontend/src/App.js index 463566a..569ff74 100644 --- a/frontend/src/App.js +++ b/frontend/src/App.js @@ -1,4 +1,3 @@ -import logo from './logo.png'; import './App.css'; import Path from './Path'; @@ -13,7 +12,7 @@ function App() {
- {/* Sidebar for Options, LogComponent, and Logo */} + {/* Sidebar for route info, controls, and log output */}
@@ -21,12 +20,9 @@ function App() {
-
+
-
- logo -
); diff --git a/frontend/src/Info.js b/frontend/src/Info.js index 3fcfa3e..d46d914 100644 --- a/frontend/src/Info.js +++ b/frontend/src/Info.js @@ -1,9 +1,10 @@ import React from 'react'; -import usePolling from './usePolling'; +import useSSE from './useSSE'; import InfoDisplay from './InfoDisplay'; // Import the DataDisplay component const Info = () => { - const { data, error } = usePolling('http://localhost:8000/route_info', 1000); + const { data: sseData, error } = useSSE('http://localhost:8000/data_stream'); + const data = sseData ? sseData.route_info : null; return (
diff --git a/frontend/src/InfoDisplay.js b/frontend/src/InfoDisplay.js index 2253802..c8a77c1 100644 --- a/frontend/src/InfoDisplay.js +++ b/frontend/src/InfoDisplay.js @@ -36,7 +36,7 @@ const InfoDisplay = ({ data }) => { {riskPercent} ({riskCost} isk) - Capital: + Peak Capital: {capitalFormatted} isk diff --git a/frontend/src/Options.js b/frontend/src/Options.js index 4c8a7e9..b490f67 100644 --- a/frontend/src/Options.js +++ b/frontend/src/Options.js @@ -1,29 +1,173 @@ -import React, { useState } from 'react'; +import React, { useState, useEffect } from 'react'; import './Options.css'; +import useSSE from './useSSE'; const Options = () => { const [message, setMessage] = useState(''); + const [messageType, setMessageType] = useState('info'); + const [isSettingWaypoints, setIsSettingWaypoints] = useState(false); + const [settings, setSettings] = useState({ + cargo: '', + max_warp_speed: '', + max_subwarp_speed: '', + risk_cost: '', + ship_cost: '', + tax_rate: '', + profit_threshold: '', + max_capital: '', + snipe_profit_threshold: '' + }); + const { data: sseData } = useSSE('http://localhost:8000/data_stream'); + const route = Array.isArray(sseData?.route) ? sseData.route : []; + const hasRouteWaypoints = route.length > 1; - const handleClick = async () => { - setMessage('Sending request...'); + useEffect(() => { + fetch('http://localhost:8000/settings') + .then(res => res.json()) + .then(data => { + setSettings({ + cargo: data.cargo, + max_warp_speed: data.max_warp_speed, + max_subwarp_speed: data.max_subwarp_speed, + risk_cost: data.risk_cost, + ship_cost: data.ship_cost, + tax_rate: data.tax_rate, + profit_threshold: data.profit_threshold, + max_capital: data.max_capital, + snipe_profit_threshold: data.snipe_profit_threshold + }); + }) + .catch(err => console.error("Error fetching settings", err)); + }, []); + + const handleSettingsChange = (e) => { + const { name, value } = e.target; + setSettings(prev => ({ ...prev, [name]: value })); + }; + + const handleSaveSettings = async () => { try { - const response = await fetch('http://localhost:8000/create_route', { method: 'POST' }); - const data = await response.json(); - setMessage(`Response received: ${JSON.stringify(data)}`); + const response = await fetch('http://localhost:8000/settings', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + cargo: Number(settings.cargo), + max_warp_speed: Number(settings.max_warp_speed), + max_subwarp_speed: Number(settings.max_subwarp_speed), + risk_cost: Number(settings.risk_cost), + ship_cost: Number(settings.ship_cost), + tax_rate: Number(settings.tax_rate), + profit_threshold: Number(settings.profit_threshold), + max_capital: Number(settings.max_capital), + snipe_profit_threshold: Number(settings.snipe_profit_threshold) + }) + }); + if (response.ok) { + setMessageType('success'); + setMessage('Settings saved'); + setTimeout(() => setMessage(''), 3000); + } + } catch (error) { + setMessageType('error'); + setMessage(`Error: ${error.message}`); + } + }; + + const handleCreateRoute = async () => { + try { + await fetch('http://localhost:8000/create_route', { method: 'POST' }); } catch (error) { setMessage(`Error: ${error}`); } }; + const handleSetWaypoints = async () => { + if (!hasRouteWaypoints) { + setMessageType('error'); + setMessage('No route exists'); + return; + } + + setIsSettingWaypoints(true); + setMessageType('info'); + setMessage('Sending to EVE...'); + try { + const response = await fetch('http://localhost:8000/set_waypoints', { method: 'POST' }); + const resData = await response.json(); + const nextMessage = resData.message || resData.status || 'Waypoint request completed'; + const nextType = resData.success ? 'success' : resData.status === 'partial' ? 'warning' : 'error'; + setMessageType(nextType); + setMessage(nextMessage); + if (resData.success) { + setTimeout(() => setMessage(''), 4000); + } + } catch (error) { + setMessageType('error'); + setMessage(`Error: ${error}`); + } finally { + setIsSettingWaypoints(false); + } + }; + + const messageClassName = messageType === 'error' + ? 'mb-1 text-red-400' + : messageType === 'warning' + ? 'mb-1 text-yellow-300' + : 'mb-1 text-green-400'; + + const waypointButtonClassName = hasRouteWaypoints && !isSettingWaypoints + ? 'options-button flex-1 px-2 py-1 text-sm bg-green-700 hover:bg-green-600 text-white rounded' + : 'options-button flex-1 px-2 py-1 text-sm bg-green-900 text-white rounded opacity-50 cursor-not-allowed'; + return ( -
- + +
+ + {message &&

{message}

} + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + -

{message}

); } diff --git a/frontend/src/Path.js b/frontend/src/Path.js index f375743..3fd50ea 100644 --- a/frontend/src/Path.js +++ b/frontend/src/Path.js @@ -1,9 +1,10 @@ import React from 'react'; -import usePolling from './usePolling'; +import useSSE from './useSSE'; import DataDisplay from './DataDisplay'; // Import the DataDisplay component const Path = () => { - const { data, error } = usePolling('http://localhost:8000/route', 1000); + const { data: sseData, error } = useSSE('http://localhost:8000/data_stream'); + const data = sseData ? sseData.route : null; return (
diff --git a/readme.md b/readme.md index b145239..1daf57d 100644 --- a/readme.md +++ b/readme.md @@ -38,16 +38,17 @@ The project comprises: ### **Backend (Python)** - **Framework**: FastAPI for high-performance API development. -- **Asynchronous Processing**: `asyncio` for handling multiple concurrent API calls and tasks. +- **Asynchronous Processing**: `asyncio` and `aiohttp` for handling multiple concurrent API calls and non-blocking tasks. +- **Database**: In-memory SQLite (`aiosqlite`) for high-performance market order processing and indexing. - **API Integration**: - - Extensive use of the **EVE Swagger Interface (ESI)** API for fetching in-game data, such as trade orders, solar system details, and player locations. -- **Automation**: Automatically sets waypoints in the EVE Online client through API interactions. + - Extensive use of the **EVE Swagger Interface (ESI)** API for fetching in-game data, such as trade orders, solar system details, player locations, and wallet transactions. +- **Automation**: Set waypoints in the EVE Online client through API interactions on command. - **Streaming Data**: Implements `StreamingResponse` to deliver real-time updates to the frontend. ### **Frontend (React)** - **Framework**: React for building dynamic, responsive user interfaces. -- **State Updates**: Use of both polling and server-side events to update state +- **State Updates**: Real-time state updates using server-sent events (SSE). ### **Experimental Rust Backend** - A rewrite of the Python backend focused on improving performance in API-intensive operations like route calculations @@ -62,14 +63,14 @@ The project comprises: - Travel time, warp speeds, and stargate connections. - Risk factors, including security ratings of solar systems. - Suggests optimized paths for asset transportation and trading. - - Automatically sets waypoints in the EVE Online client, simplifying navigation for players. + - Allows pushing calculated routes as waypoints to the EVE Online client with a single click. 2. **Profit Optimization** - Identifies arbitrage opportunities by analyzing buy/sell orders across regions. - - Uses efficient filtering algorithms to recommend profitable trades. + - Tracks confirmed trades automatically via wallet transaction history to remove filled orders locally. 3. **Real-Time API Integration** - - Fetches live trade orders, player locations, and solar system details using the EVE Swagger Interface (ESI) API. + - Fetches live trade orders, player locations, recent wallet transactions, and solar system details using the EVE Swagger Interface (ESI) API. - Continuously updates routes and trade data in real-time. - Utilizes server-sent events to deliver live updates on trade opportunities, routes, and player location changes. @@ -78,30 +79,29 @@ The project comprises: **Algorithm Criteria & Limits** ------------------------------- -The route planning and arbitrage algorithms evaluate the profitability and safety of moving items using the following constraints: +The route planning and arbitrage algorithms evaluate the profitability and safety of moving items using configurable constraints: 1. **Ship & Financial Limits** - - **Cargo Capacity**: Hard limit of 600m³. Orders that don't fit are ignored. - - **Maximum Route Capital**: Caps the up-front item purchase cost at 100,000,000 ISK when compounding multiple items in a route. - - **Profit Threshold**: A single trade must yield a minimum gross profit of 100,000 ISK to be considered. + - **Cargo Capacity**: Default limit of 4000m³ (configurable). + - **Maximum Route Capital**: Caps the up-front item purchase cost at 20,000,000 ISK initially (configurable). + - **Profit Threshold**: A single trade must yield a minimum gross profit (default 100,000 ISK, configurable). 2. **Time & Navigational Costs** - - **Ship Speed**: Calculations factor in `max_warp_speed = 8.22 AU/s` and `max_subwarp_speed = 216.5 m/s`. + - **Ship Speed**: Subject to specified `max_warp_speed` and `max_subwarp_speed` (configurable via UI). - **Stargate Delay**: Each jump is assigned a fixed 9.0-second delay. - **In-system Warp**: Calculates exact 3D geometry for acceleration and deceleration. - - **Player Time Cost**: The algorithm assigns a real-life time value of ~4,166 ISK per second (`15_000_000 / 3600`) to punish overly long routes. + - **Player Time Cost**: The algorithm assigns a real-life configurable time value. 3. **Risk & Safety Penalties** - **Gank Zones**: Hardcoded danger zones (e.g., Ahbazon) multiply the risk penalty heavily. - **Security Status**: High-sec space carries virtually no penalty, low-sec applies a 0.005 penalty, and null-sec adds a massive 0.5 risk penalty. - - **Value at Risk**: The calculated risk percentage is multiplied against a theoretical 60m ISK penalty + the 50m ship value + the cargo value. + - **Value at Risk**: The calculated risk percentage is multiplied against the fixed risk penalty plus cargo values. 4. **Trade Execution & Filtering Strategy** - - **Market Tax**: Accounts for an 8% transaction tax on all trades automatically. - - **Arbitrage Capital Limits**: Rejects individual items exceeding 20,000,000 ISK up-front capital during initial data processing but allows up to 100,000,000 ISK when compounding multiple items continuously. - - **Same-Region Filtering**: Ensures that pickup and dropoff stations for a single item are within the same region. - - **Evaluation Limit**: Limits pathfinding to the top 20,000 trades to conserve CPU overhead. - - **Snipe Feature Execution**: Background polling monitors loop regions actively for single trades that boast >20,000,000 ISK gross profit. If triggered, it executes an automatic ESI API call to forcefully open your in-game client's market window to intercept the trade instantly. + - **Market Tax**: Accounts for a configurable transaction tax (default 8%). + - **Cross-Region Pairing**: Allows pickup and dropoff stations for a single item to be in different regions when both stations are present in the loaded market and navigation datasets. + - **Evaluation Limit**: Limits pathfinding to the top scoring trade triplets to conserve CPU overhead. + - **Snipe Feature Execution**: Background polling monitors loop regions actively for single trades that boast >20,000,000 ISK (configurable) gross profit. If triggered, it executes an automatic ESI API call to forcefully open your in-game client's market window to intercept the trade instantly. - **ESI Error Limits**: Pauses backend routines entirely if the X-Esi-Error-Limit-Remain headers fall below 50. - **The Final Metric**: The route chosen is the one that produces the highest `profit_rate` (which is `net_profit = total_profit - risk * capital` divided by `transport_time`). diff --git a/scripts/setup_auth.py b/scripts/setup_auth.py index d675ddd..c4f60c5 100644 --- a/scripts/setup_auth.py +++ b/scripts/setup_auth.py @@ -24,6 +24,7 @@ SCOPES = [ "esi-assets.read_assets.v1", "esi-location.read_location.v1", "esi-location.read_online.v1", + "esi-wallet.read_character_wallet.v1", "esi-ui.open_window.v1", "esi-ui.write_waypoint.v1", ] @@ -80,6 +81,7 @@ def main() -> int: secret_key = os.getenv("SECRET_KEY") or prompt_for_value("SECRET_KEY", secret=True) print("Open this URL in your browser and complete the EVE SSO flow:\n") + print("If you authenticated earlier without wallet access, reauthorization is required for transaction-backed route confirmation.\n") print(build_auth_url(client_id)) print("\nAfter the redirect, copy the 'code' query parameter and paste it below.") code = input("Authorization code: ").strip() -- cgit v1.3