TraderX_Tools/traderx_check.py
2026-09-06 22:47:27 +03:00

625 lines
25 KiB
Python

#!/usr/bin/env python3
"""
traderx_check.py - Validate and (optionally) fix TraderX legacy configuration.
Checks TraderXConfig/Products/*.json and TraderXConfig/Categories/*.json against
the rules enforced by the mod's Enforce Script loaders (TraderXProductId.IsValidUUID,
TraderXCategoryId.IsValidUUID, TraderXJsonProduct, TraderXJsonCategory), plus
cross-references to TraderXGeneralSettings.json.
Usage:
python3 traderx_check.py /path/to/profiles/TraderX # report only
python3 traderx_check.py /path/to/profiles/TraderX --fix # apply safe fixes
python3 traderx_check.py /path/to/profiles/TraderX --fix --no-backup
python3 traderx_check.py /path/to/profiles/TraderX --decode # also print decoded bitfields
The path can be the TraderX root, the TraderXConfig dir, or the profiles dir.
Fixes applied with --fix:
* rename product/category files whose names fail validation
(sanitise, append _001 / bump number on collision) and rewrite every
reference to the old id in categories, attachments and variants
* add missing fields with the mod's defaults
* coerce numeric strings ("5") to numbers, floats to ints where an int is expected
* drop unknown keys (e.g. a stray "productId" inside legacy files)
* remove references to products/categories that do not exist
* rewrite files with consistent 2-space indentation
Never fixed automatically (reported only): invalid JSON syntax, missing className,
prices/stock out of range, coefficient with unlimited stock, orphan products,
compiled files shadowing legacy files, license ids not found.
"""
import argparse
import json
import os
import re
import shutil
import sys
from datetime import datetime
# ---------------------------------------------------------------------------
# Rules mirrored from the mod source
# ---------------------------------------------------------------------------
PROD_RE = re.compile(r"^prod_([A-Za-z0-9_]+)_(\d{3})$")
CAT_RE = re.compile(r"^cat_([a-z0-9_]+)_(\d{3})$")
PRODUCT_SCHEMA = {
# key: (type, default)
"className": (str, ""),
"coefficient": (float, 1.0),
"maxStock": (int, -1),
"tradeQuantity": (int, 18), # buy full / sell full
"buyPrice": (int, -1),
"sellPrice": (int, -1),
"stockSettings": (int, 0),
"attachments": (list, []),
"variants": (list, []),
}
PRODUCT_ORDER = list(PRODUCT_SCHEMA.keys())
CATEGORY_SCHEMA = {
"isVisible": (bool, True),
"icon": (str, ""),
"categoryName": (str, ""),
"licensesRequired": (list, []),
"productIds": (list, []),
}
CATEGORY_ORDER = list(CATEGORY_SCHEMA.keys())
SELL_MODES = {0: "NO_MATTER", 1: "EMPTY", 2: "FULL", 3: "COEFFICIENT", 4: "STATIC"}
BUY_MODES = {0: "EMPTY", 1: "EMPTY(1)", 2: "FULL", 3: "COEFFICIENT", 4: "STATIC"}
RESTART_MODES = {0: "keep", 1: "reset to max", 2: "random"}
# ---------------------------------------------------------------------------
# Reporting
# ---------------------------------------------------------------------------
class Report:
def __init__(self):
self.errors = []
self.warnings = []
self.fixes = []
self.infos = []
def error(self, where, msg):
self.errors.append((where, msg))
def warn(self, where, msg):
self.warnings.append((where, msg))
def fix(self, where, msg):
self.fixes.append((where, msg))
def info(self, where, msg):
self.infos.append((where, msg))
def dump(self, show_info, group_threshold=10, ignore=()):
def section(title, items):
items = [(w, m) for w, m in items if not any(i in m for i in ignore)]
if not items:
return
print(f"\n=== {title} ({len(items)}) ===")
# group identical messages (ignoring the numbers in them) when repeated a lot
groups = {}
for where, msg in items:
key = re.sub(r"[-0-9.]+", "#", msg)
groups.setdefault(key, []).append((where, msg))
for key, lst in groups.items():
if len(lst) >= group_threshold:
print(f" ({len(lst)}x) {lst[0][1]}")
print(" " + ", ".join(w for w, _ in lst[:8]) + (", ..." if len(lst) > 8 else ""))
else:
for where, msg in lst:
print(f" [{where}] {msg}")
section("ERRORS", self.errors)
section("WARNINGS", self.warnings)
section("FIXES APPLIED" if applied else "FIXES AVAILABLE (run with --fix)", self.fixes)
if show_info:
section("INFO", self.infos)
print(
f"\nSummary: {len(self.errors)} errors, {len(self.warnings)} warnings, "
f"{len(self.fixes)} fixes {'applied' if applied else 'available'}"
)
applied = False # set by main()
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def find_config_dir(path):
"""Accept profiles dir, TraderX dir or TraderXConfig dir."""
path = os.path.abspath(path)
candidates = [
path,
os.path.join(path, "TraderXConfig"),
os.path.join(path, "TraderX", "TraderXConfig"),
]
for c in candidates:
if os.path.isdir(os.path.join(c, "Products")) or os.path.isdir(os.path.join(c, "Categories")):
return c
sys.exit(f"Could not find TraderXConfig with Products/ or Categories/ under {path}")
def load_json(path, rep):
try:
with open(path, "r", encoding="utf-8-sig") as f:
return json.load(f)
except json.JSONDecodeError as e:
rep.error(os.path.basename(path), f"invalid JSON: {e.msg} at line {e.lineno} col {e.colno}")
except OSError as e:
rep.error(os.path.basename(path), f"cannot read: {e}")
return None
def save_json(path, data):
with open(path, "w", encoding="utf-8") as f:
json.dump(data, f, indent=2, ensure_ascii=False)
f.write("\n")
def sanitize_middle(s, lowercase):
if lowercase:
s = s.lower()
s = re.sub(r"[^A-Za-z0-9_]", "_", s)
s = re.sub(r"_+", "_", s).strip("_")
return s or "item"
def make_valid_id(stem, prefix, regex, taken):
"""Turn an arbitrary filename stem into a valid id not present in `taken`."""
lowercase = prefix == "cat_"
if regex.match(stem) and stem not in taken:
return stem
m = regex.match(stem)
if m:
middle, num = m.group(1), int(m.group(2))
else:
body = stem[len(prefix):] if stem.lower().startswith(prefix) else stem
# if it already ends with _NNN (but middle was invalid) keep the number
m2 = re.match(r"^(.*)_(\d{3})$", body)
if m2:
middle, num = sanitize_middle(m2.group(1), lowercase), int(m2.group(2))
else:
middle, num = sanitize_middle(body, lowercase), 1
num = max(1, min(num, 999))
while True:
cand = f"{prefix}{middle}_{num:03d}"
if cand not in taken:
return cand
num += 1
if num > 999:
raise RuntimeError(f"no free id for {stem}")
def coerce(value, typ):
"""Return (coerced_value, changed) or (None, False) if impossible."""
if typ is bool:
if isinstance(value, bool):
return value, False
if isinstance(value, (int, float)):
return bool(value), True
if isinstance(value, str) and value.strip().lower() in ("true", "1", "yes"):
return True, True
if isinstance(value, str) and value.strip().lower() in ("false", "0", "no", ""):
return False, True
return None, False
if typ is int:
if isinstance(value, bool):
return int(value), True
if isinstance(value, int):
return value, False
if isinstance(value, float):
return int(round(value)), True
if isinstance(value, str):
try:
return int(round(float(value.strip()))), True
except ValueError:
return None, False
return None, False
if typ is float:
if isinstance(value, bool):
return float(value), True
if isinstance(value, (int, float)):
return value, False
if isinstance(value, str):
try:
return float(value.strip()), True
except ValueError:
return None, False
return None, False
if typ is str:
if isinstance(value, str):
return value, False
return str(value), True
if typ is list:
if isinstance(value, list):
return value, False
if value is None or value == "":
return [], True
if isinstance(value, str):
return [v.strip() for v in value.split(";") if v.strip()], True
return None, False
return value, False
def decode_trade_quantity(tq):
return (
f"sell={SELL_MODES.get(tq & 7, tq & 7)} buy={BUY_MODES.get((tq >> 3) & 7, (tq >> 3) & 7)} "
f"sellVal={(tq >> 6) & 0x1FFF} buyVal={tq >> 19}"
)
def decode_stock_settings(ss):
return f"destock={(ss & 0x7F) / 100:.2f} restart={RESTART_MODES.get((ss >> 7) & 3, (ss >> 7) & 3)}"
# ---------------------------------------------------------------------------
# Loading
# ---------------------------------------------------------------------------
def scan_folder(folder, rep):
"""Return {stem: {'path':..., 'data':...}} for every .json in folder."""
out = {}
if not os.path.isdir(folder):
return out
for name in sorted(os.listdir(folder)):
if not name.lower().endswith(".json"):
continue
stem = name[:-5]
data = load_json(os.path.join(folder, name), rep)
out[stem] = {"path": os.path.join(folder, name), "data": data}
return out
def schema_fix(stem, data, schema, order, rep, do_fix):
"""Add defaults, coerce types, drop unknown keys. Mutates & returns data."""
if not isinstance(data, dict):
rep.error(stem, f"top-level JSON is a {type(data).__name__}, expected an object {{...}} - file skipped")
return data, False
changed = False
for key, (typ, default) in schema.items():
if key not in data:
rep.fix(stem, f"missing '{key}' -> default {default!r}")
data[key] = default if not isinstance(default, list) else []
changed = True
continue
newval, ch = coerce(data[key], typ)
if newval is None:
rep.error(stem, f"'{key}' has wrong type {type(data[key]).__name__} = {data[key]!r}")
elif ch:
rep.fix(stem, f"'{key}': {data[key]!r} -> {newval!r}")
data[key] = newval
changed = True
for key in list(data.keys()):
if key not in schema:
rep.fix(stem, f"unknown key '{key}' removed")
del data[key]
changed = True
ordered = {k: data[k] for k in order if k in data}
if list(data.keys()) != list(ordered.keys()):
changed = True
data.clear()
data.update(ordered)
return data, changed
# ---------------------------------------------------------------------------
# Main validation
# ---------------------------------------------------------------------------
def main():
global applied
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("path", help="TraderX root, TraderXConfig dir, or server profiles dir")
ap.add_argument("--fix", action="store_true", help="apply safe fixes (rename files, add defaults, fix refs)")
ap.add_argument("--no-backup", action="store_true", help="do not create a backup before fixing")
ap.add_argument("--fix-settings", action="store_true",
help="with --fix: also rewrite renamed category ids inside TraderXGeneralSettings.json traders[].categoriesId")
ap.add_argument("--decode", action="store_true", help="print decoded tradeQuantity/stockSettings per product")
ap.add_argument("--ignore", action="append", default=[], metavar="TEXT",
help="hide warnings/errors whose message contains TEXT (repeatable)")
ap.add_argument("--no-group", action="store_true", help="never collapse repeated messages")
ap.add_argument("--quiet-info", action="store_true", help="hide INFO section")
args = ap.parse_args()
applied = args.fix
cfg = find_config_dir(args.path)
prod_dir = os.path.join(cfg, "Products")
cat_dir = os.path.join(cfg, "Categories")
rep = Report()
print(f"TraderXConfig: {cfg}")
# --- compiled shadowing ------------------------------------------------
for fname in ("Products_compiled.json", "Categories_compiled.json"):
p = os.path.join(cfg, "Compiled", fname)
if os.path.exists(p) and os.path.getsize(p) > 2:
rep.warn("Compiled/" + fname,
"exists - the mod loads this INSTEAD of the Products/ and Categories/ folders. "
"Delete it (or switch to CSV/COMPILED workflow) if you edit legacy files.")
# --- backup ------------------------------------------------------------
if args.fix and not args.no_backup:
stamp = datetime.now().strftime("%Y%m%d_%H%M%S")
bdir = os.path.join(cfg, f"Backup_traderx_check_{stamp}")
os.makedirs(bdir, exist_ok=True)
for d in (prod_dir, cat_dir):
if os.path.isdir(d):
shutil.copytree(d, os.path.join(bdir, os.path.basename(d)))
print(f"Backup written to {bdir}")
products = scan_folder(prod_dir, rep)
categories = scan_folder(cat_dir, rep)
print(f"Found {len(products)} product files, {len(categories)} category files")
# --- 1. filenames / ids ------------------------------------------------
rename_prod = {} # old stem -> new stem
taken = set(s for s in products if PROD_RE.match(s))
for stem in list(products):
if PROD_RE.match(stem):
continue
new = make_valid_id(stem, "prod_", PROD_RE, taken)
taken.add(new)
rename_prod[stem] = new
rep.error(stem + ".json", f"invalid product filename - mod will SKIP this file (fix: rename to {new}.json)")
rep.fix(stem, f"rename -> {new}")
rename_cat = {}
taken = set(s for s in categories if CAT_RE.match(s))
for stem in list(categories):
if CAT_RE.match(stem):
continue
new = make_valid_id(stem, "cat_", CAT_RE, taken)
taken.add(new)
rename_cat[stem] = new
rep.error(stem + ".json", f"invalid category filename - mod will SKIP this file (fix: rename to {new}.json)")
rep.fix(stem, f"rename -> {new}")
def new_pid(old):
return rename_prod.get(old, old)
def new_cid(old):
return rename_cat.get(old, old)
all_product_ids = {new_pid(s) for s in products if products[s]["data"] is not None}
all_category_ids = {new_cid(s) for s in categories if categories[s]["data"] is not None}
# --- 2. product contents ----------------------------------------------
dirty_products = set()
class_seen = {}
for stem, entry in products.items():
data = entry["data"]
if data is None:
continue
label = new_pid(stem)
data, changed = schema_fix(label, data, PRODUCT_SCHEMA, PRODUCT_ORDER, rep, args.fix)
if changed:
dirty_products.add(stem)
if not isinstance(data, dict):
continue
cn = data.get("className", "")
if not isinstance(cn, str) or not cn.strip():
rep.error(label, "className is empty")
else:
key = cn.lower()
if key in class_seen:
rep.info(label, f"className '{cn}' also used by {class_seen[key]} (ok if intentional, e.g. variants)")
else:
class_seen[key] = label
bp, sp, ms, co = data.get("buyPrice"), data.get("sellPrice"), data.get("maxStock"), data.get("coefficient")
if isinstance(bp, int) and bp < -1:
rep.error(label, f"buyPrice {bp} < -1")
if isinstance(sp, int) and sp < -1:
rep.error(label, f"sellPrice {sp} < -1")
if isinstance(bp, int) and isinstance(sp, int) and bp == -1 and sp == -1:
rep.warn(label, "buyPrice and sellPrice are both -1 - item can neither be bought nor sold")
if isinstance(bp, int) and isinstance(sp, int) and bp >= 0 and sp > bp:
rep.warn(label, f"sellPrice {sp} > buyPrice {bp} - players can profit by buy/sell loops")
if isinstance(ms, int) and ms < -1:
rep.error(label, f"maxStock {ms} < -1")
if isinstance(co, (int, float)):
if co <= 0:
rep.error(label, f"coefficient {co} must be > 0")
elif co != 1.0 and ms == -1:
rep.warn(label, f"coefficient {co} has no effect because maxStock is -1 (unlimited)")
elif co < 0.5 and isinstance(ms, int) and ms > 3:
rep.warn(label, f"coefficient {co} is very low - price at stock {ms} = {co ** (ms - 1) * 100:.1f}% of base")
tq, ss = data.get("tradeQuantity"), data.get("stockSettings")
if isinstance(tq, int):
if tq < 0:
rep.error(label, f"tradeQuantity {tq} negative")
elif (tq & 7) > 4 or ((tq >> 3) & 7) > 4:
rep.warn(label, f"tradeQuantity {tq} has unknown mode bits ({decode_trade_quantity(tq)})")
elif args.decode:
rep.info(label, f"tradeQuantity {tq}: {decode_trade_quantity(tq)}")
if isinstance(ss, int):
if ss < 0 or (ss & 0x7F) > 100 or ((ss >> 7) & 3) == 3:
rep.warn(label, f"stockSettings {ss} out of range ({decode_stock_settings(ss)})")
elif args.decode:
rep.info(label, f"stockSettings {ss}: {decode_stock_settings(ss)}")
for field in ("attachments", "variants"):
lst = data.get(field)
if not isinstance(lst, list):
continue
newlst = []
for ref in lst:
if not isinstance(ref, str):
rep.fix(label, f"{field}: non-string entry {ref!r} removed")
continue
r = new_pid(ref)
if r != ref:
rep.fix(label, f"{field}: reference {ref} -> {r}")
if r not in all_product_ids:
rep.error(label, f"{field}: references unknown product '{ref}'")
rep.fix(label, f"{field}: dangling reference '{ref}' removed")
continue
if r == label:
rep.warn(label, f"{field}: references itself")
if r in newlst:
rep.fix(label, f"{field}: duplicate '{r}' removed")
continue
newlst.append(r)
if newlst != lst:
data[field] = newlst
dirty_products.add(stem)
# --- 3. category contents ---------------------------------------------
dirty_cats = set()
used_products = set()
name_seen = {}
# license ids from general settings (optional)
license_ids = None
gs_path = os.path.join(cfg, "TraderXGeneralSettings.json")
gs = load_json(gs_path, rep) if os.path.exists(gs_path) else None
if isinstance(gs, dict):
license_ids = {l.get("licenseId") for l in gs.get("licenses", []) if isinstance(l, dict)}
license_names = {l.get("licenseName", "").lower(): l.get("licenseId") for l in gs.get("licenses", []) if isinstance(l, dict)}
else:
license_names = {}
for stem, entry in categories.items():
data = entry["data"]
if data is None:
continue
label = new_cid(stem)
data, changed = schema_fix(label, data, CATEGORY_SCHEMA, CATEGORY_ORDER, rep, args.fix)
if changed:
dirty_cats.add(stem)
if not isinstance(data, dict):
continue
name = data.get("categoryName", "")
if not name:
rep.warn(label, "categoryName is empty")
elif name.lower() in name_seen:
rep.warn(label, f"categoryName '{name}' duplicated in {name_seen[name.lower()]}")
else:
name_seen[name.lower()] = label
lst = data.get("productIds", [])
newlst = []
for ref in lst:
if not isinstance(ref, str):
rep.fix(label, f"productIds: non-string entry {ref!r} removed")
continue
r = new_pid(ref)
if r != ref:
rep.fix(label, f"productIds: {ref} -> {r}")
if r not in all_product_ids:
rep.error(label, f"productIds: unknown product '{ref}'")
rep.fix(label, f"productIds: dangling '{ref}' removed")
continue
if r in newlst:
rep.fix(label, f"productIds: duplicate '{r}' removed")
continue
newlst.append(r)
used_products.add(r)
if newlst != lst:
data["productIds"] = newlst
dirty_cats.add(stem)
if not newlst:
rep.warn(label, "category has no products")
lic = data.get("licensesRequired", [])
if license_ids is not None:
newlic = []
for l in lic:
if l in license_ids:
newlic.append(l)
elif isinstance(l, str) and l.lower() in license_names:
rep.fix(label, f"licensesRequired: name '{l}' -> id {license_names[l.lower()]}")
newlic.append(license_names[l.lower()])
else:
rep.error(label, f"licensesRequired: '{l}' is not a licenseId in TraderXGeneralSettings.json")
newlic.append(l) # keep - don't silently unlock categories
if newlic != lic:
data["licensesRequired"] = newlic
dirty_cats.add(stem)
# --- 4. orphans & trader references ------------------------------------
for pid in sorted(all_product_ids - used_products):
rep.warn(pid, "product is not listed in any category (only reachable as attachment/variant)")
if isinstance(gs, dict):
seen_npc = set()
for t in gs.get("traders", []):
if not isinstance(t, dict):
continue
tid = t.get("npcId")
tl = f"trader {tid} ({t.get('givenName', '?')})"
if tid in seen_npc:
rep.error(tl, "duplicate npcId")
seen_npc.add(tid)
for c in t.get("categoriesId", []):
if new_cid(c) not in all_category_ids:
rep.error(tl, f"categoriesId references unknown category '{c}'")
elif c in rename_cat:
if args.fix_settings:
rep.fix(tl, f"categoriesId '{c}' -> '{new_cid(c)}' in TraderXGeneralSettings.json")
else:
rep.warn(tl, f"categoriesId '{c}' will be renamed to '{new_cid(c)}' - "
f"update TraderXGeneralSettings.json manually or rerun with --fix --fix-settings")
if not t.get("categoriesId"):
rep.warn(tl, "trader has no categories")
# --- 5. apply ----------------------------------------------------------
if args.fix:
# write product files (under new names)
for stem, entry in products.items():
data = entry["data"]
if data is None:
continue
new = new_pid(stem)
newpath = os.path.join(prod_dir, new + ".json")
if stem != new:
os.rename(entry["path"], newpath)
save_json(newpath, data)
elif stem in dirty_products:
save_json(newpath, data)
for stem, entry in categories.items():
data = entry["data"]
if data is None:
continue
new = new_cid(stem)
newpath = os.path.join(cat_dir, new + ".json")
if stem != new:
os.rename(entry["path"], newpath)
save_json(newpath, data)
elif stem in dirty_cats:
save_json(newpath, data)
if args.fix_settings and isinstance(gs, dict) and rename_cat:
shutil.copy2(gs_path, gs_path + ".bak")
for t in gs.get("traders", []):
if isinstance(t, dict):
t["categoriesId"] = [new_cid(c) for c in t.get("categoriesId", [])]
save_json(gs_path, gs)
# stock files follow the product id
stock_dir = os.path.join(os.path.dirname(cfg), "TraderXDatabase", "Stock")
for old, new in rename_prod.items():
sp = os.path.join(stock_dir, old + ".json")
if os.path.exists(sp):
d = load_json(sp, rep)
if isinstance(d, dict):
d["productId"] = new
save_json(os.path.join(stock_dir, new + ".json"), d)
os.remove(sp)
rep.fix(old, f"stock file renamed -> {new}.json")
rep.dump(show_info=not args.quiet_info,
group_threshold=10 ** 9 if args.no_group else 10,
ignore=tuple(args.ignore))
return 1 if rep.errors and not args.fix else 0
if __name__ == "__main__":
sys.exit(main())