Fix the issues with entvalidate.py

It doesn't properly handle mixins / special templates
    On python 3.8+, XML elements are no longer sorted and this fails
validation against entity.rng
    mixins are not properly applied because the | split is incorrect (3
max items instead of 2)
    merge="" is not supported
    Parents aren't always passed when they should be.
    on windows it is incorrectly assumed that the encoding is 1252

Patch by: @baco
Tweaks by: @Stan
Based on a patch by: @wraitii
Differential Revision: https://code.wildfiregames.com/D4698
This was SVN commit r27161.

source/tools/entity/entvalidate.py

@@ -1,44 +1,116 @@
 #!/usr/bin/env python3
-from os import chdir
+import argparse
+import logging
 from pathlib import Path
+import shutil
 from subprocess import run, CalledProcessError
-from sys import exit
+import sys
+from typing import Sequence
+
 from xml.etree import ElementTree
-from scriptlib import warn, SimulTemplateEntity, find_files
 
+from scriptlib import SimulTemplateEntity, find_files
 
-def main():
-    root = Path(__file__).resolve().parents[3]
-    relaxng_schema = root / 'binaries' / 'system' / 'entity.rng'
-    if not relaxng_schema.exists():
-        warn(f"""Relax NG schema non existant.
-Please create the file {relaxng_schema.relative_to(root)}
-You can do that by running 'pyrogenesis -dumpSchema' in the 'system' directory""")
-        exit(1)
-    if run(['xmllint', '--version'], capture_output=True).returncode != 0:
-        warn("xmllint not found in your PATH, please install it (usually in libxml2 package)")
-        exit(2)
-    vfs_root = root / 'binaries' / 'data' / 'mods'
-    simul_templates_path = Path('simulation/templates')
-    simul_template_entity = SimulTemplateEntity(vfs_root)
-    count = 0
-    failed = 0
-    for fp, _ in sorted(find_files(vfs_root, ['public'], 'simulation/templates', 'xml')):
-        if fp.stem.startswith('template_'):
+SIMUL_TEMPLATES_PATH = Path("simulation/templates")
+ENTITY_RELAXNG_FNAME = "entity.rng"
+RELAXNG_SCHEMA_ERROR_MSG = """Relax NG schema non existant.
+Please create the file: {}
+You can do that by running 'pyrogenesis -dumpSchema' in the 'system' directory
+"""
+XMLLINT_ERROR_MSG = ("xmllint not found in your PATH, please install it "
+                     "(usually in libxml2 package)")
+
+class SingleLevelFilter(logging.Filter):
+    def __init__(self, passlevel, reject):
+        self.passlevel = passlevel
+        self.reject = reject
+
+    def filter(self, record):
+        if self.reject:
+            return (record.levelno != self.passlevel)
+        else:
+            return (record.levelno == self.passlevel)
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+# create a console handler, seems nicer to Windows and for future uses
+ch = logging.StreamHandler(sys.stdout)
+ch.setLevel(logging.INFO)
+ch.setFormatter(logging.Formatter('%(levelname)s - %(message)s'))
+f1 = SingleLevelFilter(logging.INFO, False)
+ch.addFilter(f1)
+logger.addHandler(ch)
+errorch =logging. StreamHandler(sys.stderr)
+errorch.setLevel(logging.WARNING)
+errorch.setFormatter(logging.Formatter('%(levelname)s - %(message)s'))
+logger.addHandler(errorch)
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description="Validate templates")
+    parser.add_argument("-m", "--mod-name", required=True,
+                        help="The name of the mod to validate.")
+    parser.add_argument("-r", "--root", dest="vfs_root", default=Path(),
+                        type=Path, help="The path to mod's root location.")
+    parser.add_argument("-s", "--relaxng-schema",
+                        default=Path() / ENTITY_RELAXNG_FNAME, type=Path,
+                        help="The path to mod's root location.")
+    parser.add_argument("-t", "--templates", nargs="*",
+                        help="Optionally, a list of templates to validate.")
+    parser.add_argument("-v", "--verbose",
+                        help="Be verbose about the output.",  default=False)
+
+    args = parser.parse_args(argv)
+
+    if not args.relaxng_schema.exists():
+        logging.error(RELAXNG_SCHEMA_ERROR_MSG.format(args.relaxng_schema))
+        return 1
+
+    if not shutil.which("xmllint"):
+        logging.error(XMLLINT_ERROR_MSG)
+        return 2
+
+    if args.templates:
+        templates = sorted([(Path(t), None) for t in args.templates])
+    else:
+        templates = sorted(find_files(args.vfs_root, [args.mod_name],
+                                      SIMUL_TEMPLATES_PATH.as_posix(), "xml"))
+
+    simul_template_entity = SimulTemplateEntity(args.vfs_root, logger)
+    count, failed = 0, 0
+    for fp, _ in templates:
+        if fp.stem.startswith("template_"):
             continue
-        print(f"# {fp}...")
+
+        path = fp.as_posix()
+        if (path.startswith(f"{SIMUL_TEMPLATES_PATH.as_posix()}/mixins/")
+                or path.startswith(
+                    f"{SIMUL_TEMPLATES_PATH.as_posix()}/special/")):
+            continue
+
+        if (args.verbose):
+            logger.info(f"Parsing {fp}...")
         count += 1
-        entity = simul_template_entity.load_inherited(simul_templates_path, str(fp.relative_to(simul_templates_path)), ['public'])
-        xmlcontent = ElementTree.tostring(entity, encoding='unicode')
+        entity = simul_template_entity.load_inherited(
+            SIMUL_TEMPLATES_PATH,
+            str(fp.relative_to(SIMUL_TEMPLATES_PATH)),
+            [args.mod_name]
+        )
+        xmlcontent = ElementTree.tostring(entity, encoding="unicode")
         try:
-            run(['xmllint', '--relaxng', str(relaxng_schema.resolve()), '-'], input=xmlcontent, capture_output=True, text=True, check=True)
+            run(["xmllint", "--relaxng",
+                 str(args.relaxng_schema.resolve()), "-"],
+                input=xmlcontent, encoding="utf-8", capture_output=True, text=True, check=True)
         except CalledProcessError as e:
             failed += 1
-            print(e.stderr)
-            print(e.stdout)
-    print(f"\nTotal: {count}; failed: {failed}")
+            if (e.stderr):
+                logger.error(e.stderr)
+            if (e.stdout):
+                logger.info(e.stdout)
+
+    logger.info(f"Total: {count}; failed: {failed}")
+
+    return 0
 
 
-if __name__ == '__main__':
-    chdir(Path(__file__).resolve().parent)
-    main()
+if __name__ == "__main__":
+    raise SystemExit(main())

source/tools/entity/scriptlib/__init__.py

@@ -1,7 +1,6 @@
 from collections import Counter
 from decimal import Decimal
 from re import split
-from sys import stderr
 from xml.etree import ElementTree
 from os.path import exists
 
@@ -12,7 +11,7 @@ class SimulTemplateEntity:
 
     def get_file(self, base_path, vfs_path, mod):
         default_path = self.vfs_root / mod / base_path
-        file = (default_path/ "special" / "filter" / vfs_path).with_suffix('.xml')
+        file = (default_path / "special" / "filter" / vfs_path).with_suffix('.xml')
         if not exists(file):
             file = (default_path / "mixins" / vfs_path).with_suffix('.xml')
         if not exists(file):
@@ -48,28 +47,34 @@ class SimulTemplateEntity:
                 elif token not in final_tokens:
                     final_tokens.append(token)
             base_tag.text = ' '.join(final_tokens)
+            base_tag.set("datatype", "tokens")
         elif tag.get('op'):
             op = tag.get('op')
             op1 = Decimal(base_tag.text or '0')
             op2 = Decimal(tag.text or '0')
+            # Try converting to integers if possible, to pass validation.
             if op == 'add':
-                base_tag.text = str(op1 + op2)
+                base_tag.text = str(int(op1 + op2) if int(op1 + op2) == op1 + op2 else op1 + op2)
             elif op == 'mul':
-                base_tag.text = str(op1 * op2)
+                base_tag.text = str(int(op1 * op2) if int(op1 * op2) == op1 * op2 else op1 * op2)
             elif op == 'mul_round':
                 base_tag.text = str(round(op1 * op2))
             else:
                 raise ValueError(f"Invalid operator '{op}'")
         else:
             base_tag.text = tag.text
+            for prop in tag.attrib:
+                if prop not in ('disable', 'replace', 'parent', 'merge'):
+                    base_tag.set(prop, tag.get(prop))
         for child in tag:
             base_child = base_tag.find(child.tag)
             if 'disable' in child.attrib:
                 if base_child is not None:
                     base_tag.remove(base_child)
-            else:
+            elif ('merge' not in child.attrib) or (base_child is not None):
                 if 'replace' in child.attrib and base_child is not None:
                     base_tag.remove(base_child)
+                    base_child = None
                 if base_child is None:
                     base_child = ElementTree.Element(child.tag)
                     base_tag.append(base_child)
@@ -77,14 +82,19 @@ class SimulTemplateEntity:
                 if 'replace' in base_child.attrib:
                     del base_child.attrib['replace']
 
-    def load_inherited(self, base_path, vfs_path, mods, base = None):
+    def load_inherited(self, base_path, vfs_path, mods):
+        entity = self._load_inherited(base_path, vfs_path, mods)
+        entity[:] = sorted(entity[:], key=lambda x: x.tag)
+        return entity
+
+    def _load_inherited(self, base_path, vfs_path, mods, base=None):
         """
         vfs_path should be relative to base_path in a mod
         """
         if '|' in vfs_path:
-            paths = vfs_path.split("|", 2)
-            base = self.load_inherited(base_path, paths[1], mods, base);
-            base = self.load_inherited(base_path, paths[0], mods, base);
+            paths = vfs_path.split("|", 1)
+            base = self._load_inherited(base_path, paths[1], mods, base)
+            base = self._load_inherited(base_path, paths[0], mods, base)
             return base
 
         main_mod = self.get_main_mod(base_path, vfs_path, mods)
@@ -97,7 +107,7 @@ class SimulTemplateEntity:
                 for dup in duplicates:
                     self.logger.warning(f"Duplicate child node '{dup}' in tag {el.tag} of {fp}")
         if layer.get('parent'):
-            parent = self.load_inherited(base_path, layer.get('parent'), mods)
+            parent = self._load_inherited(base_path, layer.get('parent'), mods, base)
             self.apply_layer(parent, layer)
             return parent
         else: