gptbot/0ad
1
0
Fork 0
forked from mirrors/0ad
Code Pull Requests Activity

Improve find_files implementation

Browse Source 
Instead of listing files recursively for each directory manually, this
uses the built-in glob functionality of Python's pathlib.Path. As
part of this the found paths are returned as generator instead of a
list. Together this results in easier to read code and better
performance.
This commit is contained in:
Dunedan
2025-05-25 17:12:06 +02:00
parent 279233aca1
commit 4b3d7f018b
4 changed files with 38 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
source/tools/entity/checkrefs.py
+23 -18
View File
@@ -5,6 +5,7 @@ import sys
from argparse import ArgumentParser
from collections import defaultdict
from io import BytesIO
from itertools import chain
from json import load, loads
from logging import INFO, WARNING, Filter, Formatter, StreamHandler, getLogger
from pathlib import Path
@@ -201,13 +202,15 @@ class CheckRefs:
return self.vfs_root / fn
def find_files(self, vfs_path, *ext_list):
return find_files(self.vfs_root, self.mods, vfs_path, *ext_list)
return find_files(self.vfs_root, self.mods, Path(vfs_path), ext_list)
def add_maps_xml(self):
self.logger.info("Loading maps XML...")
mapfiles = self.find_files("maps/scenarios", "xml")
mapfiles.extend(self.find_files("maps/skirmishes", "xml"))
mapfiles.extend(self.find_files("maps/tutorials", "xml"))
mapfiles = chain(
self.find_files("maps/scenarios", "xml"),
self.find_files("maps/skirmishes", "xml"),
self.find_files("maps/tutorials", "xml"),
)
actor_prefix = "actor|"
resource_prefix = "resource|"
for fp, ffp in sorted(mapfiles):
@@ -253,8 +256,10 @@ class CheckRefs:
ffp,
)
terrains[name] = str(fp)
mapfiles = self.find_files("maps/scenarios", "pmp")
mapfiles.extend(self.find_files("maps/skirmishes", "pmp"))
mapfiles = chain(
self.find_files("maps/scenarios", "pmp"),
self.find_files("maps/skirmishes", "pmp"),
)
for fp, ffp in sorted(mapfiles):
self.files.append(fp)
self.roots.append(fp)
@@ -288,7 +293,7 @@ class CheckRefs:
def get_existing_civ_codes(self):
existing_civs = set()
for _, ffp in sorted(self.find_files("simulation/data/civs", "json")):
for _, ffp in self.find_files("simulation/data/civs", "json"):
with open(ffp, encoding="utf-8") as f:
civ = load(f)
code = civ.get("Code")
@@ -320,7 +325,7 @@ class CheckRefs:
# computing the values ourselves.
simul_template_entity = SimulTemplateEntity(self.vfs_root, self.logger)
custom_phase_techs = self.get_custom_phase_techs()
for fp, _ in sorted(self.find_files(simul_templates_path, "xml")):
for fp, _ in self.find_files(simul_templates_path, "xml"):
self.files.append(fp)
entity = simul_template_entity.load_inherited(
simul_templates_path, str(fp.relative_to(simul_templates_path)), self.mods
@@ -519,7 +524,7 @@ class CheckRefs:
def add_actors(self):
self.logger.info("Loading actors...")
for fp, ffp in sorted(self.find_files("art/actors", "xml")):
for fp, ffp in self.find_files("art/actors", "xml"):
self.files.append(fp)
self.roots.append(fp)
root = ET.parse(ffp).getroot()
@@ -536,7 +541,7 @@ class CheckRefs:
def add_variants(self):
self.logger.info("Loading variants...")
for fp, ffp in sorted(self.find_files("art/variants", "xml")):
for fp, ffp in self.find_files("art/variants", "xml"):
self.files.append(fp)
self.roots.append(fp)
variant = ET.parse(ffp).getroot()
@@ -577,7 +582,7 @@ class CheckRefs:
def add_materials(self):
self.logger.info("Loading materials...")
for fp, ffp in sorted(self.find_files("art/materials", "xml")):
for fp, ffp in self.find_files("art/materials", "xml"):
self.files.append(fp)
material_elem = ET.parse(ffp).getroot()
for alternative in material_elem.findall("alternative"):
@@ -587,7 +592,7 @@ class CheckRefs:
def add_particles(self):
self.logger.info("Loading particles...")
for fp, ffp in sorted(self.find_files("art/particles", "xml")):
for fp, ffp in self.find_files("art/particles", "xml"):
self.files.append(fp)
self.roots.append(fp)
particle = ET.parse(ffp).getroot()
@@ -597,7 +602,7 @@ class CheckRefs:
def add_soundgroups(self):
self.logger.info("Loading sound groups...")
for fp, ffp in sorted(self.find_files("audio", "xml")):
for fp, ffp in self.find_files("audio", "xml"):
self.files.append(fp)
self.roots.append(fp)
sound_group = ET.parse(ffp).getroot()
@@ -667,7 +672,7 @@ class CheckRefs:
def add_gui_xml(self):
self.logger.info("Loading GUI XML...")
gui_page_regex = re.compile(r".*[\\\/]page(_[^.\/\\]+)?\.xml$")
for fp, ffp in sorted(self.find_files("gui", "xml")):
for fp, ffp in self.find_files("gui", "xml"):
self.files.append(fp)
# GUI page definitions are assumed to be named page_[something].xml and alone in that.
if gui_page_regex.match(str(fp)):
@@ -804,7 +809,7 @@ class CheckRefs:
def add_civs(self):
self.logger.info("Loading civs...")
for fp, ffp in sorted(self.find_files("simulation/data/civs", "json")):
for fp, ffp in self.find_files("simulation/data/civs", "json"):
self.files.append(fp)
self.roots.append(fp)
with open(ffp, encoding="utf-8") as f:
@@ -857,7 +862,7 @@ class CheckRefs:
def add_techs(self):
self.logger.info("Loading techs...")
for fp, ffp in sorted(self.find_files("simulation/data/technologies", "json")):
for fp, ffp in self.find_files("simulation/data/technologies", "json"):
self.files.append(fp)
with open(ffp, encoding="utf-8") as f:
tech = load(f)
@@ -878,7 +883,7 @@ class CheckRefs:
def add_terrains(self):
self.logger.info("Loading terrains...")
for fp, ffp in sorted(self.find_files("art/terrains", "xml")):
for fp, ffp in self.find_files("art/terrains", "xml"):
# ignore terrains.xml
if str(fp).endswith("terrains.xml"):
continue
@@ -894,7 +899,7 @@ class CheckRefs:
def add_auras(self):
self.logger.info("Loading auras...")
for fp, ffp in sorted(self.find_files("simulation/data/auras", "json")):
for fp, ffp in self.find_files("simulation/data/auras", "json"):
self.files.append(fp)
with open(ffp, encoding="utf-8") as f:
aura = load(f)
source/tools/entity/entvalidate.py
+1 -1
View File
@@ -55,7 +55,7 @@ def validate_templates(
if templates:
templates = [(template, None) for template in templates]
else:
templates = find_files(vfs_root, [mod_name], SIMUL_TEMPLATES_PATH.as_posix(), "xml")
templates = find_files(vfs_root.resolve(), [mod_name], SIMUL_TEMPLATES_PATH, ["xml"])
templates_to_validate = []
for fp, _ in templates:
source/tools/entity/scriptlib/__init__.py
+10 -17
View File
@@ -1,6 +1,8 @@
from collections import Counter
from collections.abc import Generator
from decimal import Decimal
from os.path import exists
from pathlib import Path
from re import split
from xml.etree import ElementTree as ET
@@ -114,27 +116,18 @@ class SimulTemplateEntity:
return base
def find_files(vfs_root, mods, vfs_path, *ext_list):
def find_files(
vfs_root: Path, mods: list[str], vfs_path: Path, file_extensions: list[str]
) -> Generator[tuple[Path, Path], None, None]:
"""Find files.
Returns a list of 2-size tuple with:
- Path relative to the mod base
- full Path
"""
full_exts = ["." + ext for ext in ext_list]
full_file_extensions = {f".{ext}" for ext in file_extensions}
def find_recursive(dp, base):
"""(relative Path, full Path) generator."""
if dp.is_dir():
if dp.name not in (".svn", ".git") and not dp.name.endswith("~"):
for fp in dp.iterdir():
yield from find_recursive(fp, base)
elif dp.suffix in full_exts:
relative_file_path = dp.relative_to(base)
yield (relative_file_path, dp.resolve())
return [
(rp, fp)
for mod in mods
for (rp, fp) in find_recursive(vfs_root / mod / vfs_path, vfs_root / mod)
]
for mod in mods:
for path in (vfs_root / mod / vfs_path).resolve().glob("**/*"):
if not path.is_dir() and path.suffix in full_file_extensions:
yield path.relative_to(vfs_root / mod), path
source/tools/entity/validate_dae.py
+4 -3
View File
@@ -51,9 +51,10 @@ class DaeValidator:
def run(self):
is_ok = True
files = find_files(self.vfs_root, self.mods, "art/meshes", "dae")
self.log.info("Checking %i meshes for invalid weights.", len(files))
files = find_files(self.vfs_root, self.mods, Path("art/meshes"), ["dae"])
i = 0
for _, dae in files:
i += 1
status = validate_vertex(dae.as_posix(), self.log.warning)
if status >= 1:
is_ok = False
@@ -62,7 +63,7 @@ class DaeValidator:
self.log.info(
"%i out of %i files have vertices with no weight or bones.",
len(self.has_weightless_vtx),
len(files),
i,
)
return is_ok