commit 1cce419f9b08be2de097acf09b8135c902f3fd92
parent 37d553aaef65e263863fa798ea6666735ec2849d
Author: Anders Damsgaard <anders@adamsgaard.dk>
Date: Wed, 8 Apr 2026 16:11:59 +0200
fix(unpivot-xyz): replace pandas parsing with str.split() for format compatibility
The pandas sep='\s+' heuristic for detecting row indices relies on data
rows having one more field than the header. The newer TEMImage format
(v4.0.7.8) omits the project-name prefix in data rows, so data and header
have equal field counts, breaking column detection.
Replace pd.read_csv with the same str.split() approach used in core.py,
which handles both formats via an explicit len(values)==len(headers)+1
check. Also removes the pandas dependency from the standalone script.
Diffstat:
| A | unpivot-xyz.py | | | 135 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
1 file changed, 135 insertions(+), 0 deletions(-)
diff --git a/unpivot-xyz.py b/unpivot-xyz.py
@@ -0,0 +1,135 @@
+#!/usr/bin/env python
+from pathlib import Path
+import sys
+import csv
+import math
+
+
+def count_header_lines(path, comment_char='/'):
+ with open(path, 'r') as f:
+ for i, line in enumerate(f):
+ if not line.lstrip().startswith(comment_char):
+ return i
+ return 0
+
+
+def unpivot_xyz(path):
+ skiprows = count_header_lines(path)
+ with open(path, 'r') as f:
+ lines = f.readlines()
+
+ data_lines = lines[skiprows:]
+ headers = data_lines[0].split()
+ res_cols = [c for c in headers if c.startswith("Res_")]
+ thick_cols = [c for c in headers if c.startswith("Thick_")]
+ model_points = []
+ doi_points = []
+ model_layers = []
+ for line_no, line in enumerate(data_lines[1:], 1):
+ line = line.strip()
+ if not line:
+ continue
+ values = line.split()
+ if len(values) == len(headers) + 1:
+ values = values[1:]
+ row = dict(zip(headers, values))
+ try:
+ x = float(row["X"])
+ y = float(row["Y"])
+ z = float(row["Z"])
+ doi = float(row["DOI"])
+ data_residual = float(row["DataResidual"])
+ except (KeyError, ValueError) as e:
+ print(f'Skipping row {line_no}: {e}', file=sys.stderr)
+ continue
+ raw_n = row.get("NumLayers", "")
+ try:
+ n_layers = int(float(raw_n))
+ except (ValueError, TypeError):
+ n_layers = None
+ point_wkt = f'POINT Z ({x} {y} {z})'
+ z_doi = z - doi
+ doi_wkt = f'POINT Z ({x} {y} {z_doi})'
+ cum_depth = 0
+ model_points.append({
+ "X": x,
+ "Y": y,
+ "Z": z,
+ "Line": row.get("Line", ""),
+ "StationNo": row.get("StationNo", ""),
+ "DataResidual": data_residual,
+ "NumLayers": n_layers,
+ "Geometry": point_wkt
+ })
+ doi_points.append({
+ "X": x,
+ "Y": y,
+ "Z": z_doi,
+ "DOI": doi,
+ "ZDOI": z_doi,
+ "Geometry": doi_wkt
+ })
+ layer_cols = list(zip(res_cols, thick_cols))
+ if n_layers is not None:
+ layer_cols = layer_cols[:n_layers]
+ for i, (res_col, thick_col) in enumerate(layer_cols, 1):
+ try:
+ res = float(row[res_col])
+ thick = float(row[thick_col])
+ except (KeyError, ValueError):
+ print(
+ f'Skipping layer {i} for row {line_no}: '
+ f'missing {res_col} or {thick_col}.',
+ file=sys.stderr
+ )
+ break
+ if math.isnan(res) or math.isnan(thick):
+ break
+ depth_top = cum_depth
+ depth_bottom = cum_depth + thick
+ z_top = z - depth_top
+ z_bot = z - depth_bottom
+ z_mid = (z_top + z_bot) / 2
+ cum_depth = depth_bottom
+ layer_wkt = f'LINESTRING Z ({x} {y} {z_top}, {x} {y} {z_bot})'
+ model_layers.append({
+ "X": x,
+ "Y": y,
+ "Z": z,
+ "ZTop": z_top,
+ "ZMid": z_mid,
+ "ZBottom": z_bot,
+ "DepthTop": depth_top,
+ "DepthBottom": depth_bottom,
+ "Resistivity": res,
+ "Layer": i,
+ "Geometry": layer_wkt
+ })
+ store_csv(model_layers, path, '.layers.csv')
+ store_csv(model_points, path, '.points.csv')
+ store_csv(doi_points, path, '.doi.csv')
+
+
+def store_csv(entries, base_path, new_suffix):
+ f = base_path.with_suffix(new_suffix)
+ if not entries:
+ f.write_text('')
+ print(f)
+ return f
+ with open(f, 'w', newline='') as out:
+ writer = csv.DictWriter(out, fieldnames=list(entries[0].keys()))
+ writer.writeheader()
+ writer.writerows(entries)
+ print(f)
+ return f
+
+
+if __name__ == '__main__':
+ if len(sys.argv) < 2:
+ raise SystemExit(f'usage: {sys.argv[0]} FILE.xyz ..')
+ for f in sys.argv[1:]:
+ path = Path(f)
+ if path.is_file():
+ unpivot_xyz(path)
+ else:
+ raise SystemExit(f'{f} not found.')
