vibe-sketcherDocumentation
Source code for vibesketcher.document.measured_patterns
"""Measured powder patterns from simple two-column text files (#207).
Only plain text is read: whitespace-separated XY (``.xy``, ``.xye``,
``.dat``, ``.txt``) and comma- or semicolon-separated CSV (``.csv``). The
x axis must be stated by the caller (``two_theta_deg``, ``d_angstrom`` or
``q_inv_angstrom``), never guessed from the numbers, and so must the
intensity unit. Vendor and binary formats (Bruker RAW/BRML, PANalytical
XRDML, Rigaku RAS, GSAS, ...) are refused with the reason; convert them to
XY with the instrument software, which keeps their licensing out of this
package.
The provenance of every import is kept: the SHA-256 and size of the
original bytes, the file name as given (no directory), the format, the
stated axis, intensity unit and wavelength, instrument fields exactly as
given, and the file's own comment lines. A third column is read as the
intensity standard uncertainty (``esd``) when present.
"""
from __future__ import annotations
import hashlib
import math
import re
from pathlib import PurePath
from typing import Any, Literal
from pydantic import BaseModel, ConfigDict, Field
Axis = Literal["two_theta_deg", "d_angstrom", "q_inv_angstrom"]
AXES: tuple[str, ...] = ("two_theta_deg", "d_angstrom", "q_inv_angstrom")
TextFormat = Literal["xy", "csv"]
_FORMATS = {".xy": "xy", ".xye": "xy", ".dat": "xy", ".txt": "xy", ".csv": "csv"}
#: Extensions we recognise and refuse, with the reason.
_VENDOR = {
".raw": "Bruker/Siemens RAW (binary vendor format)",
".brml": "Bruker BRML (vendor archive)",
".xrdml": "PANalytical XRDML (vendor XML)",
".ras": "Rigaku RAS (vendor format)",
".rasx": "Rigaku RASX (vendor archive)",
".udf": "Philips UDF (vendor format)",
".uxd": "Bruker UXD (vendor format)",
".gsa": "GSAS data (instrument-parameter format)",
".gss": "GSAS data (instrument-parameter format)",
".fxye": "GSAS FXYE (instrument-parameter format)",
".cif": "powder CIF (pdCIF is not read here)",
}
MAX_POINTS = 1_000_000
MAX_BYTES = 64 * 1024 * 1024
MAX_HEADER_LINES = 200
_COMMENT = ("#", "!", ";", "'", "%", "//")
[docs]
class MeasuredPatternRefused(ValueError):
"""The file or its stated axis cannot be read faithfully."""
[docs]
class MeasuredSettings(BaseModel):
"""What the caller states about a measured file; nothing is inferred."""
model_config = ConfigDict(extra="forbid")
axis: str
intensity_unit: Literal["counts", "counts_per_second", "arbitrary"] = "counts"
#: Wavelength of the measurement in angstrom, when known (required to
#: convert a 2theta axis to d or Q, and to overlay in 2theta).
wavelength_angstrom: float | None = Field(default=None, gt=0.01, le=100)
#: Explicit format; otherwise taken from the file name's extension.
format: str | None = None
#: Instrument fields exactly as given (diffractometer, geometry, slits, ...).
instrument: dict[str, str] = Field(default_factory=dict, max_length=64)
sample: str | None = Field(default=None, max_length=500)
def _number(text: str) -> float | None:
try:
value = float(text)
except ValueError:
return None
return value if math.isfinite(value) else None
[docs]
def read_measured(data: bytes, filename: str, settings: MeasuredSettings) -> dict[str, Any]:
"""Points and provenance of a measured pattern, or :class:`MeasuredPatternRefused`."""
if settings.axis not in AXES:
raise MeasuredPatternRefused(
f"unknown x axis {settings.axis!r}; state one of {', '.join(AXES)}"
)
name = PurePath(filename.replace("\\", "/")).name or "pattern"
suffix = PurePath(name).suffix.lower()
fmt = settings.format
if fmt is None:
if suffix in _VENDOR:
raise MeasuredPatternRefused(
f"{name}: {_VENDOR[suffix]} is not supported; export a two-column XY or CSV "
"with the instrument software and state its axis"
)
if suffix not in _FORMATS:
raise MeasuredPatternRefused(
f"{name}: unsupported format {suffix or '(no extension)'}; give XY "
"(.xy/.xye/.dat/.txt) or CSV (.csv), or state format=xy|csv"
)
fmt = _FORMATS[suffix]
if fmt not in ("xy", "csv"):
raise MeasuredPatternRefused(f"unsupported format {fmt!r}; formats: xy, csv")
if len(data) > MAX_BYTES:
raise MeasuredPatternRefused(f"{name}: {len(data)} bytes exceed {MAX_BYTES}")
if b"\x00" in data:
raise MeasuredPatternRefused(f"{name}: binary content; only plain text is read")
try:
text = data.decode("utf-8-sig")
except UnicodeDecodeError:
try:
text = data.decode("latin-1")
except UnicodeDecodeError: # pragma: no cover - latin-1 decodes every byte
raise MeasuredPatternRefused(f"{name}: not a text file") from None
encoding = "latin-1"
else:
encoding = "utf-8"
header: list[str] = []
columns: list[str] | None = None
x: list[float] = []
y: list[float] = []
esd: list[float] = []
width: int | None = None
splitter = re.compile(r"\s*[,;]\s*" if fmt == "csv" else r"\s+")
for number, raw in enumerate(text.splitlines(), start=1):
line = raw.strip()
if not line:
continue
if line.startswith(_COMMENT):
if len(header) < MAX_HEADER_LINES:
header.append(line[:500])
continue
fields = [f.strip() for f in splitter.split(line) if f.strip() != ""]
values = [_number(f) for f in fields]
if any(v is None for v in values):
if not x and columns is None:
columns = [f[:80] for f in fields[:8]] # one column-name row before the data
continue
raise MeasuredPatternRefused(f"{name} line {number}: not numeric: {line[:80]!r}")
if len(values) < 2:
raise MeasuredPatternRefused(f"{name} line {number}: needs at least two columns")
if width is None:
width = len(values)
elif len(values) != width:
raise MeasuredPatternRefused(
f"{name} line {number}: {len(values)} columns where earlier lines have {width}"
)
if width > 3:
raise MeasuredPatternRefused(
f"{name}: {width} columns; give x, intensity and optionally its esd"
)
x.append(float(values[0])) # type: ignore[arg-type]
y.append(float(values[1])) # type: ignore[arg-type]
if width == 3:
esd.append(float(values[2])) # type: ignore[arg-type]
if len(x) > MAX_POINTS:
raise MeasuredPatternRefused(f"{name}: more than {MAX_POINTS} points")
if len(x) < 2:
raise MeasuredPatternRefused(f"{name}: fewer than two data points")
increasing = all(b > a for a, b in zip(x, x[1:], strict=False))
decreasing = all(b < a for a, b in zip(x, x[1:], strict=False))
if not (increasing or decreasing):
raise MeasuredPatternRefused(f"{name}: the x column is not strictly monotonic")
if settings.axis == "two_theta_deg" and not all(0 < v < 180 for v in x):
raise MeasuredPatternRefused(f"{name}: 2theta values must lie in (0, 180) degrees")
if settings.axis != "two_theta_deg" and not all(v > 0 for v in x):
raise MeasuredPatternRefused(f"{name}: d or Q values must be positive")
if esd and any(v < 0 for v in esd):
raise MeasuredPatternRefused(f"{name}: negative intensity uncertainties")
provenance = {
"file_name": name,
"sha256": hashlib.sha256(data).hexdigest(),
"bytes": len(data),
"encoding": encoding,
"format": fmt,
"axis": settings.axis,
"intensity_unit": settings.intensity_unit,
"wavelength_angstrom": settings.wavelength_angstrom,
"instrument": dict(settings.instrument),
"sample": settings.sample,
"header": header,
"columns": columns,
"points": len(x),
"x_range": [min(x), max(x)],
"has_esd": bool(esd),
}
return {"kind": "measured_powder_pattern", "x": x, "intensity": y,
"esd": esd or None, "provenance": provenance} # fmt: skip
[docs]
def read_measured_file(path: str, settings: MeasuredSettings) -> dict[str, Any]:
""":func:`read_measured` for a file on disk (bounded read)."""
from vibesketcher.limits import read_file
return read_measured(read_file(path, "the measured pattern", limit=MAX_BYTES),
path, settings) # fmt: skip
[docs]
def convert_axis(
values: list[float], source: str, target: str, wavelength: float | None
) -> list[float]:
"""Convert x values between 2theta (degrees), d (angstrom) and Q = 2 pi / d (1/angstrom).
Bragg's law ``lambda = 2 d sin(theta)``; a 2theta axis on either side needs
the wavelength, and a d or Q beyond the wavelength's reach is refused.
"""
if source == target:
return list(values)
if "two_theta_deg" in (source, target) and wavelength is None:
raise MeasuredPatternRefused(
"converting a 2theta axis needs the measurement wavelength; state it"
)
if source == "two_theta_deg":
assert wavelength is not None
d = [wavelength / (2 * math.sin(math.radians(v / 2))) for v in values]
elif source == "d_angstrom":
d = list(values)
else:
d = [2 * math.pi / v for v in values]
if target == "d_angstrom":
return d
if target == "q_inv_angstrom":
return [2 * math.pi / v for v in d]
assert wavelength is not None
out = []
for v in d:
ratio = wavelength / (2 * v)
if ratio >= 1:
raise MeasuredPatternRefused(
f"d = {v:.4f} A is not reachable at wavelength {wavelength:g} A"
)
out.append(math.degrees(2 * math.asin(ratio)))
return out