m2cce/internship
m2cce/internship
1
Fork 0
Code Issues Activity

More efficient ? .dat file reading

Browse source
This commit is contained in:
Edgar P. Burkhart 2022-04-07 11:06:17 +02:00
parent 2eea373e79
commit a59bcccde7
Signed by: edpibu
GPG key ID: 9833D3C5A25BD227
2 changed files with 16 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
swash/processing/read_swash.py
View file

@ -16,14 +16,12 @@ class ReadSwash:
@classmethod @classmethod
def read_nohead(cls, path): def read_nohead(cls, path):
with tempfile.TemporaryFile(mode="w+t") as tmpfile: data = []
log.info(f"Replacing \\s with \\n in '{path}'") with path.open() as inp:
subprocess.run(("sed", r"s/\s\+/\n/g;/^$/d", path), stdout=tmpfile, text=True) for line in inp:
log.info(f"Loading '{path}'") data += line.split()
tmpfile.seek(0)
a = np.asarray(tmpfile.readlines(), dtype=float) return np.asarray(data, dtype=float)
log.debug(f"path={a}")
return a
def read_time(self, path): def read_time(self, path):
self._t = np.unique(self.read_nohead(path)) self._t = np.unique(self.read_nohead(path))

23
swash/processing/sws_npz.py
View file

@ -33,20 +33,17 @@ var = {
#"dep": rsws.read_scalar, #"dep": rsws.read_scalar,
"botl": rsws.read_const, "botl": rsws.read_const,
"watl": rsws.read_scalar, "watl": rsws.read_scalar,
"pressk": rsws.read_scalar_lay, #"pressk": rsws.read_scalar_lay,
"nhprsk": rsws.read_scalar_lay, #"nhprsk": rsws.read_scalar_lay,
"zk": rsws.read_scalar_lay, #"zk": rsws.read_scalar_lay,
"velk": rsws.read_vector_lay, #"velk": rsws.read_vector_lay,
"vz": rsws.read_scalar_lay, #"vz": rsws.read_scalar_lay,
"vel": rsws.read_vector, "vel": rsws.read_vector,
} }
with ThreadPool(len(var)) as pool: for name, f in var.items():
log.info("Converting all data") log.info(f"Converting {name}")
pool.map( np.save(
lambda x: np.save( inp.joinpath(name),
inp.joinpath(x[0]), f(sws_out.joinpath(name).with_suffix(".dat")),
x[1](sws_out.joinpath(x[0]).with_suffix(".dat")),
),
var.items(),
) )