m2cce/internship
m2cce
/
internship
1
Fork 0
Code Issues Activity

Processing update to reduce memory usage

This commit is contained in:
Edgar P. Burkhart 2022-03-15 10:11:21 +01:00
parent aac1d54f6e
commit 68e8d16f7c
Signed by: edpibu
GPG Key ID: 9833D3C5A25BD227
2 changed files with 23 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
swash/processing/read_swash.py
View File

@ -1,3 +1,6 @@
import subprocess
import tempfile
import numpy as np
@ -7,43 +10,44 @@ class ReadSwash:
self._n_t = None
self._t = None
self._x = None
self._data = {}
@classmethod
def read_nohead(cls, path):
with open(path) as file:
return np.asarray(file.read().split(), dtype=np.float64)
with open(path) as file, tempfile.TemporaryFile() as tmpfile:
subprocess.run(("tr", "-d", "\n"), stdin=file, stdout=tmpfile)
return np.loadtxt(tmpfile)
def read_time(self, path):
self._t = np.unique(self.read_nohead(path))
self._n_t = self._t.size
return self.t
def read_x(self, path):
self._x = np.unique(self.read_nohead(path))
self._n_x = self._x.size
return self.x
def read_scalar(self, path, const=False):
if const:
self._data[path.stem] = self.read_nohead(path).reshape(
return self.read_nohead(path).reshape(
(self._n_t, self._n_x)
)[0, :]
return
self._data[path.stem] = self.read_nohead(path).reshape(
return self.read_nohead(path).reshape(
(self._n_t, self._n_x)
)
def read_vector(self, path):
self._data[path.stem] = self.read_nohead(path).reshape(
return self.read_nohead(path).reshape(
(self._n_t, 2, self._n_x)
)
def read_scalar_lay(self, path):
self._data[path.stem] = self.read_nohead(path).reshape(
return self.read_nohead(path).reshape(
(self._n_t, -1, self._n_x)
)
def read_vector_lay(self, path):
self._data[path.stem] = self.read_nohead(path).reshape(
return self.read_nohead(path).reshape(
(self._n_t, 2, -1, self._n_x)
)
@ -54,7 +58,3 @@ class ReadSwash:
@property
def x(self):
return self._x
@property
def data(self):
return self._data

21
swash/processing/sws_npz.py
View File

@ -27,15 +27,14 @@ rsws = ReadSwash()
rsws.read_time(sws_out.joinpath("tsec.dat"))
rsws.read_x(sws_out.joinpath("xp.dat"))
log.info("Reading 'dep'")
rsws.read_scalar(sws_out.joinpath("dep.dat"))
log.info("Reading 'botl'")
rsws.read_scalar(sws_out.joinpath("botl.dat"), const=True)
log.info("Reading 'watl'")
rsws.read_scalar(sws_out.joinpath("watl.dat"))
log.info("Reading 'vel'")
rsws.read_vector(sws_out.joinpath("vel.dat"))
log.info(f"Writing npz file in '{inp}'")
inp.mkdir(exist_ok=True)
np.savez_compressed(inp.joinpath("sws"), t=rsws.t, x=rsws.x, **rsws.data)
log.info(f"Wrinting output in '{inp}'")
log.info("Reading 'dep'")
np.save(inp.joinpath("dep"), rsws.read_scalar(sws_out.joinpath("dep.dat")))
log.info("Reading 'botl'")
np.save(inp.joinpath("botl"), rsws.read_scalar(sws_out.joinpath("botl.dat"),
const=True))
log.info("Reading 'watl'")
np.save(inp.joinpath("watl"), rsws.read_scalar(sws_out.joinpath("watl.dat")))
log.info("Reading 'vel'")
np.save(inp.joinpath("vel"), rsws.read_vector(sws_out.joinpath("vel.dat")))