Added Grafana alert silencing

This commit is contained in:
Edgar P. Burkhart 2022-01-23 16:56:50 +01:00
parent 186aaaf33e
commit 29da835843
Signed by: edpibu
GPG Key ID: 9833D3C5A25BD227
2 changed files with 69 additions and 0 deletions

View File

@ -21,3 +21,9 @@ enable = True
threads = 4 threads = 4
[reconstructPar] [reconstructPar]
enable = True enable = True
[grafana]
pause = True
url = https://mon.edgarpierre.fr/
alert_name = CPU Usage
#apikey =

View File

@ -7,9 +7,15 @@ from pprint import pp
from time import time from time import time
from datetime import timedelta from datetime import timedelta
from datetime import datetime
from pathlib import Path from pathlib import Path
import subprocess as sp import subprocess as sp
import urllib.request
import urllib.parse
import urllib.error
import json
_t0 = time() _t0 = time()
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
@ -33,6 +39,41 @@ log = logging.getLogger('openfoam')
log.info('Starting program') log.info('Starting program')
if config.getboolean('grafana', 'pause', fallback=False):
log.info('Silencing Grafana alert')
try:
with urllib.request.urlopen(
urllib.request.Request(
urllib.parse.urljoin(
config.get('grafana', 'url'),
'api/alertmanager/grafana/api/v2/silences',
),
headers={
'Authorization':
f'Bearer {config.get("grafana", "apikey")}',
},
),
data=bytes(json.dumps({
"comment": "Openfoam run",
"createdBy": "Python OpenFoam run",
"endsAt": (datetime.utcnow() + timedelta(days=1)).isoformat(),
"matchers": [
{
"isEqual": True,
"isRegex": False,
"name": "alertname",
"value": "CPU Usage"
}
],
"startsAt": datetime.utcnow().isoformat()
}), 'utf-8'),
) as api_req:
api_res = api_req.read()
grafana_alert_id = json.loads(api_res)['id']
except urllib.error.HTTPError as e:
log.error(f'Grafana silence returned code {e.code}')
sys.exit(1)
input_dir = Path(config.get('main', 'input_dir'), config.get('main', 'case'))\ input_dir = Path(config.get('main', 'input_dir'), config.get('main', 'case'))\
.expanduser() .expanduser()
work_dir = Path(config.get('main', 'work_dir')).expanduser() work_dir = Path(config.get('main', 'work_dir')).expanduser()
@ -199,5 +240,27 @@ if config.getboolean('olaFlow', 'enable', fallback=False):
log.info(f'Deleting {proc_dir}') log.info(f'Deleting {proc_dir}')
shutil.rmtree(proc_dir) shutil.rmtree(proc_dir)
if config.getboolean('grafana', 'pause', fallback=False):
log.info('Unsilencing Grafana alert')
try:
with urllib.request.urlopen(
urllib.request.Request(
urllib.parse.urljoin(
config.get('grafana', 'url'),
'api/alertmanager/grafana/api/v2/silence/'
f'{grafana_alert_id}',
),
headers={
'Authorization':
f'Bearer {config.get("grafana", "apikey")}',
},
method='DELETE',
),
) as api_req:
api_res = api_req.read()
except urllib.error.HTTPError as e:
log.error(f'Grafana unsilence returned code {e.code}')
sys.exit(1)
_t1 = time() _t1 = time()
log.info(f'Program ended successfully after {timedelta(seconds=_t1-_t0)}') log.info(f'Program ended successfully after {timedelta(seconds=_t1-_t0)}')