Added Grafana alert silencing
This commit is contained in:
parent
186aaaf33e
commit
29da835843
2 changed files with 69 additions and 0 deletions
|
@ -21,3 +21,9 @@ enable = True
|
||||||
threads = 4
|
threads = 4
|
||||||
[reconstructPar]
|
[reconstructPar]
|
||||||
enable = True
|
enable = True
|
||||||
|
|
||||||
|
[grafana]
|
||||||
|
pause = True
|
||||||
|
url = https://mon.edgarpierre.fr/
|
||||||
|
alert_name = CPU Usage
|
||||||
|
#apikey =
|
||||||
|
|
|
@ -7,9 +7,15 @@ from pprint import pp
|
||||||
|
|
||||||
from time import time
|
from time import time
|
||||||
from datetime import timedelta
|
from datetime import timedelta
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import subprocess as sp
|
import subprocess as sp
|
||||||
|
import urllib.request
|
||||||
|
import urllib.parse
|
||||||
|
import urllib.error
|
||||||
|
import json
|
||||||
|
|
||||||
|
|
||||||
_t0 = time()
|
_t0 = time()
|
||||||
parser = argparse.ArgumentParser(
|
parser = argparse.ArgumentParser(
|
||||||
|
@ -33,6 +39,41 @@ log = logging.getLogger('openfoam')
|
||||||
|
|
||||||
log.info('Starting program')
|
log.info('Starting program')
|
||||||
|
|
||||||
|
if config.getboolean('grafana', 'pause', fallback=False):
|
||||||
|
log.info('Silencing Grafana alert')
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(
|
||||||
|
urllib.request.Request(
|
||||||
|
urllib.parse.urljoin(
|
||||||
|
config.get('grafana', 'url'),
|
||||||
|
'api/alertmanager/grafana/api/v2/silences',
|
||||||
|
),
|
||||||
|
headers={
|
||||||
|
'Authorization':
|
||||||
|
f'Bearer {config.get("grafana", "apikey")}',
|
||||||
|
},
|
||||||
|
),
|
||||||
|
data=bytes(json.dumps({
|
||||||
|
"comment": "Openfoam run",
|
||||||
|
"createdBy": "Python OpenFoam run",
|
||||||
|
"endsAt": (datetime.utcnow() + timedelta(days=1)).isoformat(),
|
||||||
|
"matchers": [
|
||||||
|
{
|
||||||
|
"isEqual": True,
|
||||||
|
"isRegex": False,
|
||||||
|
"name": "alertname",
|
||||||
|
"value": "CPU Usage"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"startsAt": datetime.utcnow().isoformat()
|
||||||
|
}), 'utf-8'),
|
||||||
|
) as api_req:
|
||||||
|
api_res = api_req.read()
|
||||||
|
grafana_alert_id = json.loads(api_res)['id']
|
||||||
|
except urllib.error.HTTPError as e:
|
||||||
|
log.error(f'Grafana silence returned code {e.code}')
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
input_dir = Path(config.get('main', 'input_dir'), config.get('main', 'case'))\
|
input_dir = Path(config.get('main', 'input_dir'), config.get('main', 'case'))\
|
||||||
.expanduser()
|
.expanduser()
|
||||||
work_dir = Path(config.get('main', 'work_dir')).expanduser()
|
work_dir = Path(config.get('main', 'work_dir')).expanduser()
|
||||||
|
@ -199,5 +240,27 @@ if config.getboolean('olaFlow', 'enable', fallback=False):
|
||||||
log.info(f'Deleting {proc_dir}')
|
log.info(f'Deleting {proc_dir}')
|
||||||
shutil.rmtree(proc_dir)
|
shutil.rmtree(proc_dir)
|
||||||
|
|
||||||
|
if config.getboolean('grafana', 'pause', fallback=False):
|
||||||
|
log.info('Unsilencing Grafana alert')
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(
|
||||||
|
urllib.request.Request(
|
||||||
|
urllib.parse.urljoin(
|
||||||
|
config.get('grafana', 'url'),
|
||||||
|
'api/alertmanager/grafana/api/v2/silence/'
|
||||||
|
f'{grafana_alert_id}',
|
||||||
|
),
|
||||||
|
headers={
|
||||||
|
'Authorization':
|
||||||
|
f'Bearer {config.get("grafana", "apikey")}',
|
||||||
|
},
|
||||||
|
method='DELETE',
|
||||||
|
),
|
||||||
|
) as api_req:
|
||||||
|
api_res = api_req.read()
|
||||||
|
except urllib.error.HTTPError as e:
|
||||||
|
log.error(f'Grafana unsilence returned code {e.code}')
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
_t1 = time()
|
_t1 = time()
|
||||||
log.info(f'Program ended successfully after {timedelta(seconds=_t1-_t0)}')
|
log.info(f'Program ended successfully after {timedelta(seconds=_t1-_t0)}')
|
||||||
|
|
Reference in a new issue