#!/usr/bin/env python3
# vim: set fileencoding=utf-8
# pylint:disable=line-too-long
r""":mod:`cncopgetprofile` - download jobs
##########################################
.. module:: cncopgetprofile
:synopsis: download jobs
.. moduleauthor:: Jim Carroll <jim@carroll.net>
.. image:: ../appicons/cncopgetprofile.png
Overview
********
Connect to the cncop network and download the backup jobs assigned to this
device. Also check for software updates, and if updates exist, notify the
:mod:`ccs3updater` module.
It's intended that the ``cncopgetprofile`` module be run hourly to check
for changes to the backup jobs assigned to this device. The changes are
download from the cncop network and stored locally in
:file:`Config/backupjobs.db` database.
Software updates
================
With each connection to the cncop network, the ``cncopgetprofile`` module also
checks for software updates. If any updates exists, a message is sent to the
:mod:`ccs3updater` process which is listening on UDP port 9998.
The auto software update process can be disabled in the
:file:`Config/cncop3.ini`, by setting ``autoupdate`` to Off in the
``[version]`` section.
Open firewall request
=====================
Carroll-Net's network employs a strict set of firewall policies. Occasionally,
these policies interfere with a registered device performing backup services.
To detect and report when this happens, ``cncopgetprofile`` monitors policy
changes, and reports unintentional blockages to the cncop network.
Command line options
********************
*usage:* ``cncopgetprofile.py [-?] [-d] [-c CONFIG] [--noupdate] [--update]
[--recreate]``
Optional argument:
==================
.. option:: ?, -h, --help
Display help and exit
.. option:: -d, --debug
Generate diagnostic logging. The output is directed to the
:file:`Spool/Logs` folder and can be viewed using the cncop watcher
scripts.
.. option:: -c CONFIG, --config CONFIG
Alternate configuration file.
.. option:: --noupdate
Disable the check for software updates.
.. option:: --update
Force a check for software update, even if autoupdate is disabled in the
:file:`Config/cncop.ini` files.
.. option:: --recreate
Force backupjobs database to be recreated. Typically used to deploy schema
changes.
..
Copyright(c) 2013, Carroll-Net, Inc., All Rights Reserved"""
# pylint:enable=line-too-long
# ----------------------------------------------------------------------------
# Standard library imports
# ----------------------------------------------------------------------------
import argparse
import io
import datetime
import http.client
import logging
import logging.config
import os
import socket
import subprocess
import sys
import tempfile
import urllib
import xml.etree.ElementTree as ET # nosec
# ----------------------------------------------------------------------------
# Project imports
# ----------------------------------------------------------------------------
import cncop.identity
import cncop.mutex
import cncop.process
import cncop.profile
import cncop.settings
import cncop.utils
import cncop.web
import lib.backupjobdb
import lib.clogging
import lib.utils
# ----------------------------------------------------------------------------
# Module level initializations
# ----------------------------------------------------------------------------
__version__ = '3.0.1'
__author__ = 'Jim Carroll'
__email__ = 'jim@carroll.net'
__status__ = 'Production'
__copyright__ = 'Copyright(c) 2013, Carroll-Net, Inc., All Rights Reserved'
__icon__ = 'appicons/cncopgetprofile.ico'
AGENT = {'name': 'cncopgetprofile',
'copyright': __copyright__,
'version': __version__}
LOG = logging.getLogger(AGENT['name'])
LOG.setLevel(logging.INFO)
EPFX = 'CGP '
SPC9 = ' ' * 9
NOTIFY_PORT = 9998
ON_POSIX = 'posix' in sys.builtin_module_names
[docs]def run_cncopreporter():
r"""Run cncopreporter to upload reports to cncop network"""
app = lib.utils.appname('cncopreporter')
cmd = [sys.executable, app] if app.endswith('.py') else [app]
with subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, bufsize=0, close_fds=ON_POSIX) as task:
return cncop.process.drain_output([task])[0]
[docs]class CncOpGetProfile:
r"""Retrieve the profile for this device"""
def __init__(self, args):
self.args = args
cncop.settings.set_config_file(args.config)
ppath = cncop.settings.profile_path()
try:
kfname = os.path.join(ppath, 'identity.key')
kfile = cncop.identity.KeyFile(kfname)
pubkey = kfile.PublicKey()
except OSError as exc:
raise RuntimeError('Missing identity file %s: %s' %
(kfname, exc)) from None
self.profdb = cncop.profile.ProfileDb()
self.device = self.profdb.find_device_by_pubkey(pubkey)
if not self.device:
LOG.error(">>> ERROR: Could not find registered profile")
LOG.error(">>> Consider running 'cncopregistration.py'")
raise RuntimeError('Missing profile.db')
LOG.info('>>> %s Device role %s', EPFX, self.device.role)
self.web = cncop.web.Client(keyfile=kfile)
[docs] def check_firewall(self):
r"""Called to check/request firewall be opened"""
# Re-try the connect -- but we ONLY care about timeouts (anything other
# than tmo is something other than f/w)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.settimeout(5.0)
try:
sock.connect((self.web.host, 443))
except socket.timeout:
# Likely f/w block (or local internet outage)
pass
except Exception as exc: # pylint: disable=broad-except
LOG.error('Unexpected error: %s', str(exc))
return
org = self.profdb.get_organization(self.device.orgid)
url = ('/openfirewall/?'
'orgname=%s&'
'orgid=%s&'
'devicename=%s&'
'deviceid=%s' % (
urllib.parse.quote(org.orgname),
urllib.parse.quote(str(org.orgid)),
urllib.parse.quote(self.device.devicename),
urllib.parse.quote(str(self.device.deviceid))
))
doc = ET.Element('openfirewallrequest')
doc.append(self.device.to_xml())
msg = io.BytesIO()
msg.write(b"<?xml version='1.0' encoding='utf-8'>\n")
msg.write(ET.tostring(doc))
body = msg.getvalue()
msg.close()
try:
conn = http.client.HTTPConnection('www.carroll.net', timeout=60.0)
conn.request('POST', url, body)
conn.getresponse()
conn.close()
except Exception as exc: # pylint: disable=broad-except
LOG.error(str(exc))
[docs] def update_sw(self):
r"""Check whether there is new software for us"""
if not cncop.settings.get_autoupdate() and not self.args.update:
LOG.info('>>> %s Autoupdate disabled, skip update_sw check', EPFX)
return
try:
doc = self.web.get_latest_sw_version(self.device)
LOG.debug('%s', ET.tostring(doc))
except cncop.web.ProtocolError as exc:
LOG.warning('>>> %s WARNING No software available device.', EPFX)
LOG.debug('%s', exc)
return
swdoc = doc.find('software')
availableversion = swdoc.find('version').text
if availableversion == cncop.settings.get_currentversion():
LOG.info('%sAlready up to date', SPC9)
return
LOG.info('>>> %s New version %s available. Requesting update',
EPFX, availableversion)
msg = ET.tostring(doc)
# Let ccs3updater know there is updated software to d/l
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.sendto(msg, ('127.0.0.1', NOTIFY_PORT))
[docs] def update_jobs_db(self, doc):
r"""Extract jobs from xml doc, store in jobdb"""
LOG.info('>>> %s Save backupjobs, started %s', EPFX, lib.utils.utcstr())
fdn, tmpf = tempfile.mkstemp()
os.close(fdn)
with lib.backupjobdb.JobDb({'recreate': True, 'dsn': 'sqlite:///%s' %
tmpf}) as tmpdb:
for job in doc.findall('backupjob'):
buj = lib.backupjobdb.BackupJob.fromxml(job)
LOG.debug('%sAdd Job "%s"', SPC9, buj)
LOG.debug('%sDetails: %s', SPC9, buj.tostring())
tmpdb.session.add(buj)
for filt in job.findall('filters/filter'):
frec = lib.backupjobdb.Filter.fromxml(filt)
LOG.debug('%sAdd Filter "%s"', SPC9, filt)
LOG.debug('%sDetails: %s', SPC9, frec.tostring())
tmpdb.session.add(frec)
tmpdb.session.flush()
# Connect to main job's database and delete jobs & filters
with lib.backupjobdb.JobDb({'recreate': self.args.recreate}) \
as jobdb:
jobdb.session.query(lib.backupjobdb.BackupJob).delete()
jobdb.session.query(lib.backupjobdb.Filter).delete()
# Add downloaded jobs and filters
for job in tmpdb.findall_backupjobs():
jobdb.session.merge(job)
for filt in tmpdb.find_filters(job.jobid):
jobdb.session.merge(filt)
os.unlink(tmpf)
[docs] def Run(self):
r"""application driver"""
start = datetime.datetime.utcnow()
LOG.info('>>> %s Download Jobs, started %s', EPFX, lib.utils.utcstr())
try:
doc = self.web.get_device_jobs(self.device)
LOG.debug('%s', ET.tostring(doc, 'utf-8'))
except cncop.web.NetworkingError as exc:
LOG.error(">>> %s Network error, checking firewall", EPFX)
LOG.info(str(exc))
return self.check_firewall()
# Update the jobs database
self.update_jobs_db(doc)
# Run cncopreporter (to force delivery of cached logs)
LOG.info('>>> %s Run cncopreporter, started %s', EPFX, lib.utils.utcstr())
run_cncopreporter()
# Check for s/w updates
if not self.args.noupdate:
LOG.info('>>> %s Check for S/w updates, started %s',
EPFX, lib.utils.utcstr())
self.update_sw()
LOG.info('>>> %s Runtime %s',
EPFX, datetime.datetime.utcnow() - start)
return 0
[docs]def main():
r"""process main driver"""
# pylint: disable=too-many-return-statements
orig_config = cncop.settings.config_file()
parser = argparse.ArgumentParser(
add_help=False,
description='Download jobs from cncop network.')
parser.add_argument('-?', '-h', '--help', dest='help',
action='store_true', default=False,
help='Show this help message and exit')
parser.add_argument('-d', '--debug', action='store_true',
help='Increase logging level to DEBUG')
parser.add_argument('-c', '--config', default=orig_config,
help='Alternate configuration file.')
parser.add_argument('--noupdate', action='store_true',
help='Disable software updates')
parser.add_argument('--update', action='store_true',
help='Force check software update (even if disabled)')
parser.add_argument('--recreate', action='store_true',
help='Force backupjobs db to be recreated '
'(Used to change db schema).')
args = parser.parse_args()
if args.help:
parser.print_help()
return 0
cncop.settings.set_config_file(args.config)
mutex = cncop.mutex.Mutex('%s.lock' % AGENT['name'])
if not mutex.tryacquire():
print('>>> %s Another copy of %s is running.' % (EPFX, AGENT['name']))
return 0
if (result := lib.clogging.activate(AGENT, EPFX)):
return result
if args.debug:
LOG.setLevel(logging.DEBUG)
LOG.info('>>> %s Option: Debug enabled', EPFX)
if args.config != orig_config:
LOG.info(">>> %s Option: Config set to '%s'", EPFX, args.config)
if args.noupdate and args.update:
LOG.error('>>> %s ERROR: Cannot combine --noupdate and --update',
EPFX)
return -1
if args.noupdate:
LOG.info('>>> %s Option: Disable software updates', EPFX)
if args.update:
LOG.info('>>> %s Option: Force a check for software updates', EPFX)
if args.recreate:
LOG.info('>>> %s Option: Recreate backupjobs.db', EPFX)
try:
return CncOpGetProfile(args).Run()
except KeyboardInterrupt:
LOG.info('>>> %s Terminated by CTRL-C', EPFX)
except RuntimeError as exc:
LOG.error('>>> %s Terminated due to %s', EPFX, exc)
return -1
if __name__ == '__main__':
sys.exit(main())